micord/ervu-dashboard-etl
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

snils inn fix

Browse source
This commit is contained in:
adel.ka 2025-11-11 14:06:22 +03:00
parent f51a4d9ec8
commit 3a41351297
8 changed files with 865 additions and 838 deletions
Download patch file Download diff file Expand all files Collapse all files

288
mappings/info_recruits/citizen_tables/inn/parallel/inn_flow.hpl
View file

@ -21,50 +21,50 @@
</notepads> </notepads>
<order> <order>
<hop> <hop>
<from>Create job execution record</from> <from>inn_create_job</from>
<to>Table input</to> <to>inn_input</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Filter rows</from> <from>Filter rows</from>
<to>Change job status on success</to> <to>inn_success_job</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Detect empty stream</from> <from>Detect empty stream</from>
<to>Change job status on success</to> <to>inn_success_job</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Change job status on error</from> <from>inn_failure_job</from>
<to>Abort</to> <to>Abort</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Table input</from> <from>inn_output</from>
<to>inn_failure_job</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>inn_input</from>
<to>inn_output</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>inn_output</from>
<to>Identify last row in a stream</to> <to>Identify last row in a stream</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Identify last row in a stream</from> <from>Identify last row in a stream</from>
<to>Table output inn</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table output inn</from>
<to>Filter rows</to> <to>Filter rows</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Table output inn</from> <from>inn_input</from>
<to>Detect empty stream</to> <to>Detect empty stream</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop>
<from>Table output inn</from>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
</order> </order>
<transform> <transform>
<name>Abort</name> <name>Abort</name>
@ -83,104 +83,7 @@
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>1376</xloc> <xloc>1376</xloc>
<yloc>512</yloc> <yloc>752</yloc>
</GUI>
</transform>
<transform>
<name>Change job status on error</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
<argument>
<name>error_description</name>
</argument>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>Y</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'ERROR',
error_description = ?,
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1024</xloc>
<yloc>512</yloc>
</GUI>
</transform>
<transform>
<name>Change job status on success</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'SUCCESS',
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1632</xloc>
<yloc>160</yloc>
</GUI>
</transform>
<transform>
<name>Create job execution record</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>INSERT INTO etl.job_execution (id, job_name, status, execution_datetime, error_description, recruitment_id)
VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</sql>
<attributes/>
<GUI>
<xloc>288</xloc>
<yloc>320</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
@ -196,8 +99,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</partitioning> </partitioning>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>1632</xloc> <xloc>640</xloc>
<yloc>320</yloc> <yloc>352</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
@ -229,11 +132,11 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</value> </value>
</condition> </condition>
</compare> </compare>
<send_true_to>Change job status on success</send_true_to> <send_true_to>inn_success_job</send_true_to>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>1024</xloc> <xloc>1024</xloc>
<yloc>160</yloc> <yloc>128</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
@ -250,12 +153,77 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<resultfieldname>last_row</resultfieldname> <resultfieldname>last_row</resultfieldname>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>768</xloc> <xloc>1024</xloc>
<yloc>320</yloc> <yloc>352</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>Table input</name> <name>inn_create_job</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>INSERT INTO etl.job_execution (id, job_name, status, execution_datetime, error_description, recruitment_id)
VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</sql>
<attributes/>
<GUI>
<xloc>288</xloc>
<yloc>560</yloc>
</GUI>
</transform>
<transform>
<name>inn_failure_job</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
<argument>
<name>error_description</name>
</argument>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>Y</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'ERROR',
error_description = ?,
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1024</xloc>
<yloc>752</yloc>
</GUI>
</transform>
<transform>
<name>inn_input</name>
<type>TableInput</type> <type>TableInput</type>
<description/> <description/>
<distribute>N</distribute> <distribute>N</distribute>
@ -267,44 +235,48 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</partitioning> </partitioning>
<connection>ervu-dashboard</connection> <connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row> <execute_each_row>N</execute_each_row>
<limit/>
<sql>WITH <sql>WITH
base as ( base as (
SELECT SELECT
ri.recruit_id, ri.recruit_id,
ri.info ri.info,
FROM ervu_dashboard.recruits_info ri REGEXP_REPLACE(ri.info->'svedFL'->'svedBS'->'inn'->'aktINN'->>'inn', '[^0-9]', '', 'g') as cleaned_akt_inn,
join ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id AND '${IDM_ID}' != '' -- Проверка на пустую строку REGEXP_REPLACE(ri.info->'svedFL'->'svedBS'->'inn'->'predINN'->>'inn', '[^0-9]', '', 'g') as cleaned_pred_inn
AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) FROM ervu_dashboard.recruits_info ri
JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id
WHERE
'${IDM_ID}' != ''
AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
) )
SELECT SELECT
recruit_id, recruit_id,
info->'svedFL'->'svedBS'->'inn'->'aktINN'->>'id' AS source_id, info->'svedFL'->'svedBS'->'inn'->'aktINN'->>'id' AS source_id,
to_date(info->'svedFL'->'svedBS'->'inn'->'aktINN'->>'dataSved', 'YYYY-MM-DD') AS source_update_date, to_date(info->'svedFL'->'svedBS'->'inn'->'aktINN'->>'dataSved', 'YYYY-MM-DD') AS source_update_date,
REGEXP_REPLACE(info->'svedFL'->'svedBS'->'inn'->'aktINN'->>'inn', '[^0-9]', '', 'g') AS inn, cleaned_akt_inn AS inn,
true as actual true as actual
FROM base FROM base
WHERE LENGTH(REGEXP_REPLACE(info->'svedFL'->'svedBS'->'inn'->'aktINN'->>'inn', '[^0-9]', '', 'g')) = 12 WHERE LENGTH(cleaned_akt_inn) = 12
UNION ALL UNION ALL
SELECT SELECT
recruit_id, recruit_id,
info->'svedFL'->'svedBS'->'inn'->'predINN'->>'id' AS source_id, info->'svedFL'->'svedBS'->'inn'->'predINN'->>'id' AS source_id,
to_date(info->'svedFL'->'svedBS'->'inn'->'predINN'->>'dataSved', 'YYYY-MM-DD') AS source_update_date, to_date(info->'svedFL'->'svedBS'->'inn'->'predINN'->>'dataSved', 'YYYY-MM-DD') AS source_update_date,
REGEXP_REPLACE(info->'svedFL'->'svedBS'->'inn'->'predINN'->>'inn', '[^0-9]', '', 'g') AS inn, cleaned_pred_inn AS inn,
false as actual false as actual
FROM base FROM base
WHERE LENGTH(REGEXP_REPLACE(info->'svedFL'->'svedBS'->'inn'->'predINN'->>'inn', '[^0-9]', '', 'g')) = 12</sql> WHERE LENGTH(cleaned_pred_inn) = 12</sql>
<variables_active>Y</variables_active> <variables_active>Y</variables_active>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>496</xloc> <xloc>640</xloc>
<yloc>320</yloc> <yloc>560</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>Table output inn</name> <name>inn_output</name>
<type>TableOutput</type> <type>TableOutput</type>
<description/> <description/>
<distribute>N</distribute> <distribute>N</distribute>
@ -314,7 +286,7 @@ WHERE LENGTH(REGEXP_REPLACE(info->'svedFL'->'svedBS'->'inn'->'predINN'->>'inn',
<method>none</method> <method>none</method>
<schema_name/> <schema_name/>
</partitioning> </partitioning>
<commit>1000</commit> <commit>10000</commit>
<connection>ervu-dashboard</connection> <connection>ervu-dashboard</connection>
<fields> <fields>
<field> <field>
@ -354,13 +326,45 @@ WHERE LENGTH(REGEXP_REPLACE(info->'svedFL'->'svedBS'->'inn'->'predINN'->>'inn',
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>1024</xloc> <xloc>1024</xloc>
<yloc>320</yloc> <yloc>560</yloc>
</GUI>
</transform>
<transform>
<name>inn_success_job</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'SUCCESS',
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>640</xloc>
<yloc>128</yloc>
</GUI> </GUI>
</transform> </transform>
<transform_error_handling> <transform_error_handling>
<error> <error>
<source_transform>Table output inn</source_transform> <source_transform>inn_output</source_transform>
<target_transform>Change job status on error</target_transform> <target_transform>inn_failure_job</target_transform>
<is_enabled>Y</is_enabled> <is_enabled>Y</is_enabled>
<nr_valuename/> <nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename> <descriptions_valuename>error_description</descriptions_valuename>

384
mappings/info_recruits/citizen_tables/inn/parallel/inn_flow_delta.hpl
View file

@ -21,48 +21,48 @@
</notepads> </notepads>
<order> <order>
<hop> <hop>
<from>Create job execution record</from> <from>inn_create_job</from>
<to>Table input</to> <to>inn_input</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<to>Identify last row in a stream</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Detect empty stream</from> <from>Detect empty stream</from>
<to>Change job status on success</to> <to>inn_success_job</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Filter rows</from> <from>Filter rows</from>
<to>Change job status on success</to> <to>inn_success_job</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Change job status on error</from> <from>inn_failure_job</from>
<to>Abort</to> <to>Abort</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Identify last row in a stream</from> <from>inn_insert_or_update</from>
<to>Insert / update inn</to> <to>inn_failure_job</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Insert / update inn</from> <from>inn_input</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Insert / update inn</from>
<to>Detect empty stream</to> <to>Detect empty stream</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Insert / update inn</from> <from>inn_insert_or_update</from>
<to>Change job status on error</to> <to>Identify last row in a stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>inn_input</from>
<to>inn_insert_or_update</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
</order> </order>
@ -83,105 +83,7 @@
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>1328</xloc> <xloc>1328</xloc>
<yloc>400</yloc> <yloc>544</yloc>
</GUI>
</transform>
<transform>
<name>Change job status on error</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
<argument>
<name>error_description</name>
</argument>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>Y</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'DELTA_ERROR',
error_description = ?
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1104</xloc>
<yloc>400</yloc>
</GUI>
</transform>
<transform>
<name>Change job status on success</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'DELTA_SUCCESS'
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1408</xloc>
<yloc>64</yloc>
</GUI>
</transform>
<transform>
<name>Create job execution record</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET
status = 'DELTA_PROCESSING',
execution_datetime = DEFAULT,
error_description = NULL
where job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';</sql>
<attributes/>
<GUI>
<xloc>336</xloc>
<yloc>224</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
@ -197,8 +99,8 @@ and recruitment_id = '${IDM_ID}';</sql>
</partitioning> </partitioning>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>1408</xloc> <xloc>544</xloc>
<yloc>224</yloc> <yloc>176</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
@ -230,11 +132,11 @@ and recruitment_id = '${IDM_ID}';</sql>
</value> </value>
</condition> </condition>
</compare> </compare>
<send_true_to>Change job status on success</send_true_to> <send_true_to>inn_success_job</send_true_to>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>1104</xloc> <xloc>1104</xloc>
<yloc>64</yloc> <yloc>0</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
@ -251,12 +153,152 @@ and recruitment_id = '${IDM_ID}';</sql>
<resultfieldname>last_row</resultfieldname> <resultfieldname>last_row</resultfieldname>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>816</xloc> <xloc>1104</xloc>
<yloc>224</yloc> <yloc>176</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>Insert / update inn</name> <name>inn_create_job</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET
status = 'DELTA_PROCESSING',
execution_datetime = DEFAULT,
error_description = NULL
where job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';</sql>
<attributes/>
<GUI>
<xloc>336</xloc>
<yloc>368</yloc>
</GUI>
</transform>
<transform>
<name>inn_failure_job</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
<argument>
<name>error_description</name>
</argument>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>Y</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'DELTA_ERROR',
error_description = ?
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1104</xloc>
<yloc>544</yloc>
</GUI>
</transform>
<transform>
<name>inn_input</name>
<type>TableInput</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>WITH base AS (
SELECT
ri.recruit_id,
ri.info,
ri.info->'svedFL'->'svedBS'->'inn'->'aktINN' AS akt_inn,
ri.info->'svedFL'->'svedBS'->'inn'->'predINN' AS pred_inn,
ri.info->'svedFL'->'svedBS'->'inn'->'aktINN'->>'dataSved' AS akt_data_sved,
REGEXP_REPLACE(ri.info->'svedFL'->'svedBS'->'inn'->'aktINN'->>'inn', '[^0-9]', '', 'g') AS akt_inn_cleaned,
ri.info->'svedFL'->'svedBS'->'inn'->'predINN'->>'dataSved' AS pred_data_sved,
REGEXP_REPLACE(ri.info->'svedFL'->'svedBS'->'inn'->'predINN'->>'inn', '[^0-9]', '', 'g') AS pred_inn_cleaned
FROM ervu_dashboard.recruits_info ri
JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id
WHERE
'${IDM_ID}' != ''
AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
),
inn_data AS (
SELECT
recruit_id,
akt_inn->>'id' AS source_id,
to_date(akt_data_sved, 'YYYY-MM-DD') AS source_update_date,
akt_inn_cleaned AS inn,
true AS actual,
'aktINN' AS inn_type
FROM base
WHERE
akt_data_sved IS NOT NULL
AND to_date(akt_data_sved, 'YYYY-MM-DD') > '${MAX_SOURCE_UPDATE_DATE}'::date
AND LENGTH(akt_inn_cleaned) = 12
UNION ALL
SELECT
recruit_id,
pred_inn->>'id' AS source_id,
to_date(pred_data_sved, 'YYYY-MM-DD') AS source_update_date,
pred_inn_cleaned AS inn,
false AS actual,
'predINN' AS inn_type
FROM base
WHERE
pred_data_sved IS NOT NULL
AND to_date(pred_data_sved, 'YYYY-MM-DD') > '${MAX_SOURCE_UPDATE_DATE}'::date
AND LENGTH(pred_inn_cleaned) = 12
)
SELECT
recruit_id,
source_id,
source_update_date,
inn,
actual
FROM inn_data;</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
<xloc>544</xloc>
<yloc>368</yloc>
</GUI>
</transform>
<transform>
<name>inn_insert_or_update</name>
<type>InsertUpdate</type> <type>InsertUpdate</type>
<description/> <description/>
<distribute>N</distribute> <distribute>N</distribute>
@ -306,94 +348,44 @@ and recruitment_id = '${IDM_ID}';</sql>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>1104</xloc> <xloc>1104</xloc>
<yloc>224</yloc> <yloc>368</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>Table input</name> <name>inn_success_job</name>
<type>TableInput</type> <type>ExecSql</type>
<description/> <description/>
<distribute>N</distribute> <distribute>Y</distribute>
<custom_distribution/> <custom_distribution/>
<copies>1</copies> <copies>1</copies>
<partitioning> <partitioning>
<method>none</method> <method>none</method>
<schema_name/> <schema_name/>
</partitioning> </partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard</connection> <connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row> <execute_each_row>Y</execute_each_row>
<sql>WITH base AS ( <quoteString>N</quoteString>
SELECT <replace_variables>Y</replace_variables>
ri.recruit_id, <set_params>N</set_params>
ri.info, <single_statement>N</single_statement>
ri.info->'svedFL'->'svedBS'->'inn'->'aktINN' AS akt_inn, <sql>UPDATE etl.job_execution
ri.info->'svedFL'->'svedBS'->'inn'->'predINN' AS pred_inn SET status = 'DELTA_SUCCESS'
FROM ervu_dashboard.recruits_info ri WHERE job_name = '${JOB_NAME}'
JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id and recruitment_id = '${IDM_ID}';
WHERE
'${IDM_ID}' != '' </sql>
AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}'))
AND (
(
(ri.info->'svedFL'->'svedBS'->'inn'->'aktINN'->>'dataSved') IS NOT NULL
AND to_date(ri.info->'svedFL'->'svedBS'->'inn'->'aktINN'->>'dataSved', 'YYYY-MM-DD') > '${MAX_SOURCE_UPDATE_DATE}'::date
AND LENGTH(REGEXP_REPLACE(ri.info->'svedFL'->'svedBS'->'inn'->'aktINN'->>'inn', '[^0-9]', '', 'g')) = 12
)
OR
(
(ri.info->'svedFL'->'svedBS'->'inn'->'predINN'->>'dataSved') IS NOT NULL
AND to_date(ri.info-&gt;'svedFL'->'svedBS'->'inn'->'predINN'->>'dataSved', 'YYYY-MM-DD') > '${MAX_SOURCE_UPDATE_DATE}'::date
AND LENGTH(REGEXP_REPLACE(ri.info->'svedFL'->'svedBS'->'inn'->'predINN'->>'inn', '[^0-9]', '', 'g')) = 12
)
)
),
inn_data AS (
SELECT
recruit_id,
akt_inn->>'id' AS source_id,
to_date(akt_inn->>'dataSved', 'YYYY-MM-DD') AS source_update_date,
REGEXP_REPLACE(akt_inn->>'inn', '[^0-9]', '', 'g') AS inn,
true AS actual,
'aktINN' AS inn_type
FROM base
WHERE
(akt_inn->>'dataSved') IS NOT NULL
AND to_date(akt_inn->>'dataSved', 'YYYY-MM-DD') > '${MAX_SOURCE_UPDATE_DATE}'::date
AND LENGTH(REGEXP_REPLACE(akt_inn->>'inn', '[^0-9]', '', 'g')) = 12
UNION ALL
SELECT
recruit_id,
pred_inn->>'id' AS source_id,
to_date(pred_inn->>'dataSved', 'YYYY-MM-DD') AS source_update_date,
REGEXP_REPLACE(pred_inn->>'inn', '[^0-9]', '', 'g') AS inn,
false AS actual,
'predINN' AS inn_type
FROM base
WHERE
(pred_inn->>'dataSved') IS NOT NULL
AND to_date(pred_inn->>'dataSved', 'YYYY-MM-DD') > '${MAX_SOURCE_UPDATE_DATE}'::date
AND LENGTH(REGEXP_REPLACE(pred_inn->>'inn', '[^0-9]', '', 'g')) = 12
)
SELECT
recruit_id,
source_id,
source_update_date,
inn,
actual
FROM inn_data;</sql>
<variables_active>Y</variables_active>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>544</xloc> <xloc>544</xloc>
<yloc>224</yloc> <yloc>0</yloc>
</GUI> </GUI>
</transform> </transform>
<transform_error_handling> <transform_error_handling>
<error> <error>
<source_transform>Insert / update inn</source_transform> <source_transform>inn_insert_or_update</source_transform>
<target_transform>Change job status on error</target_transform> <target_transform>inn_failure_job</target_transform>
<is_enabled>Y</is_enabled> <is_enabled>Y</is_enabled>
<nr_valuename/> <nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename> <descriptions_valuename>error_description</descriptions_valuename>

294
mappings/info_recruits/citizen_tables/inn/parallel/inn_flow_repeat.hpl
View file

@ -21,48 +21,48 @@
</notepads> </notepads>
<order> <order>
<hop> <hop>
<from>Create job execution record</from> <from>inn_create_job</from>
<to>Table input</to> <to>inn_input</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<to>Identify last row in a stream</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Filter rows</from> <from>Filter rows</from>
<to>Change job status on success</to> <to>inn_success_job</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Detect empty stream</from> <from>Detect empty stream</from>
<to>Change job status on success</to> <to>inn_success_job</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Change job status on error</from> <from>inn_failure_job</from>
<to>Abort</to> <to>Abort</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Identify last row in a stream</from> <from>inn_insert_or_update</from>
<to>Insert / update inn</to> <to>inn_failure_job</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Insert / update inn</from> <from>inn_input</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Insert / update inn</from>
<to>Detect empty stream</to> <to>Detect empty stream</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Insert / update inn</from> <from>inn_input</from>
<to>Change job status on error</to> <to>inn_insert_or_update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>inn_insert_or_update</from>
<to>Identify last row in a stream </to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream </from>
<to>Filter rows</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
</order> </order>
@ -83,12 +83,12 @@
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>1296</xloc> <xloc>1296</xloc>
<yloc>400</yloc> <yloc>512</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>Change job status on error</name> <name>Detect empty stream</name>
<type>ExecSql</type> <type>DetectEmptyStream</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>Y</distribute>
<custom_distribution/> <custom_distribution/>
@ -97,33 +97,51 @@
<method>none</method> <method>none</method>
<schema_name/> <schema_name/>
</partitioning> </partitioning>
<arguments> <attributes/>
<argument> <GUI>
<name>error_description</name> <xloc>480</xloc>
</argument> <yloc>160</yloc>
</arguments> </GUI>
<connection>ervu-dashboard</connection> </transform>
<execute_each_row>Y</execute_each_row> <transform>
<quoteString>N</quoteString> <name>Filter rows</name>
<replace_variables>Y</replace_variables> <type>FilterRows</type>
<set_params>Y</set_params> <description/>
<single_statement>N</single_statement> <distribute>Y</distribute>
<sql>UPDATE etl.job_execution <custom_distribution/>
SET status = 'ERROR', <copies>1</copies>
error_description = ? <partitioning>
WHERE job_name = '${JOB_NAME}' <method>none</method>
and recruitment_id = '${IDM_ID}'; <schema_name/>
</partitioning>
</sql> <compare>
<condition>
<conditions>
</conditions>
<function>=</function>
<leftvalue>last_row</leftvalue>
<negated>N</negated>
<operator>-</operator>
<value>
<isnull>N</isnull>
<length>-1</length>
<name>constant</name>
<precision>-1</precision>
<text>Y</text>
<type>Boolean</type>
</value>
</condition>
</compare>
<send_true_to>inn_success_job</send_true_to>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>1008</xloc> <xloc>1008</xloc>
<yloc>400</yloc> <yloc>16</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>Change job status on success</name> <name>Identify last row in a stream </name>
<type>ExecSql</type> <type>DetectLastRow</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>Y</distribute>
<custom_distribution/> <custom_distribution/>
@ -132,28 +150,15 @@ and recruitment_id = '${IDM_ID}';
<method>none</method> <method>none</method>
<schema_name/> <schema_name/>
</partitioning> </partitioning>
<arguments> <resultfieldname>last_row</resultfieldname>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'SUCCESS'
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>1440</xloc> <xloc>1008</xloc>
<yloc>48</yloc> <yloc>160</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>Create job execution record</name> <name>inn_create_job</name>
<type>ExecSql</type> <type>ExecSql</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>Y</distribute>
@ -196,12 +201,12 @@ DO UPDATE SET
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>272</xloc> <xloc>272</xloc>
<yloc>208</yloc> <yloc>320</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>Detect empty stream</name> <name>inn_failure_job</name>
<type>DetectEmptyStream</type> <type>ExecSql</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>Y</distribute>
<custom_distribution/> <custom_distribution/>
@ -210,68 +215,85 @@ DO UPDATE SET
<method>none</method> <method>none</method>
<schema_name/> <schema_name/>
</partitioning> </partitioning>
<attributes/> <arguments>
<GUI> <argument>
<xloc>1440</xloc> <name>error_description</name>
<yloc>208</yloc> </argument>
</GUI> </arguments>
</transform> <connection>ervu-dashboard</connection>
<transform> <execute_each_row>Y</execute_each_row>
<name>Filter rows</name> <quoteString>N</quoteString>
<type>FilterRows</type> <replace_variables>Y</replace_variables>
<description/> <set_params>Y</set_params>
<distribute>Y</distribute> <single_statement>N</single_statement>
<custom_distribution/> <sql>UPDATE etl.job_execution
<copies>1</copies> SET status = 'ERROR',
<partitioning> error_description = ?
<method>none</method> WHERE job_name = '${JOB_NAME}'
<schema_name/> and recruitment_id = '${IDM_ID}';
</partitioning>
<compare> </sql>
<condition>
<conditions>
</conditions>
<function>=</function>
<leftvalue>last_row</leftvalue>
<negated>N</negated>
<operator>-</operator>
<value>
<isnull>N</isnull>
<length>-1</length>
<name>constant</name>
<precision>-1</precision>
<text>Y</text>
<type>Boolean</type>
</value>
</condition>
</compare>
<send_true_to>Change job status on success</send_true_to>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>1008</xloc> <xloc>1008</xloc>
<yloc>48</yloc> <yloc>512</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>Identify last row in a stream</name> <name>inn_input</name>
<type>DetectLastRow</type> <type>TableInput</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>N</distribute>
<custom_distribution/> <custom_distribution/>
<copies>1</copies> <copies>1</copies>
<partitioning> <partitioning>
<method>none</method> <method>none</method>
<schema_name/> <schema_name/>
</partitioning> </partitioning>
<resultfieldname>last_row</resultfieldname> <connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<sql>WITH
base as (
SELECT
ri.recruit_id,
ri.info,
REGEXP_REPLACE(ri.info->'svedFL'->'svedBS'->'inn'->'aktINN'->>'inn', '[^0-9]', '', 'g') as cleaned_akt_inn,
REGEXP_REPLACE(ri.info->'svedFL'->'svedBS'->'inn'->'predINN'->>'inn', '[^0-9]', '', 'g') as cleaned_pred_inn
FROM ervu_dashboard.recruits_info ri
JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id
WHERE
'${IDM_ID}' != ''
AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
)
SELECT
recruit_id,
info->'svedFL'->'svedBS'->'inn'->'aktINN'->>'id' AS source_id,
to_date(info->'svedFL'->'svedBS'->'inn'->'aktINN'->>'dataSved', 'YYYY-MM-DD') AS source_update_date,
cleaned_akt_inn AS inn,
true as actual
FROM base
WHERE LENGTH(cleaned_akt_inn) = 12
UNION ALL
SELECT
recruit_id,
info->'svedFL'->'svedBS'->'inn'->'predINN'->>'id' AS source_id,
to_date(info->'svedFL'->'svedBS'->'inn'->'predINN'->>'dataSved', 'YYYY-MM-DD') AS source_update_date,
cleaned_pred_inn AS inn,
false as actual
FROM base
WHERE LENGTH(cleaned_pred_inn) = 12</sql>
<variables_active>Y</variables_active>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>672</xloc> <xloc>480</xloc>
<yloc>208</yloc> <yloc>320</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>Insert / update inn</name> <name>inn_insert_or_update</name>
<type>InsertUpdate</type> <type>InsertUpdate</type>
<description/> <description/>
<distribute>N</distribute> <distribute>N</distribute>
@ -288,7 +310,6 @@ DO UPDATE SET
<condition>=</condition> <condition>=</condition>
<field>source_id</field> <field>source_id</field>
<name>source_id</name> <name>source_id</name>
<name2/>
</key> </key>
<schema>ervu_dashboard</schema> <schema>ervu_dashboard</schema>
<table>inn</table> <table>inn</table>
@ -322,61 +343,44 @@ DO UPDATE SET
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>1008</xloc> <xloc>1008</xloc>
<yloc>208</yloc> <yloc>320</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>Table input</name> <name>inn_success_job</name>
<type>TableInput</type> <type>ExecSql</type>
<description/> <description/>
<distribute>N</distribute> <distribute>Y</distribute>
<custom_distribution/> <custom_distribution/>
<copies>1</copies> <copies>1</copies>
<partitioning> <partitioning>
<method>none</method> <method>none</method>
<schema_name/> <schema_name/>
</partitioning> </partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard</connection> <connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row> <execute_each_row>Y</execute_each_row>
<sql>WITH <quoteString>N</quoteString>
base as ( <replace_variables>Y</replace_variables>
SELECT <set_params>N</set_params>
ri.recruit_id, <single_statement>N</single_statement>
ri.info <sql>UPDATE etl.job_execution
FROM ervu_dashboard.recruits_info ri SET status = 'SUCCESS'
join ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id AND '${IDM_ID}' != '' -- Проверка на пустую строку WHERE job_name = '${JOB_NAME}'
AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) and recruitment_id = '${IDM_ID}';
)
SELECT
recruit_id,
info->'svedFL'->'svedBS'->'inn'->'aktINN'->>'id' AS source_id,
to_date(info->'svedFL'->'svedBS'->'inn'->'aktINN'->>'dataSved', 'YYYY-MM-DD') AS source_update_date,
REGEXP_REPLACE(info->'svedFL'->'svedBS'->'inn'->'aktINN'->>'inn', '[^0-9]', '', 'g') AS inn,
true as actual
FROM base
WHERE LENGTH(REGEXP_REPLACE(info->'svedFL'->'svedBS'->'inn'->'aktINN'->>'inn', '[^0-9]', '', 'g')) = 12
UNION ALL </sql>
SELECT
recruit_id,
info->'svedFL'->'svedBS'->'inn'->'predINN'->>'id' AS source_id,
to_date(info->'svedFL'->'svedBS'->'inn'->'predINN'->>'dataSved', 'YYYY-MM-DD') AS source_update_date,
REGEXP_REPLACE(info->'svedFL'->'svedBS'->'inn'->'predINN'->>'inn', '[^0-9]', '', 'g') AS inn,
false as actual
FROM base
WHERE LENGTH(REGEXP_REPLACE(info->'svedFL'->'svedBS'->'inn'->'predINN'->>'inn', '[^0-9]', '', 'g')) = 12</sql>
<variables_active>Y</variables_active>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>480</xloc> <xloc>480</xloc>
<yloc>208</yloc> <yloc>16</yloc>
</GUI> </GUI>
</transform> </transform>
<transform_error_handling> <transform_error_handling>
<error> <error>
<source_transform>Insert / update inn</source_transform> <source_transform>inn_insert_or_update</source_transform>
<target_transform>Change job status on error</target_transform> <target_transform>inn_failure_job</target_transform>
<is_enabled>Y</is_enabled> <is_enabled>Y</is_enabled>
<nr_valuename/> <nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename> <descriptions_valuename>error_description</descriptions_valuename>

302
mappings/info_recruits/citizen_tables/snils/parallel/snils_flow.hpl
View file

@ -21,48 +21,48 @@
</notepads> </notepads>
<order> <order>
<hop> <hop>
<from>Create job execution record</from> <from>snils_create_job</from>
<to>Table input</to> <to>snils_input</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Filter rows</from> <from>Filter rows</from>
<to>Change job status on success</to> <to>snils_success_job</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Detect empty stream</from> <from>Detect empty stream</from>
<to>Change job status on success</to> <to>snils_success_job</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Change job status on error</from> <from>snils_failure_job</from>
<to>Abort</to> <to>Abort</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Table input</from> <from>snils_output</from>
<to>Identify last row in a stream</to> <to>snils_failure_job</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Identify last row in a stream</from> <from>snils_input</from>
<to>Table output</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table output</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table output</from>
<to>Detect empty stream</to> <to>Detect empty stream</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Table output</from> <from>snils_output</from>
<to>Change job status on error</to> <to>Identify last row in a stream </to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream </from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>snils_input</from>
<to>snils_output</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
</order> </order>
@ -83,104 +83,7 @@
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>1376</xloc> <xloc>1376</xloc>
<yloc>512</yloc> <yloc>640</yloc>
</GUI>
</transform>
<transform>
<name>Change job status on error</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
<argument>
<name>error_description</name>
</argument>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>Y</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'ERROR',
error_description = ?,
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1024</xloc>
<yloc>512</yloc>
</GUI>
</transform>
<transform>
<name>Change job status on success</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'SUCCESS',
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1632</xloc>
<yloc>160</yloc>
</GUI>
</transform>
<transform>
<name>Create job execution record</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>INSERT INTO etl.job_execution (id, job_name, status, execution_datetime, error_description, recruitment_id)
VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</sql>
<attributes/>
<GUI>
<xloc>288</xloc>
<yloc>320</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
@ -196,8 +99,8 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</partitioning> </partitioning>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>1632</xloc> <xloc>496</xloc>
<yloc>320</yloc> <yloc>240</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
@ -229,15 +132,15 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</value> </value>
</condition> </condition>
</compare> </compare>
<send_true_to>Change job status on success</send_true_to> <send_true_to>snils_success_job</send_true_to>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>1024</xloc> <xloc>1024</xloc>
<yloc>160</yloc> <yloc>64</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>Identify last row in a stream</name> <name>Identify last row in a stream </name>
<type>DetectLastRow</type> <type>DetectLastRow</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>Y</distribute>
@ -250,12 +153,77 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<resultfieldname>last_row</resultfieldname> <resultfieldname>last_row</resultfieldname>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>752</xloc> <xloc>1024</xloc>
<yloc>320</yloc> <yloc>240</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>Table input</name> <name>snils_create_job</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>INSERT INTO etl.job_execution (id, job_name, status, execution_datetime, error_description, recruitment_id)
VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</sql>
<attributes/>
<GUI>
<xloc>288</xloc>
<yloc>448</yloc>
</GUI>
</transform>
<transform>
<name>snils_failure_job</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
<argument>
<name>error_description</name>
</argument>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>Y</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'ERROR',
error_description = ?,
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>1024</xloc>
<yloc>640</yloc>
</GUI>
</transform>
<transform>
<name>snils_input</name>
<type>TableInput</type> <type>TableInput</type>
<description/> <description/>
<distribute>N</distribute> <distribute>N</distribute>
@ -267,44 +235,52 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</partitioning> </partitioning>
<connection>ervu-dashboard</connection> <connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row> <execute_each_row>N</execute_each_row>
<limit/>
<sql>WITH <sql>WITH
base as ( base as (
SELECT SELECT
ri.recruit_id, ri.recruit_id,
ri.info ri.info,
FROM ervu_dashboard.recruits_info ri ri.info->'svedFL'->'svedBS'->'snils'->'aktSNILS'->>'id' AS akt_source_id,
join ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id AND '${IDM_ID}' != '' -- Проверка на пустую строку ri.info->'svedFL'->'svedBS'->'snils'->'aktSNILS'->>'dataSved' AS akt_data_sved,
AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) REGEXP_REPLACE(ri.info->'svedFL'->'svedBS'->'snils'->'aktSNILS'->>'snils', '[^0-9]', '', 'g') AS akt_snils_cleaned,
AND '${M_R_CR_DATE}'::timestamp >= ri.created_at ri.info->'svedFL'->'svedBS'->'snils'->'predSNILS'->>'id' AS pred_source_id,
ri.info->'svedFL'->'svedBS'->'snils'->'predSNILS'->>'dataSved' AS pred_data_sved,
REGEXP_REPLACE(ri.info->'svedFL'->'svedBS'->'snils'->'predSNILS'->>'snils', '[^0-9]', '', 'g') AS pred_snils_cleaned
FROM ervu_dashboard.recruits_info ri
JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id
WHERE
'${IDM_ID}' != ''
AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
) )
SELECT SELECT
recruit_id, recruit_id,
info->'svedFL'->'svedBS'->'snils'->'aktSNILS'->>'id' AS source_id, akt_source_id AS source_id,
to_date(info->'svedFL'->'svedBS'->'snils'->'aktSNILS'->>'dataSved', 'YYYY-MM-DD') AS source_update_date, to_date(akt_data_sved, 'YYYY-MM-DD') AS source_update_date,
REGEXP_REPLACE(info->'svedFL'->'svedBS'->'snils'->'aktSNILS'->>'snils', '[^0-9]', '', 'g') AS snils, akt_snils_cleaned AS snils,
true AS actual true AS actual
FROM base FROM base
WHERE LENGTH(REGEXP_REPLACE(info->'svedFL'->'svedBS'->'snils'->'aktSNILS'->>'snils', '[^0-9]', '', 'g')) = 11 WHERE LENGTH(akt_snils_cleaned) = 11
UNION ALL UNION ALL
SELECT SELECT
recruit_id, recruit_id,
info->'svedFL'->'svedBS'->'snils'->'predSNILS'->>'id' AS source_id, pred_source_id AS source_id,
to_date(info->'svedFL'->'svedBS'->'snils'->'predSNILS'->>'dataSved', 'YYYY-MM-DD') AS source_update_date, to_date(pred_data_sved, 'YYYY-MM-DD') AS source_update_date,
REGEXP_REPLACE(info->'svedFL'->'svedBS'->'snils'->'predSNILS'->>'snils', '[^0-9]', '', 'g') AS snils, pred_snils_cleaned AS snils,
false AS actual false AS actual
FROM base FROM base
WHERE LENGTH(REGEXP_REPLACE(info->'svedFL'->'svedBS'->'snils'->'predSNILS'->>'snils', '[^0-9]', '', 'g')) = 11</sql> WHERE LENGTH(pred_snils_cleaned) = 11</sql>
<variables_active>Y</variables_active> <variables_active>Y</variables_active>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>496</xloc> <xloc>496</xloc>
<yloc>320</yloc> <yloc>448</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>Table output</name> <name>snils_output</name>
<type>TableOutput</type> <type>TableOutput</type>
<description/> <description/>
<distribute>N</distribute> <distribute>N</distribute>
@ -314,7 +290,7 @@ WHERE LENGTH(REGEXP_REPLACE(info->'svedFL'->'svedBS'->'snils'->'predSNILS'->>'sn
<method>none</method> <method>none</method>
<schema_name/> <schema_name/>
</partitioning> </partitioning>
<commit>1000</commit> <commit>10000</commit>
<connection>ervu-dashboard</connection> <connection>ervu-dashboard</connection>
<fields> <fields>
<field> <field>
@ -354,13 +330,45 @@ WHERE LENGTH(REGEXP_REPLACE(info->'svedFL'->'svedBS'->'snils'->'predSNILS'->>'sn
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>1024</xloc> <xloc>1024</xloc>
<yloc>320</yloc> <yloc>448</yloc>
</GUI>
</transform>
<transform>
<name>snils_success_job</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>UPDATE etl.job_execution
SET status = 'SUCCESS',
execution_end_datetime = current_timestamp
WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';
</sql>
<attributes/>
<GUI>
<xloc>496</xloc>
<yloc>64</yloc>
</GUI> </GUI>
</transform> </transform>
<transform_error_handling> <transform_error_handling>
<error> <error>
<source_transform>Table output</source_transform> <source_transform>snils_output</source_transform>
<target_transform>Change job status on error</target_transform> <target_transform>snils_failure_job</target_transform>
<is_enabled>Y</is_enabled> <is_enabled>Y</is_enabled>
<nr_valuename/> <nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename> <descriptions_valuename>error_description</descriptions_valuename>

144
mappings/info_recruits/citizen_tables/snils/parallel/snils_flow_delta.hpl
View file

@ -21,50 +21,50 @@
</notepads> </notepads>
<order> <order>
<hop> <hop>
<from>Create job execution record</from> <from>snils_create_job</from>
<to>Table input</to> <to>snils_input</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<to>Identify last row in a stream</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Detect empty stream</from> <from>Detect empty stream</from>
<to>Change job status on success</to> <to>snils_success_job</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Filter rows</from> <from>Filter rows</from>
<to>Change job status on success</to> <to>snils_success_job</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Change job status on error</from> <from>snils_failure_job</from>
<to>Abort</to> <to>Abort</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Insert / update snils</from> <from>snils_insert_or_update</from>
<to>snils_failure_job</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>snils_insert_or_update</from>
<to>Identify last row in a stream 2</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream 2</from>
<to>Filter rows</to> <to>Filter rows</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Insert / update snils</from> <from>snils_input</from>
<to>snils_insert_or_update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>snils_input</from>
<to>Detect empty stream</to> <to>Detect empty stream</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop>
<from>Insert / update snils</from>
<to>Change job status on error</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Insert / update snils</to>
<enabled>Y</enabled>
</hop>
</order> </order>
<transform> <transform>
<name>Abort</name> <name>Abort</name>
@ -83,11 +83,11 @@
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>1328</xloc> <xloc>1328</xloc>
<yloc>400</yloc> <yloc>608</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>Change job status on error</name> <name>snils_failure_job</name>
<type>ExecSql</type> <type>ExecSql</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>Y</distribute>
@ -103,8 +103,11 @@
</argument> </argument>
</arguments> </arguments>
<connection>ervu-dashboard</connection> <connection>ervu-dashboard</connection>
<delete_field/>
<execute_each_row>Y</execute_each_row> <execute_each_row>Y</execute_each_row>
<insert_field/>
<quoteString>N</quoteString> <quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables> <replace_variables>Y</replace_variables>
<set_params>Y</set_params> <set_params>Y</set_params>
<single_statement>N</single_statement> <single_statement>N</single_statement>
@ -116,14 +119,15 @@ WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}'; and recruitment_id = '${IDM_ID}';
</sql> </sql>
<update_field/>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>1104</xloc> <xloc>1104</xloc>
<yloc>400</yloc> <yloc>608</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>Change job status on success</name> <name>snils_success_job</name>
<type>ExecSql</type> <type>ExecSql</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>Y</distribute>
@ -136,8 +140,11 @@ and recruitment_id = '${IDM_ID}';
<arguments> <arguments>
</arguments> </arguments>
<connection>ervu-dashboard</connection> <connection>ervu-dashboard</connection>
<delete_field/>
<execute_each_row>Y</execute_each_row> <execute_each_row>Y</execute_each_row>
<insert_field/>
<quoteString>N</quoteString> <quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables> <replace_variables>Y</replace_variables>
<set_params>N</set_params> <set_params>N</set_params>
<single_statement>N</single_statement> <single_statement>N</single_statement>
@ -148,14 +155,15 @@ WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}'; and recruitment_id = '${IDM_ID}';
</sql> </sql>
<update_field/>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>1408</xloc> <xloc>544</xloc>
<yloc>64</yloc> <yloc>64</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>Create job execution record</name> <name>snils_create_job</name>
<type>ExecSql</type> <type>ExecSql</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>Y</distribute>
@ -168,8 +176,11 @@ and recruitment_id = '${IDM_ID}';
<arguments> <arguments>
</arguments> </arguments>
<connection>ervu-dashboard</connection> <connection>ervu-dashboard</connection>
<delete_field/>
<execute_each_row>N</execute_each_row> <execute_each_row>N</execute_each_row>
<insert_field/>
<quoteString>N</quoteString> <quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables> <replace_variables>Y</replace_variables>
<set_params>N</set_params> <set_params>N</set_params>
<single_statement>N</single_statement> <single_statement>N</single_statement>
@ -180,10 +191,11 @@ SET
error_description = NULL error_description = NULL
where job_name = '${JOB_NAME}' where job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}';</sql> and recruitment_id = '${IDM_ID}';</sql>
<update_field/>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>336</xloc> <xloc>336</xloc>
<yloc>224</yloc> <yloc>432</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
@ -199,8 +211,8 @@ and recruitment_id = '${IDM_ID}';</sql>
</partitioning> </partitioning>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>1408</xloc> <xloc>544</xloc>
<yloc>224</yloc> <yloc>288</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
@ -232,7 +244,7 @@ and recruitment_id = '${IDM_ID}';</sql>
</value> </value>
</condition> </condition>
</compare> </compare>
<send_true_to>Change job status on success</send_true_to> <send_true_to>snils_success_job</send_true_to>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>1104</xloc> <xloc>1104</xloc>
@ -240,7 +252,7 @@ and recruitment_id = '${IDM_ID}';</sql>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>Identify last row in a stream</name> <name>Identify last row in a stream 2</name>
<type>DetectLastRow</type> <type>DetectLastRow</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>Y</distribute>
@ -253,12 +265,12 @@ and recruitment_id = '${IDM_ID}';</sql>
<resultfieldname>last_row</resultfieldname> <resultfieldname>last_row</resultfieldname>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>816</xloc> <xloc>1104</xloc>
<yloc>224</yloc> <yloc>240</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>Insert / update snils</name> <name>snils_insert_or_update</name>
<type>InsertUpdate</type> <type>InsertUpdate</type>
<description/> <description/>
<distribute>N</distribute> <distribute>N</distribute>
@ -275,6 +287,7 @@ and recruitment_id = '${IDM_ID}';</sql>
<condition>=</condition> <condition>=</condition>
<field>source_id</field> <field>source_id</field>
<name>source_id</name> <name>source_id</name>
<name2/>
</key> </key>
<schema>ervu_dashboard</schema> <schema>ervu_dashboard</schema>
<table>snils</table> <table>snils</table>
@ -308,11 +321,11 @@ and recruitment_id = '${IDM_ID}';</sql>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>1104</xloc> <xloc>1104</xloc>
<yloc>224</yloc> <yloc>432</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>Table input</name> <name>snils_input</name>
<type>TableInput</type> <type>TableInput</type>
<description/> <description/>
<distribute>N</distribute> <distribute>N</distribute>
@ -324,60 +337,51 @@ and recruitment_id = '${IDM_ID}';</sql>
</partitioning> </partitioning>
<connection>ervu-dashboard</connection> <connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row> <execute_each_row>N</execute_each_row>
<limit/>
<sql>WITH base AS ( <sql>WITH base AS (
SELECT SELECT
ri.recruit_id, ri.recruit_id,
ri.info, ri.info,
ri.info->'svedFL'->'svedBS'->'snils'->'aktSNILS' AS akt_snils, ri.info->'svedFL'->'svedBS'->'snils'->'aktSNILS' AS akt_snils,
ri.info->'svedFL'->'svedBS'->'snils'->'predSNILS' AS pred_snils ri.info->'svedFL'->'svedBS'->'snils'->'predSNILS' AS pred_snils,
ri.info->'svedFL'->'svedBS'->'snils'->'aktSNILS'->>'dataSved' AS akt_data_sved,
REGEXP_REPLACE(ri.info->'svedFL'->'svedBS'->'snils'->'aktSNILS'->>'snils', '[^0-9]', '', 'g') AS akt_snils_cleaned,
ri.info->'svedFL'->'svedBS'->'snils'->'predSNILS'->>'dataSved' AS pred_data_sved,
REGEXP_REPLACE(ri.info->'svedFL'->'svedBS'->'snils'->'predSNILS'->>'snils', '[^0-9]', '', 'g') AS pred_snils_cleaned
FROM ervu_dashboard.recruits_info ri FROM ervu_dashboard.recruits_info ri
JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id
WHERE WHERE
'${IDM_ID}' != '' '${IDM_ID}' != ''
AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
AND
(
(
(ri.info->'svedFL'->'svedBS'->'snils'->'aktSNILS'->>'dataSved') IS NOT NULL
AND to_date(ri.info->'svedFL'->'svedBS'->'snils'->'aktSNILS'->>'dataSved', 'YYYY-MM-DD') > '${MAX_SOURCE_UPDATE_DATE}'::date
AND LENGTH(REGEXP_REPLACE(ri.info->'svedFL'->'svedBS'->'snils'->'aktSNILS'->>'snils', '[^0-9]', '', 'g')) = 11
)
OR
(
(ri.info->'svedFL'->'svedBS'->'snils'->'predSNILS'->>'dataSved') IS NOT NULL
AND to_date(ri.info->'svedFL'->'svedBS'->'snils'->'predSNILS'->>'dataSved', 'YYYY-MM-DD') > '${MAX_SOURCE_UPDATE_DATE}'::date
AND LENGTH(REGEXP_REPLACE(ri.info->'svedFL'->'svedBS'->'snils'->'predSNILS'->>'snils', '[^0-9]', '', 'g')) = 11
)
)
), ),
snils_data AS ( snils_data AS (
SELECT SELECT
recruit_id, recruit_id,
akt_snils->>'id' AS source_id, akt_snils->>'id' AS source_id,
to_date(akt_snils->>'dataSved', 'YYYY-MM-DD') AS source_update_date, to_date(akt_data_sved, 'YYYY-MM-DD') AS source_update_date,
REGEXP_REPLACE(akt_snils->>'snils', '[^0-9]', '', 'g') AS snils, akt_snils_cleaned AS snils,
true AS actual, true AS actual,
'aktsnils' AS snils_type 'aktSNILS' AS snils_type
FROM base FROM base
WHERE WHERE
(akt_snils->>'dataSved') IS NOT NULL akt_data_sved IS NOT NULL
AND to_date(akt_snils->>'dataSved', 'YYYY-MM-DD') > '${MAX_SOURCE_UPDATE_DATE}'::date AND to_date(akt_data_sved, 'YYYY-MM-DD') > '${MAX_SOURCE_UPDATE_DATE}'::date
AND LENGTH(REGEXP_REPLACE(akt_snils->>'snils', '[^0-9]', '', 'g')) = 11 AND LENGTH(akt_snils_cleaned) = 11
UNION ALL UNION ALL
SELECT SELECT
recruit_id, recruit_id,
pred_snils->>'id' AS source_id, pred_snils->>'id' AS source_id,
to_date(pred_snils->>'dataSved', 'YYYY-MM-DD') AS source_update_date, to_date(pred_data_sved, 'YYYY-MM-DD') AS source_update_date,
REGEXP_REPLACE(pred_snils->>'snils', '[^0-9]', '', 'g') AS snils, pred_snils_cleaned AS snils,
false AS actual, false AS actual,
'predsnils' AS snils_type 'predSNILS' AS snils_type
FROM base FROM base
WHERE WHERE
(pred_snils->>'dataSved') IS NOT NULL pred_data_sved IS NOT NULL
AND to_date(pred_snils->>'dataSved', 'YYYY-MM-DD') > '${MAX_SOURCE_UPDATE_DATE}'::date AND to_date(pred_data_sved, 'YYYY-MM-DD') > '${MAX_SOURCE_UPDATE_DATE}'::date
AND LENGTH(REGEXP_REPLACE(pred_snils->>'snils', '[^0-9]', '', 'g')) = 11 AND LENGTH(pred_snils_cleaned) = 11
) )
SELECT SELECT
recruit_id, recruit_id,
@ -390,13 +394,13 @@ FROM snils_data;</sql>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>544</xloc> <xloc>544</xloc>
<yloc>224</yloc> <yloc>432</yloc>
</GUI> </GUI>
</transform> </transform>
<transform_error_handling> <transform_error_handling>
<error> <error>
<source_transform>Insert / update snils</source_transform> <source_transform>snils_insert_or_update</source_transform>
<target_transform>Change job status on error</target_transform> <target_transform>snils_failure_job</target_transform>
<is_enabled>Y</is_enabled> <is_enabled>Y</is_enabled>
<nr_valuename/> <nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename> <descriptions_valuename>error_description</descriptions_valuename>

287
mappings/info_recruits/citizen_tables/snils/parallel/snils_flow_repeat.hpl
View file

@ -21,48 +21,48 @@
</notepads> </notepads>
<order> <order>
<hop> <hop>
<from>Create job execution record</from> <from>snils_create_job</from>
<to>Table input</to> <to>snils_input</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Table input</from>
<to>Identify last row in a stream</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Filter rows</from> <from>Filter rows</from>
<to>Change job status on success</to> <to>snils_success_job</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Detect empty stream</from> <from>Detect empty stream</from>
<to>Change job status on success</to> <to>snils_success_job</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Change job status on error</from> <from>snils_failure_job</from>
<to>Abort</to> <to>Abort</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Insert / update snils</from> <from>snils_insert_or_update</from>
<to>Filter rows</to> <to>snils_failure_job</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Insert / update snils</from> <from>snils_input</from>
<to>snils_insert_or_update</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>snils_input</from>
<to>Detect empty stream</to> <to>Detect empty stream</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Insert / update snils</from> <from>snils_insert_or_update</from>
<to>Change job status on error</to> <to>Identify last row in a stream </to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Identify last row in a stream</from> <from>Identify last row in a stream </from>
<to>Insert / update snils</to> <to>Filter rows</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
</order> </order>
@ -83,11 +83,11 @@
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>1296</xloc> <xloc>1296</xloc>
<yloc>400</yloc> <yloc>592</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>Change job status on error</name> <name>snils_failure_job</name>
<type>ExecSql</type> <type>ExecSql</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>Y</distribute>
@ -103,8 +103,11 @@
</argument> </argument>
</arguments> </arguments>
<connection>ervu-dashboard</connection> <connection>ervu-dashboard</connection>
<delete_field/>
<execute_each_row>Y</execute_each_row> <execute_each_row>Y</execute_each_row>
<insert_field/>
<quoteString>N</quoteString> <quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables> <replace_variables>Y</replace_variables>
<set_params>Y</set_params> <set_params>Y</set_params>
<single_statement>N</single_statement> <single_statement>N</single_statement>
@ -116,14 +119,15 @@ WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}'; and recruitment_id = '${IDM_ID}';
</sql> </sql>
<update_field/>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>1008</xloc> <xloc>1008</xloc>
<yloc>400</yloc> <yloc>592</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>Change job status on success</name> <name>snils_success_job</name>
<type>ExecSql</type> <type>ExecSql</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>Y</distribute>
@ -136,8 +140,11 @@ and recruitment_id = '${IDM_ID}';
<arguments> <arguments>
</arguments> </arguments>
<connection>ervu-dashboard</connection> <connection>ervu-dashboard</connection>
<delete_field/>
<execute_each_row>Y</execute_each_row> <execute_each_row>Y</execute_each_row>
<insert_field/>
<quoteString>N</quoteString> <quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables> <replace_variables>Y</replace_variables>
<set_params>N</set_params> <set_params>N</set_params>
<single_statement>N</single_statement> <single_statement>N</single_statement>
@ -148,14 +155,86 @@ WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}'; and recruitment_id = '${IDM_ID}';
</sql> </sql>
<update_field/>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>1440</xloc> <xloc>480</xloc>
<yloc>48</yloc> <yloc>48</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>Create job execution record</name> <name>Detect empty stream</name>
<type>DetectEmptyStream</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<attributes/>
<GUI>
<xloc>480</xloc>
<yloc>208</yloc>
</GUI>
</transform>
<transform>
<name>Filter rows</name>
<type>FilterRows</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<compare>
<condition>
<conditions>
</conditions>
<function>=</function>
<leftvalue>last_row</leftvalue>
<negated>N</negated>
<operator>-</operator>
<value>
<isnull>N</isnull>
<length>-1</length>
<name>constant</name>
<precision>-1</precision>
<text>Y</text>
<type>Boolean</type>
</value>
</condition>
</compare>
<send_true_to>snils_success_job</send_true_to>
<attributes/>
<GUI>
<xloc>1008</xloc>
<yloc>48</yloc>
</GUI>
</transform>
<transform>
<name>Identify last row in a stream </name>
<type>DetectLastRow</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>1008</xloc>
<yloc>224</yloc>
</GUI>
</transform>
<transform>
<name>snils_create_job</name>
<type>ExecSql</type> <type>ExecSql</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>Y</distribute>
@ -198,82 +277,67 @@ DO UPDATE SET
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>272</xloc> <xloc>272</xloc>
<yloc>208</yloc> <yloc>400</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>Detect empty stream</name> <name>snils_input</name>
<type>DetectEmptyStream</type> <type>TableInput</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>N</distribute>
<custom_distribution/> <custom_distribution/>
<copies>1</copies> <copies>1</copies>
<partitioning> <partitioning>
<method>none</method> <method>none</method>
<schema_name/> <schema_name/>
</partitioning> </partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<sql>WITH
base as (
SELECT
ri.recruit_id,
ri.info,
ri.info->'svedFL'->'svedBS'->'snils'->'aktSNILS'->>'id' AS akt_source_id,
ri.info->'svedFL'->'svedBS'->'snils'->'aktSNILS'->>'dataSved' AS akt_data_sved,
REGEXP_REPLACE(ri.info->'svedFL'->'svedBS'->'snils'->'aktSNILS'->>'snils', '[^0-9]', '', 'g') AS akt_snils_cleaned,
ri.info->'svedFL'->'svedBS'->'snils'->'predSNILS'->>'id' AS pred_source_id,
ri.info->'svedFL'->'svedBS'->'snils'->'predSNILS'->>'dataSved' AS pred_data_sved,
REGEXP_REPLACE(ri.info->'svedFL'->'svedBS'->'snils'->'predSNILS'->>'snils', '[^0-9]', '', 'g') AS pred_snils_cleaned
FROM ervu_dashboard.recruits_info ri
JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id
WHERE
'${IDM_ID}' != ''
AND COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
)
SELECT
recruit_id,
akt_source_id AS source_id,
to_date(akt_data_sved, 'YYYY-MM-DD') AS source_update_date,
akt_snils_cleaned AS snils,
true AS actual
FROM base
WHERE LENGTH(akt_snils_cleaned) = 11
UNION ALL
SELECT
recruit_id,
pred_source_id AS source_id,
to_date(pred_data_sved, 'YYYY-MM-DD') AS source_update_date,
pred_snils_cleaned AS snils,
false AS actual
FROM base
WHERE LENGTH(pred_snils_cleaned) = 11</sql>
<variables_active>Y</variables_active>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>1440</xloc> <xloc>480</xloc>
<yloc>208</yloc> <yloc>400</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>Filter rows</name> <name>snils_insert_or_update</name>
<type>FilterRows</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<compare>
<condition>
<conditions>
</conditions>
<function>=</function>
<leftvalue>last_row</leftvalue>
<negated>N</negated>
<operator>-</operator>
<value>
<isnull>N</isnull>
<length>-1</length>
<name>constant</name>
<precision>-1</precision>
<text>Y</text>
<type>Boolean</type>
</value>
</condition>
</compare>
<send_true_to>Change job status on success</send_true_to>
<attributes/>
<GUI>
<xloc>1008</xloc>
<yloc>48</yloc>
</GUI>
</transform>
<transform>
<name>Identify last row in a stream</name>
<type>DetectLastRow</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>736</xloc>
<yloc>208</yloc>
</GUI>
</transform>
<transform>
<name>Insert / update snils</name>
<type>InsertUpdate</type> <type>InsertUpdate</type>
<description/> <description/>
<distribute>N</distribute> <distribute>N</distribute>
@ -290,7 +354,6 @@ DO UPDATE SET
<condition>=</condition> <condition>=</condition>
<field>source_id</field> <field>source_id</field>
<name>source_id</name> <name>source_id</name>
<name2/>
</key> </key>
<schema>ervu_dashboard</schema> <schema>ervu_dashboard</schema>
<table>snils</table> <table>snils</table>
@ -324,61 +387,13 @@ DO UPDATE SET
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>1008</xloc> <xloc>1008</xloc>
<yloc>208</yloc> <yloc>400</yloc>
</GUI>
</transform>
<transform>
<name>Table input</name>
<type>TableInput</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<sql>WITH
base as (
SELECT
ri.recruit_id,
ri.info
FROM ervu_dashboard.recruits_info ri
join ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id AND '${IDM_ID}' != '' -- Проверка на пустую строку
AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}'))
)
SELECT
recruit_id,
info->'svedFL'->'svedBS'->'snils'->'aktSNILS'->>'id' AS source_id,
to_date(info->'svedFL'->'svedBS'->'snils'->'aktSNILS'->>'dataSved', 'YYYY-MM-DD') AS source_update_date,
REGEXP_REPLACE(info->'svedFL'->'svedBS'->'snils'->'aktSNILS'->>'snils', '[^0-9]', '', 'g') AS snils,
true AS actual
FROM base
WHERE LENGTH(REGEXP_REPLACE(info->'svedFL'->'svedBS'->'snils'->'aktSNILS'->>'snils', '[^0-9]', '', 'g')) = 11
UNION ALL
SELECT
recruit_id,
info->'svedFL'->'svedBS'->'snils'->'predSNILS'->>'id' AS source_id,
to_date(info->'svedFL'->'svedBS'->'snils'->'predSNILS'->>'dataSved', 'YYYY-MM-DD') AS source_update_date,
REGEXP_REPLACE(info->'svedFL'->'svedBS'->'snils'->'predSNILS'->>'snils', '[^0-9]', '', 'g') AS snils,
false AS actual
FROM base
WHERE LENGTH(REGEXP_REPLACE(info->'svedFL'->'svedBS'->'snils'->'predSNILS'->>'snils', '[^0-9]', '', 'g')) = 11</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
<xloc>480</xloc>
<yloc>208</yloc>
</GUI> </GUI>
</transform> </transform>
<transform_error_handling> <transform_error_handling>
<error> <error>
<source_transform>Insert / update snils</source_transform> <source_transform>snils_insert_or_update</source_transform>
<target_transform>Change job status on error</target_transform> <target_transform>snils_failure_job</target_transform>
<is_enabled>Y</is_enabled> <is_enabled>Y</is_enabled>
<nr_valuename/> <nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename> <descriptions_valuename>error_description</descriptions_valuename>

2
mappings/info_recruits/citizen_tables/snils/recruitment_five_flow_on_error.hpl
View file

@ -59,7 +59,7 @@
</partitioning> </partitioning>
<connection>ervu-dashboard</connection> <connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row> <execute_each_row>N</execute_each_row>
<limit>0</limit> <limit/>
<sql>SELECT <sql>SELECT
r.idm_id as recruitment_id r.idm_id as recruitment_id
FROM ervu_dashboard.recruitment r FROM ervu_dashboard.recruitment r

2
mappings/info_recruits/citizen_tables/snils/recruitments_five_flow.hpl
View file

@ -59,7 +59,7 @@
</partitioning> </partitioning>
<connection>ervu-dashboard</connection> <connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row> <execute_each_row>N</execute_each_row>
<limit>0</limit> <limit/>
<sql>SELECT <sql>SELECT
idm_id AS recruitment idm_id AS recruitment
FROM ervu_dashboard.recruitment FROM ervu_dashboard.recruitment