micord/ervu-dashboard-etl
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

ERVU-505: фикс для первички

Browse source
This commit is contained in:
adel.kalimullin 2025-08-06 11:08:46 +03:00
parent 91181a0256
commit 63ffd8dbbc
6 changed files with 330 additions and 300 deletions
Download patch file Download diff file Expand all files Collapse all files

116
mappings_new/info_recruits/citizen_tables/passport/delta_passport.hpl
View file

@ -31,6 +31,63 @@
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>Execute SQL script</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
<argument>
<name>recruit_id</name>
</argument>
<argument>
<name>passport_number</name>
</argument>
<argument>
<name>passport_series</name>
</argument>
<argument>
<name>organization_name</name>
</argument>
<argument>
<name>unit_code</name>
</argument>
<argument>
<name>issue_date</name>
</argument>
<argument>
<name>actual</name>
</argument>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>N</replace_variables>
<set_params>Y</set_params>
<single_statement>N</single_statement>
<sql>INSERT INTO ervu_dashboard.passport (
recruit_id, passport_number, passport_series,
organization_name, unit_code, issue_date, actual
) VALUES (?, ?, ?, ?, ?, ?, ?)
ON CONFLICT (recruit_id, actual) DO UPDATE
SET
passport_number = EXCLUDED.passport_number,
passport_series = EXCLUDED.passport_series,
organization_name = EXCLUDED.organization_name,
unit_code = EXCLUDED.unit_code,
issue_date = EXCLUDED.issue_date;</sql>
<attributes/>
<GUI>
<xloc>880</xloc>
<yloc>208</yloc>
</GUI>
</transform>
<transform>
<name>Insert / update</name>
<type>InsertUpdate</type>
@ -135,7 +192,7 @@ JOIN LATERAL (
SELECT passport_data
FROM UNNEST(ARRAY[
ri.info->'svedFL'->'svedBS'->'pasportRF'->'aktPasportRF',
ri.info->'svedFL'->'svedBS'->'pasportRF'->'aktPasportRF'
ri.info->'svedFL'->'svedBS'->'pasportRF'->'predPasportRF'
]) AS passport_data
WHERE passport_data IS NOT NULL AND passport_data::text &lt;&gt; 'null'
) AS p ON TRUE
@ -147,63 +204,6 @@ JOIN LATERAL (
<yloc>352</yloc>
</GUI>
</transform>
<transform>
<name>Execute SQL script</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
<argument>
<name>recruit_id</name>
</argument>
<argument>
<name>passport_number</name>
</argument>
<argument>
<name>passport_series</name>
</argument>
<argument>
<name>organization_name</name>
</argument>
<argument>
<name>unit_code</name>
</argument>
<argument>
<name>issue_date</name>
</argument>
<argument>
<name>actual</name>
</argument>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>N</replace_variables>
<set_params>Y</set_params>
<single_statement>N</single_statement>
<sql>INSERT INTO ervu_dashboard.passport (
recruit_id, passport_number, passport_series,
organization_name, unit_code, issue_date, actual
) VALUES (?, ?, ?, ?, ?, ?, ?)
ON CONFLICT (recruit_id, actual) DO UPDATE
SET
passport_number = EXCLUDED.passport_number,
passport_series = EXCLUDED.passport_series,
organization_name = EXCLUDED.organization_name,
unit_code = EXCLUDED.unit_code,
issue_date = EXCLUDED.issue_date;</sql>
<attributes/>
<GUI>
<xloc>880</xloc>
<yloc>208</yloc>
</GUI>
</transform>
<transform_error_handling>
</transform_error_handling>
<attributes/>

128
mappings_new/info_recruits/citizen_tables/passport/parallel/passport_flow1.hpl
View file

@ -38,67 +38,10 @@
</hop>
<hop>
<from>Table input</from>
<to>Execute SQL script</to>
<to>Table output</to>
<enabled>Y</enabled>
</hop>
</order>
<transform>
<name>Execute SQL script</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
<argument>
<name>recruit_id</name>
</argument>
<argument>
<name>passport_number</name>
</argument>
<argument>
<name>passport_series</name>
</argument>
<argument>
<name>organization_name</name>
</argument>
<argument>
<name>unit_code</name>
</argument>
<argument>
<name>issue_date</name>
</argument>
<argument>
<name>actual</name>
</argument>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>N</replace_variables>
<set_params>Y</set_params>
<single_statement>N</single_statement>
<sql>INSERT INTO ervu_dashboard.passport (
recruit_id, passport_number, passport_series,
organization_name, unit_code, issue_date, actual
) VALUES (?, ?, ?, ?, ?, ?, ?)
ON CONFLICT (recruit_id, actual) DO UPDATE
SET
passport_number = EXCLUDED.passport_number,
passport_series = EXCLUDED.passport_series,
organization_name = EXCLUDED.organization_name,
unit_code = EXCLUDED.unit_code,
issue_date = EXCLUDED.issue_date;</sql>
<attributes/>
<GUI>
<xloc>800</xloc>
<yloc>80</yloc>
</GUI>
</transform>
<transform>
<name>Get variables</name>
<type>GetVariable</type>
@ -191,7 +134,7 @@ SET
<update_bypassed>N</update_bypassed>
<attributes/>
<GUI>
<xloc>944</xloc>
<xloc>896</xloc>
<yloc>288</yloc>
</GUI>
</transform>
@ -267,7 +210,7 @@ JOIN LATERAL (
SELECT passport_data
FROM UNNEST(ARRAY[
ri.info->'svedFL'->'svedBS'->'pasportRF'->'aktPasportRF',
ri.info->'svedFL'->'svedBS'->'pasportRF'->'aktPasportRF'
ri.info->'svedFL'->'svedBS'->'pasportRF'->'predPasportRF'
]) AS passport_data
WHERE passport_data IS NOT NULL AND passport_data::text &lt;&gt; 'null'
) AS p ON TRUE
@ -275,10 +218,73 @@ JOIN LATERAL (
<variables_active>Y</variables_active>
<attributes/>
<GUI>
<xloc>448</xloc>
<xloc>528</xloc>
<yloc>272</yloc>
</GUI>
</transform>
<transform>
<name>Table output</name>
<type>TableOutput</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<commit>1000</commit>
<connection>ervu-dashboard</connection>
<fields>
<field>
<column_name>recruit_id</column_name>
<stream_name>recruit_id</stream_name>
</field>
<field>
<column_name>actual</column_name>
<stream_name>actual</stream_name>
</field>
<field>
<column_name>passport_series</column_name>
<stream_name>passport_series</stream_name>
</field>
<field>
<column_name>passport_number</column_name>
<stream_name>passport_number</stream_name>
</field>
<field>
<column_name>issue_date</column_name>
<stream_name>issue_date</stream_name>
</field>
<field>
<column_name>organization_name</column_name>
<stream_name>organization_name</stream_name>
</field>
<field>
<column_name>unit_code</column_name>
<stream_name>unit_code</stream_name>
</field>
</fields>
<ignore_errors>N</ignore_errors>
<only_when_have_rows>N</only_when_have_rows>
<partitioning_daily>N</partitioning_daily>
<partitioning_enabled>N</partitioning_enabled>
<partitioning_monthly>Y</partitioning_monthly>
<return_field>passport_id</return_field>
<return_keys>Y</return_keys>
<schema>ervu_dashboard</schema>
<specify_fields>Y</specify_fields>
<table>passport</table>
<tablename_in_field>N</tablename_in_field>
<tablename_in_table>Y</tablename_in_table>
<truncate>N</truncate>
<use_batch>N</use_batch>
<attributes/>
<GUI>
<xloc>864</xloc>
<yloc>192</yloc>
</GUI>
</transform>
<transform_error_handling>
<error>
<source_transform>Select values 4 2</source_transform>

96
mappings_new/info_recruits/citizen_tables/passport/parallel/passport_flow2.hpl
View file

@ -27,7 +27,7 @@
</hop>
<hop>
<from>Table input</from>
<to>Execute SQL script</to>
<to>Table output</to>
<enabled>Y</enabled>
</hop>
</order>
@ -135,7 +135,7 @@ JOIN LATERAL (
SELECT passport_data
FROM UNNEST(ARRAY[
ri.info->'svedFL'->'svedBS'->'pasportRF'->'aktPasportRF',
ri.info->'svedFL'->'svedBS'->'pasportRF'->'aktPasportRF'
ri.info->'svedFL'->'svedBS'->'pasportRF'->'predPasportRF'
]) AS passport_data
WHERE passport_data IS NOT NULL AND passport_data::text &lt;&gt; 'null'
) AS p ON TRUE
@ -148,8 +148,8 @@ JOIN LATERAL (
</GUI>
</transform>
<transform>
<name>Execute SQL script</name>
<type>ExecSql</type>
<name>Table output</name>
<type>TableOutput</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
@ -158,50 +158,56 @@ JOIN LATERAL (
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
<argument>
<name>recruit_id</name>
</argument>
<argument>
<name>passport_number</name>
</argument>
<argument>
<name>passport_series</name>
</argument>
<argument>
<name>organization_name</name>
</argument>
<argument>
<name>unit_code</name>
</argument>
<argument>
<name>issue_date</name>
</argument>
<argument>
<name>actual</name>
</argument>
</arguments>
<commit>1000</commit>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>N</replace_variables>
<set_params>Y</set_params>
<single_statement>N</single_statement>
<sql>INSERT INTO ervu_dashboard.passport (
recruit_id, passport_number, passport_series,
organization_name, unit_code, issue_date, actual
) VALUES (?, ?, ?, ?, ?, ?, ?)
ON CONFLICT (recruit_id, actual) DO UPDATE
SET
passport_number = EXCLUDED.passport_number,
passport_series = EXCLUDED.passport_series,
organization_name = EXCLUDED.organization_name,
unit_code = EXCLUDED.unit_code,
issue_date = EXCLUDED.issue_date;</sql>
<fields>
<field>
<column_name>recruit_id</column_name>
<stream_name>recruit_id</stream_name>
</field>
<field>
<column_name>actual</column_name>
<stream_name>actual</stream_name>
</field>
<field>
<column_name>passport_series</column_name>
<stream_name>passport_series</stream_name>
</field>
<field>
<column_name>passport_number</column_name>
<stream_name>passport_number</stream_name>
</field>
<field>
<column_name>issue_date</column_name>
<stream_name>issue_date</stream_name>
</field>
<field>
<column_name>organization_name</column_name>
<stream_name>organization_name</stream_name>
</field>
<field>
<column_name>unit_code</column_name>
<stream_name>unit_code</stream_name>
</field>
</fields>
<ignore_errors>N</ignore_errors>
<only_when_have_rows>N</only_when_have_rows>
<partitioning_daily>N</partitioning_daily>
<partitioning_enabled>N</partitioning_enabled>
<partitioning_monthly>Y</partitioning_monthly>
<return_field>passport_id</return_field>
<return_keys>Y</return_keys>
<schema>ervu_dashboard</schema>
<specify_fields>Y</specify_fields>
<table>passport</table>
<tablename_in_field>N</tablename_in_field>
<tablename_in_table>Y</tablename_in_table>
<truncate>N</truncate>
<use_batch>N</use_batch>
<attributes/>
<GUI>
<xloc>944</xloc>
<yloc>112</yloc>
<xloc>1040</xloc>
<yloc>192</yloc>
</GUI>
</transform>
<transform_error_handling>

96
mappings_new/info_recruits/citizen_tables/passport/parallel/passport_flow3.hpl
View file

@ -27,7 +27,7 @@
</hop>
<hop>
<from>Table input</from>
<to>Execute SQL script</to>
<to>Table output</to>
<enabled>Y</enabled>
</hop>
</order>
@ -135,7 +135,7 @@ JOIN LATERAL (
SELECT passport_data
FROM UNNEST(ARRAY[
ri.info->'svedFL'->'svedBS'->'pasportRF'->'aktPasportRF',
ri.info->'svedFL'->'svedBS'->'pasportRF'->'aktPasportRF'
ri.info->'svedFL'->'svedBS'->'pasportRF'->'predPasportRF'
]) AS passport_data
WHERE passport_data IS NOT NULL AND passport_data::text &lt;&gt; 'null'
) AS p ON TRUE
@ -148,8 +148,8 @@ JOIN LATERAL (
</GUI>
</transform>
<transform>
<name>Execute SQL script</name>
<type>ExecSql</type>
<name>Table output</name>
<type>TableOutput</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
@ -158,50 +158,56 @@ JOIN LATERAL (
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
<argument>
<name>recruit_id</name>
</argument>
<argument>
<name>passport_number</name>
</argument>
<argument>
<name>passport_series</name>
</argument>
<argument>
<name>organization_name</name>
</argument>
<argument>
<name>unit_code</name>
</argument>
<argument>
<name>issue_date</name>
</argument>
<argument>
<name>actual</name>
</argument>
</arguments>
<commit>1000</commit>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>N</replace_variables>
<set_params>Y</set_params>
<single_statement>N</single_statement>
<sql>INSERT INTO ervu_dashboard.passport (
recruit_id, passport_number, passport_series,
organization_name, unit_code, issue_date, actual
) VALUES (?, ?, ?, ?, ?, ?, ?)
ON CONFLICT (recruit_id, actual) DO UPDATE
SET
passport_number = EXCLUDED.passport_number,
passport_series = EXCLUDED.passport_series,
organization_name = EXCLUDED.organization_name,
unit_code = EXCLUDED.unit_code,
issue_date = EXCLUDED.issue_date;</sql>
<fields>
<field>
<column_name>recruit_id</column_name>
<stream_name>recruit_id</stream_name>
</field>
<field>
<column_name>actual</column_name>
<stream_name>actual</stream_name>
</field>
<field>
<column_name>passport_series</column_name>
<stream_name>passport_series</stream_name>
</field>
<field>
<column_name>passport_number</column_name>
<stream_name>passport_number</stream_name>
</field>
<field>
<column_name>issue_date</column_name>
<stream_name>issue_date</stream_name>
</field>
<field>
<column_name>organization_name</column_name>
<stream_name>organization_name</stream_name>
</field>
<field>
<column_name>unit_code</column_name>
<stream_name>unit_code</stream_name>
</field>
</fields>
<ignore_errors>N</ignore_errors>
<only_when_have_rows>N</only_when_have_rows>
<partitioning_daily>N</partitioning_daily>
<partitioning_enabled>N</partitioning_enabled>
<partitioning_monthly>Y</partitioning_monthly>
<return_field>passport_id</return_field>
<return_keys>Y</return_keys>
<schema>ervu_dashboard</schema>
<specify_fields>Y</specify_fields>
<table>passport</table>
<tablename_in_field>N</tablename_in_field>
<tablename_in_table>Y</tablename_in_table>
<truncate>N</truncate>
<use_batch>N</use_batch>
<attributes/>
<GUI>
<xloc>1024</xloc>
<yloc>224</yloc>
<xloc>1120</xloc>
<yloc>272</yloc>
</GUI>
</transform>
<transform_error_handling>

96
mappings_new/info_recruits/citizen_tables/passport/parallel/passport_flow4.hpl
View file

@ -27,7 +27,7 @@
</hop>
<hop>
<from>Table input</from>
<to>Execute SQL script</to>
<to>Table output</to>
<enabled>Y</enabled>
</hop>
</order>
@ -135,7 +135,7 @@ JOIN LATERAL (
SELECT passport_data
FROM UNNEST(ARRAY[
ri.info->'svedFL'->'svedBS'->'pasportRF'->'aktPasportRF',
ri.info->'svedFL'->'svedBS'->'pasportRF'->'aktPasportRF'
ri.info->'svedFL'->'svedBS'->'pasportRF'->'predPasportRF'
]) AS passport_data
WHERE passport_data IS NOT NULL AND passport_data::text &lt;&gt; 'null'
) AS p ON TRUE
@ -148,8 +148,8 @@ JOIN LATERAL (
</GUI>
</transform>
<transform>
<name>Execute SQL script</name>
<type>ExecSql</type>
<name>Table output</name>
<type>TableOutput</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
@ -158,50 +158,56 @@ JOIN LATERAL (
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
<argument>
<name>recruit_id</name>
</argument>
<argument>
<name>passport_number</name>
</argument>
<argument>
<name>passport_series</name>
</argument>
<argument>
<name>organization_name</name>
</argument>
<argument>
<name>unit_code</name>
</argument>
<argument>
<name>issue_date</name>
</argument>
<argument>
<name>actual</name>
</argument>
</arguments>
<commit>1000</commit>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>N</replace_variables>
<set_params>Y</set_params>
<single_statement>N</single_statement>
<sql>INSERT INTO ervu_dashboard.passport (
recruit_id, passport_number, passport_series,
organization_name, unit_code, issue_date, actual
) VALUES (?, ?, ?, ?, ?, ?, ?)
ON CONFLICT (recruit_id, actual) DO UPDATE
SET
passport_number = EXCLUDED.passport_number,
passport_series = EXCLUDED.passport_series,
organization_name = EXCLUDED.organization_name,
unit_code = EXCLUDED.unit_code,
issue_date = EXCLUDED.issue_date;</sql>
<fields>
<field>
<column_name>recruit_id</column_name>
<stream_name>recruit_id</stream_name>
</field>
<field>
<column_name>actual</column_name>
<stream_name>actual</stream_name>
</field>
<field>
<column_name>passport_series</column_name>
<stream_name>passport_series</stream_name>
</field>
<field>
<column_name>passport_number</column_name>
<stream_name>passport_number</stream_name>
</field>
<field>
<column_name>issue_date</column_name>
<stream_name>issue_date</stream_name>
</field>
<field>
<column_name>organization_name</column_name>
<stream_name>organization_name</stream_name>
</field>
<field>
<column_name>unit_code</column_name>
<stream_name>unit_code</stream_name>
</field>
</fields>
<ignore_errors>N</ignore_errors>
<only_when_have_rows>N</only_when_have_rows>
<partitioning_daily>N</partitioning_daily>
<partitioning_enabled>N</partitioning_enabled>
<partitioning_monthly>Y</partitioning_monthly>
<return_field>passport_id</return_field>
<return_keys>Y</return_keys>
<schema>ervu_dashboard</schema>
<specify_fields>Y</specify_fields>
<table>passport</table>
<tablename_in_field>N</tablename_in_field>
<tablename_in_table>Y</tablename_in_table>
<truncate>N</truncate>
<use_batch>N</use_batch>
<attributes/>
<GUI>
<xloc>928</xloc>
<yloc>160</yloc>
<xloc>1008</xloc>
<yloc>240</yloc>
</GUI>
</transform>
<transform_error_handling>

98
mappings_new/info_recruits/citizen_tables/passport/parallel/passport_flow5.hpl
View file

@ -27,7 +27,7 @@
</hop>
<hop>
<from>Table input</from>
<to>Execute SQL script</to>
<to>Table output</to>
<enabled>Y</enabled>
</hop>
</order>
@ -97,7 +97,7 @@
<attributes/>
<GUI>
<xloc>1040</xloc>
<yloc>368</yloc>
<yloc>416</yloc>
</GUI>
</transform>
<transform>
@ -135,7 +135,7 @@ JOIN LATERAL (
SELECT passport_data
FROM UNNEST(ARRAY[
ri.info->'svedFL'->'svedBS'->'pasportRF'->'aktPasportRF',
ri.info->'svedFL'->'svedBS'->'pasportRF'->'aktPasportRF'
ri.info->'svedFL'->'svedBS'->'pasportRF'->'predPasportRF'
]) AS passport_data
WHERE passport_data IS NOT NULL AND passport_data::text &lt;&gt; 'null'
) AS p ON TRUE
@ -148,8 +148,8 @@ JOIN LATERAL (
</GUI>
</transform>
<transform>
<name>Execute SQL script</name>
<type>ExecSql</type>
<name>Table output</name>
<type>TableOutput</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
@ -158,50 +158,56 @@ JOIN LATERAL (
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
<argument>
<name>recruit_id</name>
</argument>
<argument>
<name>passport_number</name>
</argument>
<argument>
<name>passport_series</name>
</argument>
<argument>
<name>organization_name</name>
</argument>
<argument>
<name>unit_code</name>
</argument>
<argument>
<name>issue_date</name>
</argument>
<argument>
<name>actual</name>
</argument>
</arguments>
<commit>1000</commit>
<connection>ervu-dashboard</connection>
<execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>N</replace_variables>
<set_params>Y</set_params>
<single_statement>N</single_statement>
<sql>INSERT INTO ervu_dashboard.passport (
recruit_id, passport_number, passport_series,
organization_name, unit_code, issue_date, actual
) VALUES (?, ?, ?, ?, ?, ?, ?)
ON CONFLICT (recruit_id, actual) DO UPDATE
SET
passport_number = EXCLUDED.passport_number,
passport_series = EXCLUDED.passport_series,
organization_name = EXCLUDED.organization_name,
unit_code = EXCLUDED.unit_code,
issue_date = EXCLUDED.issue_date;</sql>
<fields>
<field>
<column_name>recruit_id</column_name>
<stream_name>recruit_id</stream_name>
</field>
<field>
<column_name>actual</column_name>
<stream_name>actual</stream_name>
</field>
<field>
<column_name>passport_series</column_name>
<stream_name>passport_series</stream_name>
</field>
<field>
<column_name>passport_number</column_name>
<stream_name>passport_number</stream_name>
</field>
<field>
<column_name>issue_date</column_name>
<stream_name>issue_date</stream_name>
</field>
<field>
<column_name>organization_name</column_name>
<stream_name>organization_name</stream_name>
</field>
<field>
<column_name>unit_code</column_name>
<stream_name>unit_code</stream_name>
</field>
</fields>
<ignore_errors>N</ignore_errors>
<only_when_have_rows>N</only_when_have_rows>
<partitioning_daily>N</partitioning_daily>
<partitioning_enabled>N</partitioning_enabled>
<partitioning_monthly>Y</partitioning_monthly>
<return_field>passport_id</return_field>
<return_keys>Y</return_keys>
<schema>ervu_dashboard</schema>
<specify_fields>Y</specify_fields>
<table>passport</table>
<tablename_in_field>N</tablename_in_field>
<tablename_in_table>Y</tablename_in_table>
<truncate>N</truncate>
<use_batch>N</use_batch>
<attributes/>
<GUI>
<xloc>912</xloc>
<yloc>240</yloc>
<xloc>992</xloc>
<yloc>288</yloc>
</GUI>
</transform>
<transform_error_handling>