micord/ervu-dashboard-etl
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

some fixes

Browse source
This commit is contained in:
Fusionshh 2025-10-20 13:24:19 +03:00
parent 8d16084e4a
commit 03f8368d20
1 changed files with 108 additions and 133 deletions
Download patch file Download diff file Expand all files Collapse all files

241
mappings/info_recruits/citizen_tables/work/employer/parallel/employer_flow_repeat.hpl
View file

@ -9,6 +9,16 @@
<pipeline_type>Normal</pipeline_type> <pipeline_type>Normal</pipeline_type>
<pipeline_status>0</pipeline_status> <pipeline_status>0</pipeline_status>
<parameters> <parameters>
<parameter>
<name>IDM_ID</name>
<default_value>0e0535ab-5303-43ea-b36a-8c0a2a8d0ed5</default_value>
<description/>
</parameter>
<parameter>
<name>JOB_NAME</name>
<default_value>employer_job</default_value>
<description/>
</parameter>
</parameters> </parameters>
<capture_transform_performance>N</capture_transform_performance> <capture_transform_performance>N</capture_transform_performance>
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay> <transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
@ -31,59 +41,49 @@
<to>employer_input</to> <to>employer_input</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop>
<from>employer_input</from>
<to>Identify last row in a stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>employer_upsert</to>
<enabled>Y</enabled>
</hop>
<hop> <hop>
<from>employer_upsert</from> <from>employer_upsert</from>
<to>Change job status on error</to> <to>Change job status on error</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop>
<from>Detect empty stream</from>
<to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop> <hop>
<from>Filter rows</from> <from>Filter rows</from>
<to>Change job status on success</to> <to>Change job status on success</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop>
<from>employer_input</from>
<to>Detect empty stream</to>
<enabled>Y</enabled>
</hop>
<hop> <hop>
<from>filter_non_actual_employers</from> <from>filter_non_actual_employers</from>
<to>Update</to> <to>Update</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Identify last row in a stream</from> <from>employer_input</from>
<to>filter_non_actual_employers</to> <to>filter_non_actual_employers</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop>
<from>employer_upsert</from>
<to>Filter rows</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>employer_input</from>
<to>employer_upsert</to>
<enabled>Y</enabled>
</hop>
<hop> <hop>
<from>Update</from> <from>Update</from>
<to>Change job status on error</to> <to>Change job status on error</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>employer_upsert</from> <from>Detect empty stream</from>
<to>Block until transforms finish</to> <to>Change job status on success</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Block until transforms finish</from>
<to>Filter rows</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
</order> </order>
@ -103,30 +103,8 @@
<row_threshold>0</row_threshold> <row_threshold>0</row_threshold>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>1520</xloc> <xloc>1184</xloc>
<yloc>560</yloc> <yloc>608</yloc>
</GUI>
</transform>
<transform>
<name>Block until transforms finish</name>
<type>BlockUntilTransformsFinish</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<transforms>
<transform>
<name>Update</name>
</transform>
</transforms>
<attributes/>
<GUI>
<xloc>1328</xloc>
<yloc>400</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
@ -145,7 +123,7 @@
<name>error_description</name> <name>error_description</name>
</argument> </argument>
</arguments> </arguments>
<connection>ervu-dashboard</connection> <connection>ervu-dashboard-118</connection>
<execute_each_row>Y</execute_each_row> <execute_each_row>Y</execute_each_row>
<quoteString>N</quoteString> <quoteString>N</quoteString>
<replace_variables>Y</replace_variables> <replace_variables>Y</replace_variables>
@ -161,8 +139,8 @@ and recruitment_id = '${IDM_ID}';
</sql> </sql>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>1136</xloc> <xloc>976</xloc>
<yloc>560</yloc> <yloc>608</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
@ -179,8 +157,11 @@ and recruitment_id = '${IDM_ID}';
<arguments> <arguments>
</arguments> </arguments>
<connection>ervu-dashboard</connection> <connection>ervu-dashboard</connection>
<delete_field/>
<execute_each_row>Y</execute_each_row> <execute_each_row>Y</execute_each_row>
<insert_field/>
<quoteString>N</quoteString> <quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables> <replace_variables>Y</replace_variables>
<set_params>N</set_params> <set_params>N</set_params>
<single_statement>N</single_statement> <single_statement>N</single_statement>
@ -191,9 +172,10 @@ WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}'; and recruitment_id = '${IDM_ID}';
</sql> </sql>
<update_field/>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>1520</xloc> <xloc>1280</xloc>
<yloc>256</yloc> <yloc>256</yloc>
</GUI> </GUI>
</transform> </transform>
@ -211,8 +193,11 @@ and recruitment_id = '${IDM_ID}';
<arguments> <arguments>
</arguments> </arguments>
<connection>ervu-dashboard</connection> <connection>ervu-dashboard</connection>
<delete_field/>
<execute_each_row>N</execute_each_row> <execute_each_row>N</execute_each_row>
<insert_field/>
<quoteString>N</quoteString> <quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables> <replace_variables>Y</replace_variables>
<set_params>N</set_params> <set_params>N</set_params>
<single_statement>N</single_statement> <single_statement>N</single_statement>
@ -223,6 +208,7 @@ DO UPDATE
SET status = 'PROCESSING', SET status = 'PROCESSING',
error_description = null, error_description = null,
execution_datetime = current_timestamp;</sql> execution_datetime = current_timestamp;</sql>
<update_field/>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>352</xloc> <xloc>352</xloc>
@ -242,7 +228,7 @@ SET status = 'PROCESSING',
</partitioning> </partitioning>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>704</xloc> <xloc>544</xloc>
<yloc>256</yloc> <yloc>256</yloc>
</GUI> </GUI>
</transform> </transform>
@ -279,25 +265,7 @@ SET status = 'PROCESSING',
<send_true_to>Change job status on success</send_true_to> <send_true_to>Change job status on success</send_true_to>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>1520</xloc> <xloc>1280</xloc>
<yloc>400</yloc>
</GUI>
</transform>
<transform>
<name>Identify last row in a stream</name>
<type>DetectLastRow</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>704</xloc>
<yloc>400</yloc> <yloc>400</yloc>
</GUI> </GUI>
</transform> </transform>
@ -315,11 +283,13 @@ SET status = 'PROCESSING',
<commit>10000</commit> <commit>10000</commit>
<connection>ervu-dashboard</connection> <connection>ervu-dashboard</connection>
<error_ignored>N</error_ignored> <error_ignored>N</error_ignored>
<ignore_flag_field/>
<lookup> <lookup>
<key> <key>
<condition>=</condition> <condition>=</condition>
<field>recruit_id</field> <field>recruit_id</field>
<name>recruit_id</name> <name>recruit_id</name>
<name2/>
</key> </key>
<schema>ervu_dashboard</schema> <schema>ervu_dashboard</schema>
<table>citizen</table> <table>citizen</table>
@ -332,15 +302,15 @@ SET status = 'PROCESSING',
<use_batch>Y</use_batch> <use_batch>Y</use_batch>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>896</xloc> <xloc>736</xloc>
<yloc>560</yloc> <yloc>608</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>employer_input</name> <name>employer_input</name>
<type>TableInput</type> <type>TableInput</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>N</distribute>
<custom_distribution/> <custom_distribution/>
<copies>1</copies> <copies>1</copies>
<partitioning> <partitioning>
@ -350,61 +320,65 @@ SET status = 'PROCESSING',
<connection>ervu-dashboard</connection> <connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row> <execute_each_row>N</execute_each_row>
<limit>0</limit> <limit>0</limit>
<sql>with filteredData as (select ri.recruit_id, <sql>with filteredData as (
ri.info -> 'svedFL' -> 'svedRabotodat' -> 'rabotodat' -> 'aktRabotodat' as rabotodat, select ri.recruit_id,
true as actual_employer ri.info -> 'svedFL' -> 'svedRabotodat' -> 'rabotodat' -> 'aktRabotodat' as rabotodat,
from ervu_dashboard.recruits_info ri true as actual_employer,
where ri.info -> 'svedFL' -> 'svedRabotodat' -> 'rabotodat' ->> 'aktRabotodat' != 'null' ctid
AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) from ervu_dashboard.recruits_info ri
where COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
and (ri.info -> 'svedFL' -> 'svedRabotodat' -> 'rabotodat' ->> 'aktRabotodat') &lt;&gt; 'null'
union all union all
select ri.recruit_id, select ri.recruit_id,
ri.info -> 'svedFL' -> 'svedRabotodat' -> 'rabotodat' -> 'predRabotodat' as rabotodat, ri.info -> 'svedFL' -> 'svedRabotodat' -> 'rabotodat' -> 'predRabotodat' as rabotodat,
false as actual_employer false as actual_employer,
from ervu_dashboard.recruits_info ri ctid
where ri.info -> 'svedFL' -&gt; 'svedRabotodat' -> 'rabotodat' ->> 'predRabotodat' != 'null' from ervu_dashboard.recruits_info ri
AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) where COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
and (ri.info -> 'svedFL' -> 'svedRabotodat' -> 'rabotodat' ->> 'predRabotodat') &lt;&gt; 'null'
) )
select select
fd.recruit_id, fd.recruit_id,
to_date(rabotodat ->> 'dataSved', 'YYYY-MM-DD') as source_update_date, to_date(rabotodat ->> 'dataSved', 'YYYY-MM-DD') as source_update_date,
rabotodat ->> 'id' as source_id, rabotodat ->> 'id' as source_id,
COALESCE(rabotodat -> 'extend' ->> 'status', '0') = '1' as status, COALESCE(rabotodat -> 'extend' ->> 'status', '0') = '1' as status,
rabotodat -> 'extend' ->> 'adresRabotodat' as address, rabotodat -> 'extend' ->> 'adresRabotodat' as address,
rabotodat -> 'extend' ->> 'tipPodrazdRabotodat' as separate_unit_type, rabotodat -> 'extend' ->> 'tipPodrazdRabotodat' as separate_unit_type,
rabotodat -> 'extend' ->> 'adresPodrazdRabotodat' as separate_unit_address, rabotodat -> 'extend' ->> 'adresPodrazdRabotodat' as separate_unit_address,
case case
when rabotodat ->> 'svedIP' is not null and rabotodat ->> 'svedIP' != 'null' when rabotodat ->> 'svedIP' is not null and rabotodat ->> 'svedIP' != 'null'
then 'Индивидуальный предприниматель' then 'Индивидуальный предприниматель'
when rabotodat ->> 'svedYUL' is not null and rabotodat ->> 'svedYUL' != 'null' then 'Юридическое лицо' when rabotodat ->> 'svedYUL' is not null and rabotodat ->> 'svedYUL' != 'null' then 'Юридическое лицо'
when rabotodat ->> 'svedGlKFH' is not null and rabotodat ->> 'svedGlKFH' != 'null' then 'Глава КФХ' when rabotodat ->> 'svedGlKFH' is not null and rabotodat ->> 'svedGlKFH' != 'null' then 'Глава КФХ'
when rabotodat ->> 'svedPlatFL' is not null and rabotodat ->> 'svedPlatFL' != 'null' then 'Физическое лицо' when rabotodat ->> 'svedPlatFL' is not null and rabotodat ->> 'svedPlatFL' != 'null' then 'Физическое лицо'
end as employer_category_name, end as employer_category_name,
rabotodat -> 'svedYUL' -> 'naimOrg' as name, rabotodat -> 'svedYUL' ->> 'naimOrg' as name,
case case
when rabotodat ->> 'svedIP' is not null and rabotodat ->> 'svedIP' != 'null' when rabotodat ->> 'svedIP' is not null and rabotodat ->> 'svedIP' != 'null'
then rabotodat -> 'svedIP' ->> 'innfl' then rabotodat -> 'svedIP' ->> 'innfl'
when rabotodat ->> 'svedYUL' is not null and rabotodat ->> 'svedYUL' != 'null' when rabotodat ->> 'svedYUL' is not null and rabotodat ->> 'svedYUL' != 'null'
then rabotodat -> 'svedYUL' ->> 'innyul' then rabotodat -> 'svedYUL' ->> 'innyul'
when rabotodat ->> 'svedGlKFH' is not null and rabotodat ->> 'svedGlKFH' != 'null' when rabotodat ->> 'svedGlKFH' is not null and rabotodat ->> 'svedGlKFH' != 'null'
then rabotodat -> 'svedGlKFH' ->> 'innfl' then rabotodat -> 'svedGlKFH' ->> 'innfl'
when rabotodat ->> 'svedPlatFL' is not null and rabotodat ->> 'svedPlatFL' != 'null' when rabotodat ->> 'svedPlatFL' is not null and rabotodat ->> 'svedPlatFL' != 'null'
then rabotodat -> 'svedPlatFL' ->> 'innfl' then rabotodat -> 'svedPlatFL' ->> 'innfl'
end as inn, end as inn,
case case
when rabotodat ->> 'svedIP' is not null and rabotodat ->> 'svedIP' != 'null' when rabotodat ->> 'svedIP' is not null and rabotodat ->> 'svedIP' != 'null'
then rabotodat -> 'extend' -> 'svedYUL' ->> 'ogrn' then rabotodat -> 'extend' -> 'svedYUL' ->> 'ogrn'
when rabotodat ->> 'svedYUL' is not null and rabotodat ->> 'svedYUL' != 'null' when rabotodat ->> 'svedYUL' is not null and rabotodat ->> 'svedYUL' != 'null'
then rabotodat -> 'svedIP' ->> 'ogrnip' then rabotodat -> 'svedIP' ->> 'ogrnip'
end as ogrn, end as ogrn,
case case
when rabotodat ->> 'svedYUL' is not null and rabotodat ->> 'svedYUL' != 'null' when rabotodat ->> 'svedYUL' is not null and rabotodat ->> 'svedYUL' != 'null'
then rabotodat -> 'extend' -> 'svedYUL' ->> 'kpp' then rabotodat -> 'extend' -> 'svedYUL' ->> 'kpp'
end as kpp, end as kpp,
coalesce(rabotodat -> 'svedYUL' -> 'extend' ->> 'priznakOpk', '0') = '1' as is_opk_org, coalesce(rabotodat -> 'svedYUL' -> 'extend' ->> 'priznakOpk', '0') = '1' as is_opk_org,
actual_employer, actual_employer,
coalesce(rabotodat -> 'extend' ->> 'statusRabotodat', '0') = '1' as status coalesce(rabotodat -> 'extend' ->&gt; 'statusRabotodat', '0') = '1' as status,
fd.ctid = (select max(ctid) from filteredData) as last_row
from filteredData fd;</sql> from filteredData fd;</sql>
<variables_active>Y</variables_active> <variables_active>Y</variables_active>
<attributes/> <attributes/>
@ -431,6 +405,7 @@ from filteredData fd;</sql>
<condition>=</condition> <condition>=</condition>
<field>source_id</field> <field>source_id</field>
<name>source_id</name> <name>source_id</name>
<name2/>
</key> </key>
<schema>ervu_dashboard</schema> <schema>ervu_dashboard</schema>
<table>employer</table> <table>employer</table>
@ -508,7 +483,7 @@ from filteredData fd;</sql>
<update_bypassed>N</update_bypassed> <update_bypassed>N</update_bypassed>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>1136</xloc> <xloc>976</xloc>
<yloc>400</yloc> <yloc>400</yloc>
</GUI> </GUI>
</transform> </transform>
@ -544,8 +519,8 @@ from filteredData fd;</sql>
<send_true_to>Update</send_true_to> <send_true_to>Update</send_true_to>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>704</xloc> <xloc>544</xloc>
<yloc>560</yloc> <yloc>608</yloc>
</GUI> </GUI>
</transform> </transform>
<transform_error_handling> <transform_error_handling>