micord/ervu-dashboard-etl
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge branch 'develop' of 10.10.31.70:/ervu-dashboard-etl into develop

Browse source
This commit is contained in:
Ruslan 2025-10-17 13:32:37 +05:00
commit 699b804ef9
3 changed files with 62 additions and 63 deletions
Download patch file Download diff file Expand all files Collapse all files

92
mappings/info_recruits/citizen_tables/work/employer/parallel/employer_flow.hpl
View file

@ -7,7 +7,18 @@
<extended_description/> <extended_description/>
<pipeline_version/> <pipeline_version/>
<pipeline_type>Normal</pipeline_type> <pipeline_type>Normal</pipeline_type>
<pipeline_status>0</pipeline_status>
<parameters> <parameters>
<parameter>
<name>IDM_ID</name>
<default_value>0e0535ab-5303-43ea-b36a-8c0a2a8d0ed5</default_value>
<description/>
</parameter>
<parameter>
<name>JOB_NAME</name>
<default_value>employer_job</default_value>
<description/>
</parameter>
</parameters> </parameters>
<capture_transform_performance>N</capture_transform_performance> <capture_transform_performance>N</capture_transform_performance>
<transform_performance_capturing_delay>1000</transform_performance_capturing_delay> <transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
@ -20,11 +31,6 @@
<notepads> <notepads>
</notepads> </notepads>
<order> <order>
<hop>
<from>Create job execution record</from>
<to>employer_input</to>
<enabled>Y</enabled>
</hop>
<hop> <hop>
<from>Change job status on error</from> <from>Change job status on error</from>
<to>Abort</to> <to>Abort</to>
@ -85,6 +91,11 @@
<to>Filter rows</to> <to>Filter rows</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop>
<from>Create job execution record</from>
<to>employer_input</to>
<enabled>Y</enabled>
</hop>
</order> </order>
<transform> <transform>
<name>Abort</name> <name>Abort</name>
@ -196,35 +207,6 @@ and recruitment_id = '${IDM_ID}';
<yloc>112</yloc> <yloc>112</yloc>
</GUI> </GUI>
</transform> </transform>
<transform>
<name>Create job execution record</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>INSERT INTO etl.job_execution (id, job_name, status, execution_datetime, error_description, recruitment_id)
VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</sql>
<attributes/>
<GUI>
<xloc>176</xloc>
<yloc>288</yloc>
</GUI>
</transform>
<transform> <transform>
<name>Detect empty stream</name> <name>Detect empty stream</name>
<type>DetectEmptyStream</type> <type>DetectEmptyStream</type>
@ -342,11 +324,10 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<method>none</method> <method>none</method>
<schema_name/> <schema_name/>
</partitioning> </partitioning>
<connection>ervu-dashboard</connection> <connection>ervu-dashboard-118</connection>
<execute_each_row>N</execute_each_row> <execute_each_row>N</execute_each_row>
<limit>0</limit> <limit>0</limit>
<sql>with filteredData as (select ri.recruit_id, <sql>with filteredData as (select ri.recruit_id,
(ri.info -> 'svedFL' -> 'svedRabotodat' -> 'rabotodat' ->> 'id') as source_id,
ri.info -> 'svedFL' -> 'svedRabotodat' -> 'rabotodat' -> 'aktRabotodat' as rabotodat, ri.info -> 'svedFL' -> 'svedRabotodat' -> 'rabotodat' -> 'aktRabotodat' as rabotodat,
true as actual_employer true as actual_employer
from ervu_dashboard.recruits_info ri from ervu_dashboard.recruits_info ri
@ -356,17 +337,17 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
union all union all
select ri.recruit_id, select ri.recruit_id,
(ri.info -> 'svedFL' -> 'svedRabotodat' -> 'rabotodat' ->> 'id') as source_id,
ri.info -> 'svedFL' -> 'svedRabotodat' -> 'rabotodat' -> 'predRabotodat' as rabotodat, ri.info -> 'svedFL' -> 'svedRabotodat' -> 'rabotodat' -> 'predRabotodat' as rabotodat,
false as actual_employer false as actual_employer
from ervu_dashboard.recruits_info ri from ervu_dashboard.recruits_info ri
where ri.info -> 'svedFL' -> 'svedRabotodat' -> 'rabotodat' ->> 'predRabotodat' != 'null' where ri.info -> 'svedFL' -&gt; 'svedRabotodat' -> 'rabotodat' ->> 'predRabotodat' != 'null'
AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}'))
) )
select fd.source_id, select
fd.recruit_id, fd.recruit_id,
to_date(rabotodat ->> 'dataSved', 'YYYY-MM-DD') as source_update_date, to_date(rabotodat ->> 'dataSved', 'YYYY-MM-DD') as source_update_date,
COALESCE(rabotodat -> 'extend' ->> 'status', '0') = '1' AS status, rabotodat ->> 'id' as source_id,
COALESCE(rabotodat -> 'extend' ->> 'status', '0') = '1' as status,
rabotodat -> 'extend' ->> 'adresRabotodat' as address, rabotodat -> 'extend' ->> 'adresRabotodat' as address,
rabotodat -> 'extend' ->> 'tipPodrazdRabotodat' as separate_unit_type, rabotodat -> 'extend' ->> 'tipPodrazdRabotodat' as separate_unit_type,
rabotodat -> 'extend' ->> 'adresPodrazdRabotodat' as separate_unit_address, rabotodat -> 'extend' ->> 'adresPodrazdRabotodat' as separate_unit_address,
@ -535,6 +516,35 @@ from filteredData fd;</sql>
<yloc>448</yloc> <yloc>448</yloc>
</GUI> </GUI>
</transform> </transform>
<transform>
<name>Create job execution record</name>
<type>ExecSql</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<arguments>
</arguments>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<quoteString>N</quoteString>
<replace_variables>Y</replace_variables>
<set_params>N</set_params>
<single_statement>N</single_statement>
<sql>INSERT INTO etl.job_execution (id, job_name, status, execution_datetime, error_description, recruitment_id)
VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</sql>
<attributes/>
<GUI>
<xloc>192</xloc>
<yloc>288</yloc>
</GUI>
</transform>
<transform_error_handling> <transform_error_handling>
<error> <error>
<source_transform>Update</source_transform> <source_transform>Update</source_transform>

20
mappings/info_recruits/citizen_tables/work/employer/parallel/employer_flow_delta.hpl
View file

@ -428,7 +428,6 @@ and recruitment_id = '${IDM_ID}';</sql>
</partitioning> </partitioning>
<connection>ervu-dashboard</connection> <connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row> <execute_each_row>N</execute_each_row>
<limit/>
<sql>WITH fd AS (SELECT ri.recruit_id, <sql>WITH fd AS (SELECT ri.recruit_id,
(ri.info -> 'svedFL' -> 'svedRabotodat' -> 'rabotodat' ->> 'id') AS source_id, (ri.info -> 'svedFL' -> 'svedRabotodat' -> 'rabotodat' ->> 'id') AS source_id,
ri.info -> 'svedFL' -> 'svedRabotodat' -> 'rabotodat' -> 'aktRabotodat' AS akt_rabotodat, ri.info -> 'svedFL' -> 'svedRabotodat' -> 'rabotodat' -> 'aktRabotodat' AS akt_rabotodat,
@ -444,7 +443,7 @@ and recruitment_id = '${IDM_ID}';</sql>
AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}'))
) )
SELECT fd.recruit_id, SELECT fd.recruit_id,
fd.source_id, pred_rabotodat ->> 'id' as pred_source_id,
to_date(pred_rabotodat ->> 'dataSved', 'YYYY-MM-DD') as pred_source_update_date, to_date(pred_rabotodat ->> 'dataSved', 'YYYY-MM-DD') as pred_source_update_date,
COALESCE(pred_rabotodat -> 'extend' ->> 'statusRabotodat' = '1', false) as pred_status, COALESCE(pred_rabotodat -> 'extend' ->> 'statusRabotodat' = '1', false) as pred_status,
pred_rabotodat -> 'extend' ->> 'adresRabotodat' as pred_address, pred_rabotodat -> 'extend' ->> 'adresRabotodat' as pred_address,
@ -467,7 +466,7 @@ SELECT fd.recruit_id,
then pred_rabotodat -> 'svedYUL' ->> 'innyul' then pred_rabotodat -> 'svedYUL' ->> 'innyul'
when pred_rabotodat ->> 'svedGlKFH' is not null and pred_rabotodat ->> 'svedGlKFH' != 'null' when pred_rabotodat ->> 'svedGlKFH' is not null and pred_rabotodat ->> 'svedGlKFH' != 'null'
then pred_rabotodat -> 'svedGlKFH' ->> 'innfl' then pred_rabotodat -> 'svedGlKFH' ->> 'innfl'
when pred_rabotodat ->> 'svedPlatFL' is not null and pred_rabotodat ->&gt; 'svedPlatFL' != 'null' when pred_rabotodat ->> 'svedPlatFL' is not null and pred_rabotodat ->> 'svedPlatFL' != 'null'
then pred_rabotodat -> 'svedPlatFL' ->> 'innfl' then pred_rabotodat -> 'svedPlatFL' ->> 'innfl'
end as pred_inn, end as pred_inn,
case case
@ -484,6 +483,7 @@ SELECT fd.recruit_id,
to_date(fd.pred_rabotodat ->> 'dataSved', 'YYYY-MM-DD') > '${MAX_SOURCE_UPDATE_DATE}' AS pred_need_update, to_date(fd.pred_rabotodat ->> 'dataSved', 'YYYY-MM-DD') > '${MAX_SOURCE_UPDATE_DATE}' AS pred_need_update,
akt_rabotodat ->> 'id' as akt_source_id,
to_date(akt_rabotodat ->> 'dataSved', 'YYYY-MM-DD') as akt_source_update_date, to_date(akt_rabotodat ->> 'dataSved', 'YYYY-MM-DD') as akt_source_update_date,
COALESCE(akt_rabotodat -> 'extend' ->> 'statusRabotodat' = '1', false) as akt_status, COALESCE(akt_rabotodat -> 'extend' ->> 'statusRabotodat' = '1', false) as akt_status,
akt_rabotodat -> 'extend' ->> 'adresRabotodat' as akt_address, akt_rabotodat -> 'extend' ->> 'adresRabotodat' as akt_address,
@ -737,11 +737,6 @@ FROM fd;</sql>
<field>source_id</field> <field>source_id</field>
<name>source_id</name> <name>source_id</name>
</key> </key>
<key>
<condition>=</condition>
<field>actual_employer</field>
<name>akt_need_update</name>
</key>
<schema>ervu_dashboard</schema> <schema>ervu_dashboard</schema>
<table>employer</table> <table>employer</table>
<value> <value>
@ -751,7 +746,7 @@ FROM fd;</sql>
</value> </value>
<value> <value>
<name>source_id</name> <name>source_id</name>
<rename>source_id</rename> <rename>akt_source_id</rename>
<update>N</update> <update>N</update>
</value> </value>
<value> <value>
@ -841,11 +836,6 @@ FROM fd;</sql>
<field>source_id</field> <field>source_id</field>
<name>source_id</name> <name>source_id</name>
</key> </key>
<key>
<condition>&lt;></condition>
<field>actual_employer</field>
<name>pred_need_update</name>
</key>
<schema>ervu_dashboard</schema> <schema>ervu_dashboard</schema>
<table>employer</table> <table>employer</table>
<value> <value>
@ -855,7 +845,7 @@ FROM fd;</sql>
</value> </value>
<value> <value>
<name>source_id</name> <name>source_id</name>
<rename>source_id</rename> <rename>pred_source_id</rename>
<update>N</update> <update>N</update>
</value> </value>
<value> <value>

13
mappings/info_recruits/citizen_tables/work/employer/parallel/employer_flow_repeat.hpl
View file

@ -350,7 +350,6 @@ SET status = 'PROCESSING',
<execute_each_row>N</execute_each_row> <execute_each_row>N</execute_each_row>
<limit>0</limit> <limit>0</limit>
<sql>with filteredData as (select ri.recruit_id, <sql>with filteredData as (select ri.recruit_id,
(ri.info -> 'svedFL' -> 'svedRabotodat' -> 'rabotodat' ->> 'id') as source_id,
ri.info -> 'svedFL' -> 'svedRabotodat' -> 'rabotodat' -> 'aktRabotodat' as rabotodat, ri.info -> 'svedFL' -> 'svedRabotodat' -> 'rabotodat' -> 'aktRabotodat' as rabotodat,
true as actual_employer true as actual_employer
from ervu_dashboard.recruits_info ri from ervu_dashboard.recruits_info ri
@ -360,22 +359,22 @@ SET status = 'PROCESSING',
union all union all
select ri.recruit_id, select ri.recruit_id,
(ri.info -> 'svedFL' -> 'svedRabotodat' -> 'rabotodat' ->> 'id') as source_id,
ri.info -> 'svedFL' -> 'svedRabotodat' -> 'rabotodat' -> 'predRabotodat' as rabotodat, ri.info -> 'svedFL' -> 'svedRabotodat' -> 'rabotodat' -> 'predRabotodat' as rabotodat,
false as actual_employer false as actual_employer
from ervu_dashboard.recruits_info ri from ervu_dashboard.recruits_info ri
where ri.info -> 'svedFL' -> 'svedRabotodat' -> 'rabotodat' ->> 'predRabotodat' != 'null' where ri.info -> 'svedFL' -&gt; 'svedRabotodat' -> 'rabotodat' ->> 'predRabotodat' != 'null'
AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}'))
) )
select fd.source_id, select
fd.recruit_id, fd.recruit_id,
to_date(rabotodat ->> 'dataSved', 'YYYY-MM-DD') as source_update_date, to_date(rabotodat ->> 'dataSved', 'YYYY-MM-DD') as source_update_date,
COALESCE(rabotodat -> 'extend' ->> 'status', '0') = '1' AS status, rabotodat ->> 'id' as source_id,
COALESCE(rabotodat -> 'extend' ->> 'status', '0') = '1' as status,
rabotodat -> 'extend' ->> 'adresRabotodat' as address, rabotodat -> 'extend' ->> 'adresRabotodat' as address,
rabotodat -> 'extend' ->> 'tipPodrazdRabotodat' as separate_unit_type, rabotodat -> 'extend' ->> 'tipPodrazdRabotodat' as separate_unit_type,
rabotodat -> 'extend' ->> 'adresPodrazdRabotodat' as separate_unit_address, rabotodat -> 'extend' ->> 'adresPodrazdRabotodat' as separate_unit_address,
case case
when rabotodat ->&gt; 'svedIP' is not null and rabotodat ->> 'svedIP' != 'null' when rabotodat ->> 'svedIP' is not null and rabotodat ->> 'svedIP' != 'null'
then 'Индивидуальный предприниматель' then 'Индивидуальный предприниматель'
when rabotodat ->> 'svedYUL' is not null and rabotodat ->> 'svedYUL' != 'null' then 'Юридическое лицо' when rabotodat ->> 'svedYUL' is not null and rabotodat ->> 'svedYUL' != 'null' then 'Юридическое лицо'
when rabotodat ->> 'svedGlKFH' is not null and rabotodat ->> 'svedGlKFH' != 'null' then 'Глава КФХ' when rabotodat ->> 'svedGlKFH' is not null and rabotodat ->> 'svedGlKFH' != 'null' then 'Глава КФХ'