micord/ervu-dashboard-etl
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

lots of fixes

Browse source
This commit is contained in:
Fusionshh 2025-10-12 11:14:53 +03:00
parent d96bd80799
commit 3bb2dab010
3 changed files with 74 additions and 67 deletions
Download patch file Download diff file Expand all files Collapse all files

78
mappings/info_recruits/citizen_tables/medicine/vich/parallel/vich_flow.hpl
View file

@ -215,6 +215,42 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<yloc>416</yloc>
</GUI>
</transform>
<transform>
<name>Filter rows 2</name>
<type>FilterRows</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<compare>
<condition>
<conditions>
</conditions>
<function>=</function>
<leftvalue>vich_registered</leftvalue>
<negated>N</negated>
<operator>-</operator>
<value>
<isnull>N</isnull>
<length>-1</length>
<name>constant</name>
<precision>-1</precision>
<text>Y</text>
<type>Boolean</type>
</value>
</condition>
</compare>
<send_true_to>Identify last row in a stream</send_true_to>
<attributes/>
<GUI>
<xloc>720</xloc>
<yloc>416</yloc>
</GUI>
</transform>
<transform>
<name>Identify last row in a stream</name>
<type>DetectLastRow</type>
@ -281,13 +317,11 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<commit>10000</commit>
<connection>ervu-dashboard</connection>
<error_ignored>N</error_ignored>
<ignore_flag_field/>
<lookup>
<key>
<condition>=</condition>
<field>recruit_id</field>
<name>recruit_id</name>
<name2/>
</key>
<schema>ervu_dashboard</schema>
<table>citizen</table>
@ -453,6 +487,7 @@ and recruitment_id = '${IDM_ID}';
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>WITH filteredData AS (SELECT ri.recruit_id,
to_date(ri.info -> 'svedFL' -> 'svedUchetVICH' -> 'svedVICH' ->> 'dataSved',
'YYYY-MM-DD') as source_update_date,
@ -462,7 +497,7 @@ and recruitment_id = '${IDM_ID}';
false) as vich_registered,
true as has_vich_info
FROM recruits_info ri
WHERE ri.info -> 'svedFL' -> 'svedUchetVICH' ->> 'prOtsSvedUchetVICH' &lt;&gt; '1'
WHERE ri.info -> 'svedFL' -> 'svedUchetVICH' ->> 'prOtsSvedUchetVICH' IS DISTINCT FROM '1'
and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}')
SELECT fd.recruit_id,
vich_elem ->> 'id' AS source_id,
@ -536,43 +571,6 @@ FROM filteredData fd
<yloc>416</yloc>
</GUI>
</transform>
<transform>
<name>Filter rows 2</name>
<type>FilterRows</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<compare>
<condition>
<conditions>
</conditions>
<function>=</function>
<leftvalue>vich_registered</leftvalue>
<negated>N</negated>
<operator>-</operator>
<value>
<isnull>N</isnull>
<length>-1</length>
<mask/>
<name>constant</name>
<precision>-1</precision>
<text>Y</text>
<type>Boolean</type>
</value>
</condition>
</compare>
<send_true_to>Identify last row in a stream</send_true_to>
<attributes/>
<GUI>
<xloc>720</xloc>
<yloc>416</yloc>
</GUI>
</transform>
<transform_error_handling>
<error>
<source_transform>Update</source_transform>

2
mappings/info_recruits/citizen_tables/medicine/vich/parallel/vich_flow_delta.hpl
View file

@ -501,7 +501,7 @@ and recruitment_id = '${IDM_ID}';
false) as vich_registered,
true as has_vich_info
FROM recruits_info ri
WHERE COALESCE(ri.info -> 'svedFL' -> 'svedUchetVICH' ->> 'prOtsSvedUchetVICH', '') &lt;&gt; '1'
WHERE ri.info -> 'svedFL' -> 'svedUchetVICH' ->> 'prOtsSvedUchetVICH' IS DISTINCT FROM '1'
and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}')
SELECT fd.recruit_id,
vich_elem ->> 'id' AS source_id,

61
mappings/info_recruits/citizen_tables/medicine/vich/parallel/vich_flow_repeat.hpl
View file

@ -414,6 +414,28 @@ and recruitment_id = '${IDM_ID}';</sql>
<yloc>864</yloc>
</GUI>
</transform>
<transform>
<name>block_until_update_finished</name>
<type>BlockUntilTransformsFinish</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<transforms>
<transform>
<name>Update</name>
</transform>
</transforms>
<attributes/>
<GUI>
<xloc>1536</xloc>
<yloc>432</yloc>
</GUI>
</transform>
<transform>
<name>change_job_status_on_update_error</name>
<type>ExecSql</type>
@ -462,20 +484,29 @@ and recruitment_id = '${IDM_ID}';
</partitioning>
<connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row>
<limit/>
<sql>WITH filteredData AS (SELECT ri.recruit_id,
to_date(ri.info -> 'svedFL' -> 'svedUchetVICH' -> 'svedVICH' ->> 'dataSved',
'YYYY-MM-DD') as source_update_date,
ri.info -> 'svedFL' -> 'svedUchetVICH' -> 'svedVICH' -> 'svedUchet' as vich_arr
ri.info -> 'svedFL' -> 'svedUchetVICH' -> 'svedVICH' -> 'svedUchet' as vich_arr,
coalesce(
ri.info -> 'svedFL' -> 'svedUchetVICH' -> 'svedVICH' ->> 'priznakNalichSved' = '1',
false) as vich_registered,
true as has_vich_info
FROM recruits_info ri
WHERE jsonb_typeof(ri.info -> 'svedFL' -> 'svedUchetVICH' -> 'svedVICH' -> 'svedUchet') = 'array'
WHERE ri.info -> 'svedFL' -> 'svedUchetVICH' ->> 'prOtsSvedUchetVICH' IS DISTINCT FROM '1'
and COALESCE(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}')
SELECT fd.recruit_id,
vich_elem ->> 'id' AS source_id,
fd.source_update_date,
to_date(vich_elem ->> 'dataSnyat', 'YYYY-MM-DD') AS register_date,
to_date(vich_elem ->> 'dataPostUchet', 'YYYY-MM-DD') AS deregistration_date
to_date(vich_elem ->> 'dataPostUchet', 'YYYY-MM-DD') AS deregistration_date,
vich_registered,
has_vich_info
FROM filteredData fd
CROSS JOIN LATERAL jsonb_array_elements(vich_arr) AS vich_elem;</sql>
CROSS JOIN LATERAL jsonb_array_elements(COALESCE(
NULLIF(vich_arr, 'null'::jsonb),
'[]'::jsonb)) AS vich_elem;</sql>
<variables_active>Y</variables_active>
<attributes/>
<GUI>
@ -547,28 +578,6 @@ FROM filteredData fd
<yloc>432</yloc>
</GUI>
</transform>
<transform>
<name>block_until_update_finished</name>
<type>BlockUntilTransformsFinish</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<transforms>
<transform>
<name>Update</name>
</transform>
</transforms>
<attributes/>
<GUI>
<xloc>1536</xloc>
<yloc>432</yloc>
</GUI>
</transform>
<transform_error_handling>
<error>
<source_transform>Update</source_transform>