micord/ervu-dashboard-etl
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix

Browse source
This commit is contained in:
Ruslan 2025-10-16 14:28:50 +05:00
parent 5257b2188d
commit b0c41dfa5b
1 changed files with 55 additions and 95 deletions
Download patch file Download diff file Expand all files Collapse all files

150
mappings/info_recruits/citizen_tables/border_crossing/parallel/border_crossing_flow.hpl
View file

@ -79,31 +79,11 @@
<to>Change job status on error</to> <to>Change job status on error</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop>
<from>Group by</from>
<to>Identify last row in a stream</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>Identify last row in a stream</from>
<to>User defined Java expression</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>User defined Java expression</from>
<to>Update</to>
<enabled>Y</enabled>
</hop>
<hop> <hop>
<from>Table input</from> <from>Table input</from>
<to>Table output</to> <to>Table output</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop>
<from>Table output</from>
<to>Group by</to>
<enabled>Y</enabled>
</hop>
<hop> <hop>
<from>Update</from> <from>Update</from>
<to>Change job status on error</to> <to>Change job status on error</to>
@ -119,6 +99,16 @@
<to>Filter rows</to> <to>Filter rows</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop>
<from>Table output</from>
<to>sort_by_recruit_id</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>sort_by_recruit_id</from>
<to>Update</to>
<enabled>Y</enabled>
</hop>
</order> </order>
<transform> <transform>
<name>Abort</name> <name>Abort</name>
@ -290,59 +280,6 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<yloc>304</yloc> <yloc>304</yloc>
</GUI> </GUI>
</transform> </transform>
<transform>
<name>Group by</name>
<type>GroupBy</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<add_linenr>N</add_linenr>
<all_rows>N</all_rows>
<directory>${java.io.tmpdir}</directory>
<fields>
<field>
<aggregate>border_crossed_raw</aggregate>
<subject>is_border_crossed</subject>
<type>MAX</type>
</field>
</fields>
<give_back_row>N</give_back_row>
<group>
<field>
<name>recruit_id</name>
</field>
</group>
<ignore_aggregate>N</ignore_aggregate>
<prefix>grp</prefix>
<attributes/>
<GUI>
<xloc>688</xloc>
<yloc>176</yloc>
</GUI>
</transform>
<transform>
<name>Identify last row in a stream</name>
<type>DetectLastRow</type>
<description/>
<distribute>Y</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<resultfieldname>last_row</resultfieldname>
<attributes/>
<GUI>
<xloc>880</xloc>
<yloc>176</yloc>
</GUI>
</transform>
<transform> <transform>
<name>Table input</name> <name>Table input</name>
<type>TableInput</type> <type>TableInput</type>
@ -356,20 +293,34 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</partitioning> </partitioning>
<connection>ervu-dashboard</connection> <connection>ervu-dashboard</connection>
<execute_each_row>N</execute_each_row> <execute_each_row>N</execute_each_row>
<limit/>
<sql>SELECT <sql>SELECT
ri.recruit_id, ri.recruit_id,
(string_agg(elem->>'dataVyezd', ', '))::date as date_crossing, CASE
(string_agg(elem->>'dataVozvrashh', ', '))::date as return_date, WHEN elem->>'dataVyezd' ~ '^\d{4}-\d{2}-\d{2}$' THEN (elem->>'dataVyezd')::date
1 as is_border_crossed ELSE NULL
END as date_crossing,
CASE
WHEN elem->>'dataVozvrashh' ~ '^\d{4}-\d{2}-\d{2}$' THEN (elem->>'dataVozvrashh')::date
ELSE NULL
END as return_date,
true as is_border_crossed
FROM ervu_dashboard.recruits_info ri FROM ervu_dashboard.recruits_info ri
JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id AND '${IDM_ID}' != '' -- Проверка на пустую строку JOIN ervu_dashboard.citizen r ON r.recruit_id = ri.recruit_id
AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}')) AND '${IDM_ID}' != '' -- Проверка на пустую строку
AND '${M_R_CR_DATE}'::timestamp >= ri.created_at, AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}'))
LATERAL jsonb_array_elements(info->'svedFL'->'extend'->'svedPeresechGran') as elem --AND '${M_R_CR_DATE}'::timestamp >= ri.created_at
WHERE jsonb_typeof(info->'svedFL'->'extend'->'svedPeresechGran') = 'array' JOIN LATERAL (
AND (elem->>'dataVyezd' IS NOT NULL OR elem->>'dataVozvrashh' IS NOT NULL) SELECT elem
GROUP BY ri.recruit_id;</sql> FROM jsonb_array_elements(
CASE
WHEN jsonb_typeof(ri.info->'svedFL'->'extend'->'svedPeresechGran') = 'array'
THEN ri.info->'svedFL'->'extend'->'svedPeresechGran'
ELSE '[]'::jsonb
END
) as elem
WHERE (elem->>'dataVyezd' IS NOT NULL AND elem->>'dataVyezd' ~ '^\d{4}-\d{2}-\d{2}$')
OR (elem->>'dataVozvrashh' IS NOT NULL AND elem->>'dataVozvrashh' ~ '^\d{4}-\d{2}-\d{2}$')
) AS filtered_elems ON true</sql>
<variables_active>Y</variables_active> <variables_active>Y</variables_active>
<attributes/> <attributes/>
<GUI> <GUI>
@ -437,11 +388,13 @@ GROUP BY ri.recruit_id;</sql>
<commit>10000</commit> <commit>10000</commit>
<connection>ervu-dashboard</connection> <connection>ervu-dashboard</connection>
<error_ignored>Y</error_ignored> <error_ignored>Y</error_ignored>
<ignore_flag_field/>
<lookup> <lookup>
<key> <key>
<condition>=</condition> <condition>=</condition>
<field>recruit_id</field> <field>recruit_id</field>
<name>recruit_id</name> <name>recruit_id</name>
<name2/>
</key> </key>
<schema>ervu_dashboard</schema> <schema>ervu_dashboard</schema>
<table>citizen</table> <table>citizen</table>
@ -459,8 +412,8 @@ GROUP BY ri.recruit_id;</sql>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>User defined Java expression</name> <name>sort_by_recruit_id</name>
<type>Janino</type> <type>SortRows</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>Y</distribute>
<custom_distribution/> <custom_distribution/>
@ -469,17 +422,24 @@ GROUP BY ri.recruit_id;</sql>
<method>none</method> <method>none</method>
<schema_name/> <schema_name/>
</partitioning> </partitioning>
<formula> <compress>N</compress>
<field_name>border_crossed</field_name> <directory>${java.io.tmpdir}</directory>
<formula_string>border_crossed_raw == 1</formula_string> <fields>
<value_type>Boolean</value_type> <field>
<value_length>-1</value_length> <ascending>Y</ascending>
<value_precision>-1</value_precision> <case_sensitive>N</case_sensitive>
<replace_field/> <collator_enabled>N</collator_enabled>
</formula> <collator_strength>0</collator_strength>
<name>recruit_id</name>
<presorted>N</presorted>
</field>
</fields>
<sort_prefix>srt</sort_prefix>
<sort_size>1000000</sort_size>
<unique_rows>Y</unique_rows>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>1104</xloc> <xloc>688</xloc>
<yloc>176</yloc> <yloc>176</yloc>
</GUI> </GUI>
</transform> </transform>