micord/ervu-dashboard-etl
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

lots of fixes

Browse source
This commit is contained in:
Fusionshh 2025-11-18 11:31:38 +03:00
parent 537a111284
commit 237a6ed609
1 changed files with 91 additions and 89 deletions
Download patch file Download diff file Expand all files Collapse all files

180
mappings/info_recruits/citizen_tables/border_crossing/parallel/border_crossing_flow.hpl
View file

@ -18,103 +18,71 @@
<modified_date>2025/08/05 14:31:22.799</modified_date> <modified_date>2025/08/05 14:31:22.799</modified_date>
</info> </info>
<notepads> <notepads>
<notepad>
<backgroundcolorblue>251</backgroundcolorblue>
<backgroundcolorgreen>232</backgroundcolorgreen>
<backgroundcolorred>201</backgroundcolorred>
<bordercolorblue>90</bordercolorblue>
<bordercolorgreen>58</bordercolorgreen>
<bordercolorred>14</bordercolorred>
<fontbold>N</fontbold>
<fontcolorblue>90</fontcolorblue>
<fontcolorgreen>58</fontcolorgreen>
<fontcolorred>14</fontcolorred>
<fontitalic>N</fontitalic>
<fontname>Segoe UI</fontname>
<fontsize>9</fontsize>
<height>294</height>
<xloc>176</xloc>
<yloc>576</yloc>
<note>+
1. Table_input:
- string_agg же создаст перечисление дат в формате строки через запятую, как это планируеться приводить в дату и в дальнейшем инсертить в бд?
- is_border_crossed, Почему сразу не передавать true?
- зачем join на citizen, если у recruits_info уже есть поля айди военкомата?
- Зачем на уровне первичного flow, условие на дату в части created_at?
- в части where идет отбращшение к lateral elem, т.е предполагаеться что массив будет развернут до выполнения where, что будет если в исходных данных вместо массива будет null?
- elem ->> 'dataVyezd' вернет 'null', это не sql NULL, условие IS NOT NULL не отработает.
- Для чего групировка и в input и далее после table output?
- Детектить пустой стрим, можно сразу после инпута? почему пытаемся детектить только после апдейта?
2. Table_output:
- error_code нет смысла указывать в рамках error handling, под него нет колонки
3. Update:
- не указан error_description в error handling , в случае ошибки произойдет еще ошибка при попытке выполнения change job status on error
</note>
<width>1030</width>
</notepad>
</notepads> </notepads>
<order> <order>
<hop> <hop>
<from>Create job execution record</from> <from>сreate_job_execution_record_BC</from>
<to>Table input</to> <to>table_input_BC</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Filter rows</from> <from>filter_last_row_BC</from>
<to>Change job status on success</to> <to>change_job_status_on_success_BC</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Detect empty stream</from> <from>detect_empty_stream_BC</from>
<to>Change job status on success</to> <to>change_job_status_on_success_BC</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Change job status on error</from> <from>change_job_status_on_error_BC</from>
<to>Abort</to> <to>abort_BC</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Table output</from> <from>table_output_BC</from>
<to>Change job status on error</to> <to>change_job_status_on_error_BC</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Update</from> <from>update_citizen_BC</from>
<to>Change job status on error</to> <to>change_job_status_on_error_BC</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>sort_by_recruit_id</from> <from>sort_by_recruit_id_BC</from>
<to>Update</to> <to>update_citizen_BC</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Table output</from> <from>table_output_BC</from>
<to>Filter rows</to> <to>filter_last_row_BC</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Table input</from> <from>Identify_last_row_BC</from>
<to>Detect empty stream</to> <to>detect_empty_stream_BC</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Table input</from> <from>Identify_last_row_BC</from>
<to>sort_by_recruit_id</to> <to>sort_by_recruit_id_BC</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
<hop> <hop>
<from>Table input</from> <from>Identify_last_row_BC</from>
<to>Table output</to> <to>table_output_BC</to>
<enabled>Y</enabled>
</hop>
<hop>
<from>table_input_BC</from>
<to>Identify_last_row_BC</to>
<enabled>Y</enabled> <enabled>Y</enabled>
</hop> </hop>
</order> </order>
<transform> <transform>
<name>Abort</name> <name>abort_BC</name>
<type>Abort</type> <type>Abort</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>Y</distribute>
@ -126,15 +94,16 @@
</partitioning> </partitioning>
<abort_option>ABORT_WITH_ERROR</abort_option> <abort_option>ABORT_WITH_ERROR</abort_option>
<always_log_rows>Y</always_log_rows> <always_log_rows>Y</always_log_rows>
<message/>
<row_threshold>0</row_threshold> <row_threshold>0</row_threshold>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>1200</xloc> <xloc>1280</xloc>
<yloc>480</yloc> <yloc>480</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>Change job status on error</name> <name>change_job_status_on_error_BC</name>
<type>ExecSql</type> <type>ExecSql</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>Y</distribute>
@ -150,8 +119,11 @@
</argument> </argument>
</arguments> </arguments>
<connection>ervu-dashboard</connection> <connection>ervu-dashboard</connection>
<delete_field/>
<execute_each_row>Y</execute_each_row> <execute_each_row>Y</execute_each_row>
<insert_field/>
<quoteString>N</quoteString> <quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables> <replace_variables>Y</replace_variables>
<set_params>Y</set_params> <set_params>Y</set_params>
<single_statement>N</single_statement> <single_statement>N</single_statement>
@ -163,14 +135,15 @@ WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}'; and recruitment_id = '${IDM_ID}';
</sql> </sql>
<update_field/>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>1024</xloc> <xloc>1104</xloc>
<yloc>480</yloc> <yloc>480</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>Change job status on success</name> <name>change_job_status_on_success_BC</name>
<type>ExecSql</type> <type>ExecSql</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>Y</distribute>
@ -183,8 +156,11 @@ and recruitment_id = '${IDM_ID}';
<arguments> <arguments>
</arguments> </arguments>
<connection>ervu-dashboard</connection> <connection>ervu-dashboard</connection>
<delete_field/>
<execute_each_row>Y</execute_each_row> <execute_each_row>Y</execute_each_row>
<insert_field/>
<quoteString>N</quoteString> <quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables> <replace_variables>Y</replace_variables>
<set_params>N</set_params> <set_params>N</set_params>
<single_statement>N</single_statement> <single_statement>N</single_statement>
@ -195,14 +171,15 @@ WHERE job_name = '${JOB_NAME}'
and recruitment_id = '${IDM_ID}'; and recruitment_id = '${IDM_ID}';
</sql> </sql>
<update_field/>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>1200</xloc> <xloc>1280</xloc>
<yloc>192</yloc> <yloc>176</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>Create job execution record</name> <name>сreate_job_execution_record_BC</name>
<type>ExecSql</type> <type>ExecSql</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>Y</distribute>
@ -215,8 +192,11 @@ and recruitment_id = '${IDM_ID}';
<arguments> <arguments>
</arguments> </arguments>
<connection>ervu-dashboard</connection> <connection>ervu-dashboard</connection>
<delete_field/>
<execute_each_row>N</execute_each_row> <execute_each_row>N</execute_each_row>
<insert_field/>
<quoteString>N</quoteString> <quoteString>N</quoteString>
<read_field/>
<replace_variables>Y</replace_variables> <replace_variables>Y</replace_variables>
<set_params>N</set_params> <set_params>N</set_params>
<single_statement>N</single_statement> <single_statement>N</single_statement>
@ -224,6 +204,7 @@ and recruitment_id = '${IDM_ID}';
VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}'); VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</sql> </sql>
<update_field/>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>288</xloc> <xloc>288</xloc>
@ -231,7 +212,7 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>Detect empty stream</name> <name>detect_empty_stream_BC</name>
<type>DetectEmptyStream</type> <type>DetectEmptyStream</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>Y</distribute>
@ -243,12 +224,12 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</partitioning> </partitioning>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>496</xloc> <xloc>640</xloc>
<yloc>192</yloc> <yloc>176</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>Filter rows</name> <name>filter_last_row_BC</name>
<type>FilterRows</type> <type>FilterRows</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>Y</distribute>
@ -276,18 +257,18 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
</value> </value>
</condition> </condition>
</compare> </compare>
<send_true_to>Change job status on success</send_true_to> <send_true_to>change_job_status_on_success_BC</send_true_to>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>1200</xloc> <xloc>1280</xloc>
<yloc>320</yloc> <yloc>320</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>Table input</name> <name>table_input_BC</name>
<type>TableInput</type> <type>TableInput</type>
<description/> <description/>
<distribute>N</distribute> <distribute>Y</distribute>
<custom_distribution/> <custom_distribution/>
<copies>1</copies> <copies>1</copies>
<partitioning> <partitioning>
@ -298,16 +279,14 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
<execute_each_row>N</execute_each_row> <execute_each_row>N</execute_each_row>
<limit/> <limit/>
<sql>WITH filteredData AS (SELECT ri.recruit_id, <sql>WITH filteredData AS (SELECT ri.recruit_id,
ri.info->'svedFL'->'extend'->'svedPeresechGran' as border_cross_arr, ri.info->'svedFL'->'extend'->'svedPeresechGran' as border_cross_arr
ctid
FROM ervu_dashboard.recruits_info ri FROM ervu_dashboard.recruits_info ri
WHERE jsonb_typeof(ri.info->'svedFL'->'extend'->'svedPeresechGran') = 'array' WHERE jsonb_typeof(ri.info->'svedFL'->'extend'->'svedPeresechGran') = 'array'
AND coalesce(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}' AND coalesce(ri.current_recruitment_id, ri.target_recruitment_id) = '${IDM_ID}'
) )
SELECT to_date(border_cross_elem->>'dataVyezd' , 'YYYY-MM-DD') as date_crossing, SELECT to_date(border_cross_elem->>'dataVyezd' , 'YYYY-MM-DD') as date_crossing,
to_date(border_cross_elem->>'dataVozvrashh' , 'YYYY-MM-DD') as return_date, to_date(border_cross_elem->>'dataVozvrashh' , 'YYYY-MM-DD') as return_date,
true as border_crossed, true as border_crossed
fd.ctid = (select max(ctid) from filteredData) as last_row
FROM filteredData fd FROM filteredData fd
CROSS JOIN LATERAL jsonb_array_elements(border_cross_arr) AS border_cross_elem;</sql> CROSS JOIN LATERAL jsonb_array_elements(border_cross_arr) AS border_cross_elem;</sql>
<variables_active>Y</variables_active> <variables_active>Y</variables_active>
@ -318,7 +297,7 @@ FROM filteredData fd
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>Table output</name> <name>table_output_BC</name>
<type>TableOutput</type> <type>TableOutput</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>Y</distribute>
@ -348,23 +327,26 @@ FROM filteredData fd
<only_when_have_rows>N</only_when_have_rows> <only_when_have_rows>N</only_when_have_rows>
<partitioning_daily>N</partitioning_daily> <partitioning_daily>N</partitioning_daily>
<partitioning_enabled>N</partitioning_enabled> <partitioning_enabled>N</partitioning_enabled>
<partitioning_field/>
<partitioning_monthly>Y</partitioning_monthly> <partitioning_monthly>Y</partitioning_monthly>
<return_field/>
<return_keys>N</return_keys> <return_keys>N</return_keys>
<schema>ervu_dashboard</schema> <schema>ervu_dashboard</schema>
<specify_fields>Y</specify_fields> <specify_fields>Y</specify_fields>
<table>border_crossing</table> <table>border_crossing</table>
<tablename_field/>
<tablename_in_field>N</tablename_in_field> <tablename_in_field>N</tablename_in_field>
<tablename_in_table>Y</tablename_in_table> <tablename_in_table>Y</tablename_in_table>
<truncate>N</truncate> <truncate>N</truncate>
<use_batch>Y</use_batch> <use_batch>Y</use_batch>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>784</xloc> <xloc>912</xloc>
<yloc>320</yloc> <yloc>320</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>Update</name> <name>update_citizen_BC</name>
<type>Update</type> <type>Update</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>Y</distribute>
@ -377,11 +359,13 @@ FROM filteredData fd
<commit>10000</commit> <commit>10000</commit>
<connection>ervu-dashboard</connection> <connection>ervu-dashboard</connection>
<error_ignored>Y</error_ignored> <error_ignored>Y</error_ignored>
<ignore_flag_field/>
<lookup> <lookup>
<key> <key>
<condition>=</condition> <condition>=</condition>
<field>recruit_id</field> <field>recruit_id</field>
<name>recruit_id</name> <name>recruit_id</name>
<name2/>
</key> </key>
<schema>ervu_dashboard</schema> <schema>ervu_dashboard</schema>
<table>citizen</table> <table>citizen</table>
@ -394,12 +378,12 @@ FROM filteredData fd
<use_batch>Y</use_batch> <use_batch>Y</use_batch>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>784</xloc> <xloc>912</xloc>
<yloc>480</yloc> <yloc>480</yloc>
</GUI> </GUI>
</transform> </transform>
<transform> <transform>
<name>sort_by_recruit_id</name> <name>sort_by_recruit_id_BC</name>
<type>SortRows</type> <type>SortRows</type>
<description/> <description/>
<distribute>Y</distribute> <distribute>Y</distribute>
@ -428,14 +412,32 @@ FROM filteredData fd
</fields> </fields>
<attributes/> <attributes/>
<GUI> <GUI>
<xloc>496</xloc> <xloc>640</xloc>
<yloc>480</yloc> <yloc>480</yloc>
</GUI> </GUI>
</transform> </transform>
<transform>
<name>Identify_last_row_BC</name>
<type>DetectLastRow</type>
<description/>
<distribute>N</distribute>
<custom_distribution/>
<copies>1</copies>
<partitioning>
<method>none</method>
<schema_name/>
</partitioning>
<resultfieldname>result</resultfieldname>
<attributes/>
<GUI>
<xloc>640</xloc>
<yloc>320</yloc>
</GUI>
</transform>
<transform_error_handling> <transform_error_handling>
<error> <error>
<source_transform>Table output</source_transform> <source_transform>table_output_BC</source_transform>
<target_transform>Change job status on error</target_transform> <target_transform>change_job_status_on_error_BC</target_transform>
<is_enabled>Y</is_enabled> <is_enabled>Y</is_enabled>
<nr_valuename/> <nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename> <descriptions_valuename>error_description</descriptions_valuename>
@ -446,8 +448,8 @@ FROM filteredData fd
<min_pct_rows/> <min_pct_rows/>
</error> </error>
<error> <error>
<source_transform>Update</source_transform> <source_transform>update_citizen_BC</source_transform>
<target_transform>Change job status on error</target_transform> <target_transform>change_job_status_on_error_BC</target_transform>
<is_enabled>Y</is_enabled> <is_enabled>Y</is_enabled>
<nr_valuename/> <nr_valuename/>
<descriptions_valuename>error_description</descriptions_valuename> <descriptions_valuename>error_description</descriptions_valuename>