Merge branch 'develop' of 10.10.31.70:/ervu-dashboard-etl into develop

mappings/info_recruits/citizen_tables/work/employer/parallel/employer_flow.hpl

@@ -7,7 +7,18 @@
     <extended_description/>
     <pipeline_version/>
     <pipeline_type>Normal</pipeline_type>
+    <pipeline_status>0</pipeline_status>
     <parameters>
+      <parameter>
+        <name>IDM_ID</name>
+        <default_value>0e0535ab-5303-43ea-b36a-8c0a2a8d0ed5</default_value>
+        <description/>
+      </parameter>
+      <parameter>
+        <name>JOB_NAME</name>
+        <default_value>employer_job</default_value>
+        <description/>
+      </parameter>
     </parameters>
     <capture_transform_performance>N</capture_transform_performance>
     <transform_performance_capturing_delay>1000</transform_performance_capturing_delay>
@@ -20,11 +31,6 @@
   <notepads>
   </notepads>
   <order>
-    <hop>
-      <from>Create job execution record</from>
-      <to>employer_input</to>
-      <enabled>Y</enabled>
-    </hop>
     <hop>
       <from>Change job status on error</from>
       <to>Abort</to>
@@ -85,6 +91,11 @@
       <to>Filter rows</to>
       <enabled>Y</enabled>
     </hop>
+    <hop>
+      <from>Create job execution record</from>
+      <to>employer_input</to>
+      <enabled>Y</enabled>
+    </hop>
   </order>
   <transform>
     <name>Abort</name>
@@ -196,35 +207,6 @@ and recruitment_id = '${IDM_ID}';
       <yloc>112</yloc>
     </GUI>
   </transform>
-  <transform>
-    <name>Create job execution record</name>
-    <type>ExecSql</type>
-    <description/>
-    <distribute>Y</distribute>
-    <custom_distribution/>
-    <copies>1</copies>
-    <partitioning>
-      <method>none</method>
-      <schema_name/>
-    </partitioning>
-    <arguments>
-</arguments>
-    <connection>ervu-dashboard</connection>
-    <execute_each_row>N</execute_each_row>
-    <quoteString>N</quoteString>
-    <replace_variables>Y</replace_variables>
-    <set_params>N</set_params>
-    <single_statement>N</single_statement>
-    <sql>INSERT INTO etl.job_execution (id, job_name, status, execution_datetime, error_description, recruitment_id)
-VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
-
-</sql>
-    <attributes/>
-    <GUI>
-      <xloc>176</xloc>
-      <yloc>288</yloc>
-    </GUI>
-  </transform>
   <transform>
     <name>Detect empty stream</name>
     <type>DetectEmptyStream</type>
@@ -342,11 +324,10 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
       <method>none</method>
       <schema_name/>
     </partitioning>
-    <connection>ervu-dashboard</connection>
+    <connection>ervu-dashboard-118</connection>
     <execute_each_row>N</execute_each_row>
     <limit>0</limit>
     <sql>with filteredData as (select ri.recruit_id,
-                             (ri.info -> 'svedFL' -> 'svedRabotodat' -> 'rabotodat' ->> 'id')        as source_id,
                              ri.info -> 'svedFL' -> 'svedRabotodat' -> 'rabotodat' -> 'aktRabotodat' as rabotodat,
                              true                                                                    as actual_employer
                       from ervu_dashboard.recruits_info ri
@@ -356,17 +337,17 @@ VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
                       union all
 
                       select ri.recruit_id,
-                             (ri.info -> 'svedFL' -> 'svedRabotodat' -> 'rabotodat' ->> 'id')         as source_id,
                              ri.info -> 'svedFL' -> 'svedRabotodat' -> 'rabotodat' -> 'predRabotodat' as rabotodat,
                              false                                                                    as actual_employer
                       from ervu_dashboard.recruits_info ri
-                      where ri.info -> 'svedFL' -> 'svedRabotodat' -> 'rabotodat' ->> 'predRabotodat' != 'null'
+                      where ri.info -> 'svedFL' -&gt; 'svedRabotodat' -> 'rabotodat' ->> 'predRabotodat' != 'null'
                         AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}'))
 )
-select fd.source_id,
+select
        fd.recruit_id,
-       to_date(rabotodat ->> 'dataSved', 'YYYY-MM-DD')                           as source_update_date,
-       COALESCE(rabotodat -> 'extend' ->> 'status', '0') = '1'                  AS status,
+       to_date(rabotodat ->> 'dataSved', 'YYYY-MM-DD')                          as source_update_date,
+       rabotodat ->> 'id'                                                       as source_id,
+       COALESCE(rabotodat -> 'extend' ->> 'status', '0') = '1'                  as status,
        rabotodat -> 'extend' ->> 'adresRabotodat'                               as address,
        rabotodat -> 'extend' ->> 'tipPodrazdRabotodat'                          as separate_unit_type,
        rabotodat -> 'extend' ->> 'adresPodrazdRabotodat'                        as separate_unit_address,
@@ -535,6 +516,35 @@ from filteredData fd;</sql>
       <yloc>448</yloc>
     </GUI>
   </transform>
+  <transform>
+    <name>Create job execution record</name>
+    <type>ExecSql</type>
+    <description/>
+    <distribute>Y</distribute>
+    <custom_distribution/>
+    <copies>1</copies>
+    <partitioning>
+      <method>none</method>
+      <schema_name/>
+    </partitioning>
+    <arguments>
+</arguments>
+    <connection>ervu-dashboard</connection>
+    <execute_each_row>N</execute_each_row>
+    <quoteString>N</quoteString>
+    <replace_variables>Y</replace_variables>
+    <set_params>N</set_params>
+    <single_statement>N</single_statement>
+    <sql>INSERT INTO etl.job_execution (id, job_name, status, execution_datetime, error_description, recruitment_id)
+VALUES (DEFAULT, '${JOB_NAME}', 'PROCESSING', DEFAULT, null, '${IDM_ID}');
+
+</sql>
+    <attributes/>
+    <GUI>
+      <xloc>192</xloc>
+      <yloc>288</yloc>
+    </GUI>
+  </transform>
   <transform_error_handling>
     <error>
       <source_transform>Update</source_transform>

mappings/info_recruits/citizen_tables/work/employer/parallel/employer_flow_delta.hpl

@@ -428,7 +428,6 @@ and recruitment_id = '${IDM_ID}';</sql>
     </partitioning>
     <connection>ervu-dashboard</connection>
     <execute_each_row>N</execute_each_row>
-    <limit/>
     <sql>WITH fd AS (SELECT ri.recruit_id,
                    (ri.info -> 'svedFL' -> 'svedRabotodat' -> 'rabotodat' ->> 'id')         AS source_id,
                    ri.info -> 'svedFL' -> 'svedRabotodat' -> 'rabotodat' -> 'aktRabotodat'  AS akt_rabotodat,
@@ -444,7 +443,7 @@ and recruitment_id = '${IDM_ID}';</sql>
               AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}'))
 )
 SELECT fd.recruit_id,
-       fd.source_id,
+	pred_rabotodat ->> 'id'                                                               as pred_source_id,
        to_date(pred_rabotodat ->> 'dataSved', 'YYYY-MM-DD')                                  as pred_source_update_date,
        COALESCE(pred_rabotodat -> 'extend' ->> 'statusRabotodat' = '1', false)               as pred_status,
        pred_rabotodat -> 'extend' ->> 'adresRabotodat'                                       as pred_address,
@@ -467,7 +466,7 @@ SELECT fd.recruit_id,
                then pred_rabotodat -> 'svedYUL' ->> 'innyul'
            when pred_rabotodat ->> 'svedGlKFH' is not null and pred_rabotodat ->> 'svedGlKFH' != 'null'
                then pred_rabotodat -> 'svedGlKFH' ->> 'innfl'
-           when pred_rabotodat ->> 'svedPlatFL' is not null and pred_rabotodat ->&gt; 'svedPlatFL' != 'null'
+           when pred_rabotodat ->> 'svedPlatFL' is not null and pred_rabotodat ->> 'svedPlatFL' != 'null'
                then pred_rabotodat -> 'svedPlatFL' ->> 'innfl'
            end                                                                               as pred_inn,
        case
@@ -484,6 +483,7 @@ SELECT fd.recruit_id,
        to_date(fd.pred_rabotodat ->> 'dataSved', 'YYYY-MM-DD') > '${MAX_SOURCE_UPDATE_DATE}' AS pred_need_update,
 
 
+       akt_rabotodat ->> 'id'                                                                as akt_source_id,
        to_date(akt_rabotodat ->> 'dataSved', 'YYYY-MM-DD')                                   as akt_source_update_date,
        COALESCE(akt_rabotodat -> 'extend' ->> 'statusRabotodat' = '1', false)                as akt_status,
        akt_rabotodat -> 'extend' ->> 'adresRabotodat'                                        as akt_address,
@@ -737,11 +737,6 @@ FROM fd;</sql>
         <field>source_id</field>
         <name>source_id</name>
       </key>
-      <key>
-        <condition>=</condition>
-        <field>actual_employer</field>
-        <name>akt_need_update</name>
-      </key>
       <schema>ervu_dashboard</schema>
       <table>employer</table>
       <value>
@@ -751,7 +746,7 @@ FROM fd;</sql>
       </value>
       <value>
         <name>source_id</name>
-        <rename>source_id</rename>
+        <rename>akt_source_id</rename>
         <update>N</update>
       </value>
       <value>
@@ -841,11 +836,6 @@ FROM fd;</sql>
         <field>source_id</field>
         <name>source_id</name>
       </key>
-      <key>
-        <condition>&lt;></condition>
-        <field>actual_employer</field>
-        <name>pred_need_update</name>
-      </key>
       <schema>ervu_dashboard</schema>
       <table>employer</table>
       <value>
@@ -855,7 +845,7 @@ FROM fd;</sql>
       </value>
       <value>
         <name>source_id</name>
-        <rename>source_id</rename>
+        <rename>pred_source_id</rename>
         <update>N</update>
       </value>
       <value>

mappings/info_recruits/citizen_tables/work/employer/parallel/employer_flow_repeat.hpl

@@ -350,7 +350,6 @@ SET status = 'PROCESSING',
     <execute_each_row>N</execute_each_row>
     <limit>0</limit>
     <sql>with filteredData as (select ri.recruit_id,
-                             (ri.info -> 'svedFL' -> 'svedRabotodat' -> 'rabotodat' ->> 'id')        as source_id,
                              ri.info -> 'svedFL' -> 'svedRabotodat' -> 'rabotodat' -> 'aktRabotodat' as rabotodat,
                              true                                                                    as actual_employer
                       from ervu_dashboard.recruits_info ri
@@ -360,22 +359,22 @@ SET status = 'PROCESSING',
                       union all
 
                       select ri.recruit_id,
-                             (ri.info -> 'svedFL' -> 'svedRabotodat' -> 'rabotodat' ->> 'id')         as source_id,
                              ri.info -> 'svedFL' -> 'svedRabotodat' -> 'rabotodat' -> 'predRabotodat' as rabotodat,
                              false                                                                    as actual_employer
                       from ervu_dashboard.recruits_info ri
-                      where ri.info -> 'svedFL' -> 'svedRabotodat' -> 'rabotodat' ->> 'predRabotodat' != 'null'
-                      AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}'))
+                      where ri.info -> 'svedFL' -&gt; 'svedRabotodat' -> 'rabotodat' ->> 'predRabotodat' != 'null'
+                        AND (ri.current_recruitment_id = '${IDM_ID}' or (ri.current_recruitment_id is null and ri.target_recruitment_id = '${IDM_ID}'))
 )
-select fd.source_id,
+select
        fd.recruit_id,
        to_date(rabotodat ->> 'dataSved', 'YYYY-MM-DD')                          as source_update_date,
-       COALESCE(rabotodat -> 'extend' ->> 'status', '0') = '1'                  AS status,
+       rabotodat ->> 'id'                                                       as source_id,
+       COALESCE(rabotodat -> 'extend' ->> 'status', '0') = '1'                  as status,
        rabotodat -> 'extend' ->> 'adresRabotodat'                               as address,
        rabotodat -> 'extend' ->> 'tipPodrazdRabotodat'                          as separate_unit_type,
        rabotodat -> 'extend' ->> 'adresPodrazdRabotodat'                        as separate_unit_address,
        case
-           when rabotodat ->&gt; 'svedIP' is not null and rabotodat ->> 'svedIP' != 'null'
+           when rabotodat ->> 'svedIP' is not null and rabotodat ->> 'svedIP' != 'null'
                then 'Индивидуальный предприниматель'
            when rabotodat ->> 'svedYUL' is not null and rabotodat ->> 'svedYUL' != 'null' then 'Юридическое лицо'
            when rabotodat ->> 'svedGlKFH' is not null and rabotodat ->> 'svedGlKFH' != 'null' then 'Глава КФХ'