SUPPORT-9561: fix url saving and scheduler

backend/src/main/java/ervu/enums/ExcerptStatus.java

@@ -0,0 +1,14 @@
+package ervu.enums;
+
+import ru.cg.webbpm.modules.webkit.annotations.Model;
+
+/**
+ * @author gulnaz
+ */
+@Model
+public enum ExcerptStatus {
+  NONE,
+  PENDING,
+  READY,
+  NOT_FOUND
+}

backend/src/main/java/ru/micord/ervu/dao/ExcerptHistoryDao.java

@@ -5,6 +5,7 @@ import java.time.LocalDateTime;
 import java.time.temporal.ChronoUnit;
 import java.util.Optional;
 
+import ervu.enums.ExcerptStatus;
 import ervu_lkrp_ul.ervu_lkrp_ul.db_beans.public_.tables.records.ExcerptHistoryRecord;
 import org.jooq.DSLContext;
 import org.jooq.impl.DSL;
@@ -29,7 +30,7 @@ public class ExcerptHistoryDao {
   }
 
   public Optional<ExcerptHistoryRecord> get(String ervuId, String prnOid) {
-    return dslContext.select(EXCERPT_HISTORY.ID)
+    return dslContext.select(EXCERPT_HISTORY.ID, EXCERPT_HISTORY.URL)
         .from(EXCERPT_HISTORY)
         .where(EXCERPT_HISTORY.ID.eq(
             select(DSL.max(EXCERPT_HISTORY.ID))
@@ -43,19 +44,11 @@ public class ExcerptHistoryDao {
 
   public String getStatus(String ervuId, String prnOid) {
     return dslContext.select(coalesce(
-        field(
-            select(
-                case_()
-                    .when(EXCERPT_HISTORY.DATETIME.isNull(), "in-progress")
-                    .else_("ready")
-            )
+        field(select(EXCERPT_HISTORY.STATUS)
                 .from(EXCERPT_HISTORY)
-                .where(
-                    EXCERPT_HISTORY.ERVU_ID.eq(ervuId)
-                        .and(EXCERPT_HISTORY.PRN_OID.eq(prnOid))
-                )
-        ),
-        "none"
+                .where(EXCERPT_HISTORY.ERVU_ID.eq(ervuId)
+                        .and(EXCERPT_HISTORY.PRN_OID.eq(prnOid)))
+        ), ExcerptStatus.NONE.name()
     )).fetchAnyInto(String.class);
   }
 
@@ -63,21 +56,25 @@ public class ExcerptHistoryDao {
     dslContext.insertInto(EXCERPT_HISTORY)
         .set(EXCERPT_HISTORY.ERVU_ID, ervuId)
         .set(EXCERPT_HISTORY.PRN_OID, prnOid)
-        .execute();
-  }
-
-  public void insert(String ervuId, String prnOid, String url) {
-    dslContext.insertInto(EXCERPT_HISTORY)
-        .set(EXCERPT_HISTORY.ERVU_ID, ervuId)
-        .set(EXCERPT_HISTORY.PRN_OID, prnOid)
-        .set(EXCERPT_HISTORY.URL, url)
+        .set(EXCERPT_HISTORY.STATUS, ExcerptStatus.PENDING.name())
         .set(EXCERPT_HISTORY.DATETIME, Timestamp.valueOf(LocalDateTime.now()))
         .execute();
   }
 
-  public void update(Long id, String url) {
+  public void insert(String ervuId, String prnOid, String url, String status) {
+    dslContext.insertInto(EXCERPT_HISTORY)
+        .set(EXCERPT_HISTORY.ERVU_ID, ervuId)
+        .set(EXCERPT_HISTORY.PRN_OID, prnOid)
+        .set(EXCERPT_HISTORY.URL, url)
+        .set(EXCERPT_HISTORY.STATUS, status)
+        .set(EXCERPT_HISTORY.DATETIME, Timestamp.valueOf(LocalDateTime.now()))
+        .execute();
+  }
+
+  public void update(Long id, String url, String status) {
     dslContext.update(EXCERPT_HISTORY)
         .set(EXCERPT_HISTORY.URL, url)
+        .set(EXCERPT_HISTORY.STATUS, status)
         .set(EXCERPT_HISTORY.DATETIME, Timestamp.valueOf(LocalDateTime.now()))
         .where(EXCERPT_HISTORY.ID.eq(id))
         .execute();
@@ -90,10 +87,14 @@ public class ExcerptHistoryDao {
         .execute();
   }
 
-  public void deleteOlderThan(long hours) {
+  public void deleteOlderThan(long readyFileHours, long pendingFileHours) {
     dslContext.deleteFrom(EXCERPT_HISTORY)
-        .where(EXCERPT_HISTORY.DATETIME.le(
-            Timestamp.valueOf(LocalDateTime.now().minus(hours, ChronoUnit.HOURS))))
+        .where(EXCERPT_HISTORY.STATUS.eq(ExcerptStatus.READY.name())
+            .and(EXCERPT_HISTORY.DATETIME.le(
+            Timestamp.valueOf(LocalDateTime.now().minus(readyFileHours, ChronoUnit.HOURS)))))
+        .or(EXCERPT_HISTORY.STATUS.in(ExcerptStatus.PENDING.name(), ExcerptStatus.NOT_FOUND.name())
+            .and(EXCERPT_HISTORY.DATETIME.le(
+                Timestamp.valueOf(LocalDateTime.now().minus(pendingFileHours, ChronoUnit.HOURS)))))
         .execute();
   }
 }

backend/src/main/java/ru/micord/ervu/kafka/listener/ExcerptListener.java

@@ -1,9 +1,9 @@
 package ru.micord.ervu.kafka.listener;
 
 import java.util.UUID;
-import java.util.concurrent.atomic.AtomicReference;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
+import ervu.enums.ExcerptStatus;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.kafka.annotation.KafkaListener;
@@ -50,7 +50,7 @@ public class ExcerptListener {
       if (!excerptResponse.getSuccess()) {
         if (excerptResponse.getData() != null) {
           data = excerptResponse.getData();
-          excerptHistoryDao.delete(data.getOrgId(), data.getPrnOid());
+          saveUrl(data.getOrgId(), data.getPrnOid(), fileUrl);
         }
         throw new ExcerptResponseException(
             "Error with getting excerpt url " + excerptResponse.getMessage());
@@ -59,7 +59,7 @@ public class ExcerptListener {
                || excerptResponse.getData().getFileUrl().isEmpty()) {
         if (excerptResponse.getData() != null) {
           data = excerptResponse.getData();
-          excerptHistoryDao.delete(data.getOrgId(), data.getPrnOid());
+          saveUrl(data.getOrgId(), data.getPrnOid(), fileUrl);
         }
         auditService.processDownloadEvent(data, 0, fileName, 1,
             AuditConstants.FAILURE_STATUS_TYPE, fileUrl
@@ -70,14 +70,7 @@ public class ExcerptListener {
       data = excerptResponse.getData();
       fileUrl = data.getFileUrl();
       fileName = UrlUtils.extractFileNameFromUrl(fileUrl);
-      String ervuId = excerptResponse.getData().getOrgId();
-      String prnOid = excerptResponse.getData().getPrnOid();
-      AtomicReference<String> fileRef = new AtomicReference<>(fileUrl);
-      excerptHistoryDao.get(ervuId, prnOid)
-          .ifPresentOrElse(
-              record -> excerptHistoryDao.update(record.getId(), fileRef.get()),
-              () -> excerptHistoryDao.insert(ervuId, prnOid, fileRef.get())
-          );
+      saveUrl(data.getOrgId(), data.getPrnOid(), fileUrl);
     }
     catch (Exception e) {
       auditService.processDownloadEvent(data, 0, fileName, 1,
@@ -90,4 +83,12 @@ public class ExcerptListener {
   public String getGroupId() {
     return groupId + "-" + UUID.randomUUID();
   }
+
+  private void saveUrl(String ervuId, String prnOid, String fileUrl) {
+    String status = fileUrl == null ? ExcerptStatus.NOT_FOUND.name() : ExcerptStatus.READY.name();
+    excerptHistoryDao.get(ervuId, prnOid).ifPresentOrElse(
+        record -> excerptHistoryDao.update(record.getId(), fileUrl, status),
+        () -> excerptHistoryDao.insert(ervuId, prnOid, fileUrl, status)
+    );
+  }
 }

backend/src/main/java/ru/micord/ervu/scheduler/ExcerptCleanScheduler.java

@@ -14,16 +14,18 @@ public class ExcerptCleanScheduler {
 
   private final ExcerptHistoryDao excerptHistoryDao;
 
-  @Value("${excerpt.lifetime.hours:24}")
-  private long fileLifetime;
+  @Value("${excerpt.ready.lifetime.hours:24}")
+  private long readyFileLifetime;
+  @Value("${excerpt.pending.lifetime.hours:8}")
+  private long pendingFileLifetime;
 
   public ExcerptCleanScheduler(ExcerptHistoryDao excerptHistoryDao) {
     this.excerptHistoryDao = excerptHistoryDao;
   }
 
-  @Scheduled(fixedDelayString = "${excerpt.cleanup.delay:30000}")
+  @Scheduled(cron = "${excerpt.cleanup.cron:0 */30 * * * *}")
   @SchedulerLock(name = "oldExcerptFilesCleaning")
   public void cleanOld() {
-    excerptHistoryDao.deleteOlderThan(fileLifetime);
+    excerptHistoryDao.deleteOlderThan(readyFileLifetime, pendingFileLifetime);
   }
 }