diff --git a/backend/src/main/java/ervu/client/okopf/EsnsiOkopfClient.java b/backend/src/main/java/ervu/client/okopf/EsnsiOkopfClient.java index d6b98da1..619429e8 100644 --- a/backend/src/main/java/ervu/client/okopf/EsnsiOkopfClient.java +++ b/backend/src/main/java/ervu/client/okopf/EsnsiOkopfClient.java @@ -22,25 +22,26 @@ import org.springframework.stereotype.Component; */ @Component public class EsnsiOkopfClient { - private static final Logger LOGGER = LoggerFactory.getLogger(EsnsiOkopfClient.class); + private static final Logger logger = LoggerFactory.getLogger(EsnsiOkopfClient.class); @Value("${esnsi.okopf.url}") private String url; - @Retryable(maxAttemptsExpression = "${esnsi.okopf.retry.max.attempts.load:3}", backoff = - @Backoff(delayExpression = "${esnsi.okop.retry.delay.load:1000}")) + @Retryable(value = IOException.class, maxAttemptsExpression = "${esnsi.okopf.retry.max.attempts.load:3}", backoff = + @Backoff(delayExpression = "${esnsi.okop.retry.delay.load:30000}")) public String getJsonOkopFormData() { try (BufferedInputStream in = new BufferedInputStream(new URL(url).openStream()); ZipInputStream archiveStream = new ZipInputStream(in); BufferedReader br = new BufferedReader( new InputStreamReader(archiveStream, StandardCharsets.UTF_8))) { if (Objects.nonNull(archiveStream.getNextEntry())) { + logger.info("Received an archive in response."); return br.lines().collect(Collectors.joining(System.lineSeparator())); } - LOGGER.warn("Esnsi okopf client received an empty archive in response."); + logger.info("Received an empty archive in response. Skipping load okpof file process"); } - catch (IOException e) { - LOGGER.error("Failed to send HTTP request or process the response for okopf file.", e); + catch (SecurityException | IOException e) { + logger.error("Failed to send HTTP request {} or process the response for okopf file.", url, e); } return null; } diff --git a/backend/src/main/java/ervu/dao/okopf/OkopfDaoImpl.java b/backend/src/main/java/ervu/dao/okopf/OkopfDaoImpl.java index 9b791b04..739e9b0e 100644 --- a/backend/src/main/java/ervu/dao/okopf/OkopfDaoImpl.java +++ b/backend/src/main/java/ervu/dao/okopf/OkopfDaoImpl.java @@ -24,6 +24,7 @@ public class OkopfDaoImpl implements OkopfDao { private DSLContext dsl; @Override + @Transactional public void saveOrUpdate(List okopfModels) { deleteNotActualOkopfRecords(okopfModels); dsl.batchMerge(okopfModels.stream() diff --git a/backend/src/main/java/ervu/service/scheduler/EsnsiOkopfSchedulerServiceImpl.java b/backend/src/main/java/ervu/service/scheduler/EsnsiOkopfSchedulerServiceImpl.java index b182f553..3774e385 100644 --- a/backend/src/main/java/ervu/service/scheduler/EsnsiOkopfSchedulerServiceImpl.java +++ b/backend/src/main/java/ervu/service/scheduler/EsnsiOkopfSchedulerServiceImpl.java @@ -5,7 +5,6 @@ import java.util.List; import java.util.stream.Stream; import javax.annotation.PostConstruct; -import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import ervu.client.okopf.EsnsiOkopfClient; import ervu.dao.okopf.OkopfDao; @@ -22,7 +21,6 @@ import org.springframework.context.annotation.DependsOn; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; -import ru.micord.ervu.exception.JsonParsingException; import static org.springframework.scheduling.config.ScheduledTaskRegistrar.CRON_DISABLED; import static org.springframework.util.StringUtils.hasText; @@ -34,7 +32,7 @@ import static org.springframework.util.StringUtils.hasText; @Service @DependsOn({"liquibase"}) public class EsnsiOkopfSchedulerServiceImpl implements EsnsiOkopfSchedulerService { - private static final Logger LOGGER = LoggerFactory.getLogger( + private static final Logger logger = LoggerFactory.getLogger( EsnsiOkopfSchedulerServiceImpl.class); @Autowired @@ -50,11 +48,11 @@ public class EsnsiOkopfSchedulerServiceImpl implements EsnsiOkopfSchedulerServic @Transactional public void init() { if (!cronLoad.equals(CRON_DISABLED)) { - LOGGER.info("Synchronization with OKOPF enabled"); + logger.info("Synchronization with esnsi okopf enabled"); load(); } else { - LOGGER.info("Synchronization with OKOPF disabled"); + logger.info("Synchronization with esnsi okopf disabled"); } } @@ -62,21 +60,22 @@ public class EsnsiOkopfSchedulerServiceImpl implements EsnsiOkopfSchedulerServic @SchedulerLock(name = "loadOkopf") @Transactional public void load() { - LOGGER.info("Loading okopf file"); - String data = esnsiOkopfClient.getJsonOkopFormData(); + logger.info("Loading okopf file"); try { + String data = esnsiOkopfClient.getJsonOkopFormData(); if (hasText(data)) { + logger.info("Parsing from json file to okopf model"); OkopfOrgModel orgModel = mapper.readValue(data, OkopfOrgModel.class); int currentVersion = mapper.readTree(data).findValue("version").asInt(); List okopfRecords = mapToOkopfRecords(orgModel.getData(), currentVersion); + logger.info("Finished parsing from json file to okopf model"); + logger.info("Loaded {} okopf records with version {}. Beginning to save okopf data", okopfRecords.size(), currentVersion); okopfDao.saveOrUpdate(okopfRecords); + logger.info("Successfully saved okopf data"); } } - catch (JsonProcessingException e) { - throw new JsonParsingException( - "Failed to process the JSON response from the esnsi okopf client. Invalid or malformed JSON.", - e - ); + catch (Exception e) { + logger.error("Failed to load okopf data", e); } } diff --git a/config.md b/config.md index 721ab25a..23e8ba02 100644 --- a/config.md +++ b/config.md @@ -786,7 +786,7 @@ JBPM использует 3 корневых категории логирова - `ESNSI_OKOPF_CRON_LOAD` - настройка, которая указывет расписание для загрузки справочника окопф и сохранение данных по справкам в БД - `ESNSI_OKOPF_RETRY_DELAY_LOAD` - настройка, которая указывет на повторную попытку загрузить - справочник окопф с задержкой. По умолчанию задержка по времени 1000 ms + справочник окопф с задержкой. По умолчанию задержка по времени 30000 ms - `ESNSI_OKOPF_RETRY_MAX_ATTEMPTS_LOAD` - настройка, которая указывет на максимальное кол-во попыток повторно загрузить справочник окопф. По умолчанию 3 попытки diff --git a/config/micord.env b/config/micord.env index 53692220..b04988ee 100644 --- a/config/micord.env +++ b/config/micord.env @@ -48,7 +48,7 @@ ERVU_KAFKA_EXCERPT_REQUEST_TOPIC=ervu.lkrp.excerpt.request ESNSI_OKOPF_URL=https://esnsi.gosuslugi.ru/rest/ext/v1/classifiers/16271/file?extension=JSON&encoding=UTF_8 ESNSI_OKOPF_CRON_LOAD=0 0 */1 * * * ESNSI_OKOPF_RETRY_MAX_ATTEMPTS_LOAD=3 -ESNSI_OKOPF_RETRY_DELAY_LOAD=1000 +ESNSI_OKOPF_RETRY_DELAY_LOAD=30000 ERVU_FILE_UPLOAD_MAX_FILE_SIZE=5242880 ERVU_FILE_UPLOAD_MAX_REQUEST_SIZE=6291456 diff --git a/config/standalone/dev/standalone.xml b/config/standalone/dev/standalone.xml index 75ebaa8d..6dec5b05 100644 --- a/config/standalone/dev/standalone.xml +++ b/config/standalone/dev/standalone.xml @@ -85,7 +85,7 @@ - +