From a5e9483e1d339ec4691ead3778a0044b7485aef2 Mon Sep 17 00:00:00 2001 From: Zaripov Emil Date: Fri, 25 Oct 2024 11:59:24 +0300 Subject: [PATCH 01/15] set version 1.9.0-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index f33438a..56aab69 100644 --- a/pom.xml +++ b/pom.xml @@ -13,7 +13,7 @@ ru.micord.ervu.lkrp file-upload - 1.8.2 + 1.9.0-SNAPSHOT From 075b1d57b8b45ad11ef10dded628fca3a9c7e86d Mon Sep 17 00:00:00 2001 From: Zaripov Emil Date: Tue, 5 Nov 2024 12:21:02 +0300 Subject: [PATCH 02/15] set version 1.9.2-SNAPSHOT # Conflicts: # pom.xml --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 194bc8c..1c71350 100644 --- a/pom.xml +++ b/pom.xml @@ -13,7 +13,7 @@ ru.micord.ervu.lkrp file-upload - 1.9.1-SNAPSHOT + 1.9.2-SNAPSHOT From 7011ead72dbd8597440b997b37ac24d373c3cc5b Mon Sep 17 00:00:00 2001 From: Zaripov Emil Date: Fri, 8 Nov 2024 14:03:04 +0300 Subject: [PATCH 03/15] set version 1.10.0-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index e16724f..c58ab24 100644 --- a/pom.xml +++ b/pom.xml @@ -13,7 +13,7 @@ ru.micord.ervu.lkrp file-upload - 1.9.2-SNAPSHOT + 1.10.0-SNAPSHOT From fb43861d540d3c11f2d29a2490030e6078b84ae3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A5=D0=B0=D0=BB=D1=82=D0=BE=D0=B1=D0=B8=D0=BD=20=D0=95?= =?UTF-8?q?=D0=B2=D0=B3=D0=B5=D0=BD=D0=B8=D0=B9?= Date: Fri, 8 Nov 2024 15:01:00 +0300 Subject: [PATCH 04/15] removed datasource config --- micord.env | 4 ---- 1 file changed, 4 deletions(-) diff --git a/micord.env b/micord.env index 736980a..25b4ae4 100644 --- a/micord.env +++ b/micord.env @@ -30,7 +30,3 @@ S3_ACCESS_KEY=rlTdTvkmSXu9FsLhfecw S3_SECRET_KEY=NUmY0wwRIEyAd98GCKd1cOgJWvLQYAcMMul5Ulu0 S3_BUCKET_NAME=default-out-bucket S3_PATH_STYLE_ACCESS_ENABLED=true - -SPRING_DATASOURCE_URL=jdbc:postgresql://10.10.31.119:5432/ervu-lkrp-ul -SPRING_DATASOURCE_USERNAME=ervu-lkrp-ul -SPRING_DATASOURCE_PASSWORD=ervu-lkrp-ul From f130f17ea4f0e3be3b6f4d50d0888af13d99d891 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A5=D0=B0=D0=BB=D1=82=D0=BE=D0=B1=D0=B8=D0=BD=20=D0=95?= =?UTF-8?q?=D0=B2=D0=B3=D0=B5=D0=BD=D0=B8=D0=B9?= Date: Wed, 20 Nov 2024 13:13:34 +0300 Subject: [PATCH 05/15] fixed DownloadRequest --- src/main/java/ru/micord/ervu/av/kafka/dto/DownloadRequest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/ru/micord/ervu/av/kafka/dto/DownloadRequest.java b/src/main/java/ru/micord/ervu/av/kafka/dto/DownloadRequest.java index eb49660..f2ab9c7 100644 --- a/src/main/java/ru/micord/ervu/av/kafka/dto/DownloadRequest.java +++ b/src/main/java/ru/micord/ervu/av/kafka/dto/DownloadRequest.java @@ -18,7 +18,7 @@ public record DownloadRequest(OrgInfo orgInfo, @NonNull FileInfo fileInfo) { private final String fileId; private final String fileName; private final String filePatternCode; - private final String FilePatternName; + private final String filePatternName; private final String departureDateTime; private final String timeZone; @Setter From 94f9378fdae8b2556c2a29985984ff3d42e53937 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A5=D0=B0=D0=BB=D1=82=D0=BE=D0=B1=D0=B8=D0=BD=20=D0=95?= =?UTF-8?q?=D0=B2=D0=B3=D0=B5=D0=BD=D0=B8=D0=B9?= Date: Wed, 20 Nov 2024 13:35:49 +0300 Subject: [PATCH 06/15] fixed FileUploadService --- .../ru/micord/ervu/av/service/FileUploadService.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/main/java/ru/micord/ervu/av/service/FileUploadService.java b/src/main/java/ru/micord/ervu/av/service/FileUploadService.java index b4f44a7..0764c90 100644 --- a/src/main/java/ru/micord/ervu/av/service/FileUploadService.java +++ b/src/main/java/ru/micord/ervu/av/service/FileUploadService.java @@ -38,7 +38,7 @@ import ru.micord.ervu.av.s3.S3Service; */ @Service public class FileUploadService { - private static final Logger logger = LoggerFactory.getLogger(FileUploadService.class); + private static final Logger LOGGER = LoggerFactory.getLogger(FileUploadService.class); @Value("${av.check.enabled}") private boolean avCheckEnabled; @Value("${file.saving.path}") @@ -85,7 +85,7 @@ public class FileUploadService { catch (IOException e) { throw new RuntimeException("Failed to create directory " + fileSavingPath, e); } - logger.info("working in {}", System.getProperty("user.home")); + LOGGER.info("working in {}", System.getProperty("user.home")); Path filePath = Paths.get(fileSavingPath, fileUrl.fileName()); String downloadUrl = fileUrl.fileUrl(); fIleManager.downloadFile(downloadUrl, filePath); @@ -102,7 +102,7 @@ public class FileUploadService { .anyMatch(verdict -> verdict.equalsIgnoreCase(AvResponse.Scan.VERDICT_INFECTED)); } catch (FileUploadException | RetryableException e) { - logger.error(e.getMessage(), e); + LOGGER.error(e.getMessage(), e); isAvError = true; } } @@ -140,7 +140,7 @@ public class FileUploadService { catch (InvalidHttpFileUrlException e) { // считаем, что повторная обработка сообщения не нужна // ошибку логируем, отправляем сообщение с новым статусом, помечаем прочтение сообщения - logger.error(e.getMessage() + ": " + kafkaInMessage); + LOGGER.error(e.getMessage() + ": " + kafkaInMessage); downloadRequest.fileInfo().setFileUrl(null); downloadRequest.fileInfo().setFileStatus(FileStatus.FILE_STATUS_11); sendMessage(inStatusTopic.name(), downloadRequest, messageId, inKafkaTemplate); @@ -149,7 +149,7 @@ public class FileUploadService { catch (FileUploadException e) { // считаем, что нужно повторное считывание сообщения // ошибку логируем, сообщение оставляем непрочитанным - logger.error(e.getMessage(), e); + LOGGER.error(e.getMessage(), e); } } From 7163c1ae5a993cc349976fe06ca7a0e39fa0284a Mon Sep 17 00:00:00 2001 From: gulnaz Date: Thu, 28 Nov 2024 12:42:58 +0300 Subject: [PATCH 07/15] SUPPORT-8662: use synchronous request-reply; fix file manager --- micord.env | 3 + .../ervu/av/response/AvFileSendResponse.java | 11 --- .../micord/ervu/av/service/FileManager.java | 32 ++++++- .../service/ReceiveScanReportRetryable.java | 91 +++---------------- src/main/resources/application.properties | 2 + 5 files changed, 48 insertions(+), 91 deletions(-) delete mode 100644 src/main/java/ru/micord/ervu/av/response/AvFileSendResponse.java diff --git a/micord.env b/micord.env index 25b4ae4..a0d39b4 100644 --- a/micord.env +++ b/micord.env @@ -30,3 +30,6 @@ S3_ACCESS_KEY=rlTdTvkmSXu9FsLhfecw S3_SECRET_KEY=NUmY0wwRIEyAd98GCKd1cOgJWvLQYAcMMul5Ulu0 S3_BUCKET_NAME=default-out-bucket S3_PATH_STYLE_ACCESS_ENABLED=true + +WEBDAV_USERNAME=test +WEBDAV_PASSWORD=test diff --git a/src/main/java/ru/micord/ervu/av/response/AvFileSendResponse.java b/src/main/java/ru/micord/ervu/av/response/AvFileSendResponse.java deleted file mode 100644 index ea8dd00..0000000 --- a/src/main/java/ru/micord/ervu/av/response/AvFileSendResponse.java +++ /dev/null @@ -1,11 +0,0 @@ -package ru.micord.ervu.av.response; - -/** - * @author r.latypov - */ -public record AvFileSendResponse(String id, String location, Error error, String status) { - public static final String STATUS_ERROR = "error"; - - public record Error(String code, String message) { - } -} diff --git a/src/main/java/ru/micord/ervu/av/service/FileManager.java b/src/main/java/ru/micord/ervu/av/service/FileManager.java index 4b52afe..044cd58 100644 --- a/src/main/java/ru/micord/ervu/av/service/FileManager.java +++ b/src/main/java/ru/micord/ervu/av/service/FileManager.java @@ -6,12 +6,17 @@ import java.io.IOException; import java.nio.file.Path; import org.apache.http.HttpEntity; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.http.client.CredentialsProvider; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpDelete; import org.apache.http.client.methods.HttpGet; import org.apache.http.conn.HttpHostConnectException; +import org.apache.http.impl.client.BasicCredentialsProvider; import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClients; +import org.apache.http.impl.client.HttpClientBuilder; +import org.springframework.beans.factory.annotation.Value; import org.springframework.http.HttpStatus; import org.springframework.retry.annotation.Backoff; import org.springframework.retry.annotation.Retryable; @@ -20,12 +25,19 @@ import ru.micord.ervu.av.exception.FileUploadException; import ru.micord.ervu.av.exception.InvalidHttpFileUrlException; import ru.micord.ervu.av.exception.RetryableException; +import static org.springframework.util.StringUtils.hasText; + /** * @author gulnaz */ @Component public class FileManager { + @Value("${webdav.username}") + private String username; + @Value("${webdav.password}") + private String password; + @Retryable(retryFor = {InvalidHttpFileUrlException.class}, maxAttemptsExpression = "${av.retry.max.attempts.count}", backoff = @Backoff(delayExpression = "${av.retry.delay.milliseconds}")) @@ -34,7 +46,7 @@ public class FileManager { File file = filePath.toFile(); HttpGet request = new HttpGet(fileUrl); - try (CloseableHttpClient client = HttpClients.createDefault(); + try (CloseableHttpClient client = getClientBuilder().build(); CloseableHttpResponse response = client.execute(request)) { int statusCode = response.getStatusLine().getStatusCode(); @@ -67,7 +79,7 @@ public class FileManager { maxAttemptsExpression = "${av.retry.max.attempts.count}", backoff = @Backoff(delayExpression = "${av.retry.delay.milliseconds}")) public void deleteFile(String fileUrl) throws FileUploadException { - try (CloseableHttpClient client = HttpClients.createDefault()) { + try (CloseableHttpClient client = getClientBuilder().build()) { HttpDelete delete = new HttpDelete(fileUrl); try (CloseableHttpResponse response = client.execute(delete)) { @@ -83,4 +95,18 @@ public class FileManager { throw new FileUploadException(e); } } + + private HttpClientBuilder getClientBuilder() { + HttpClientBuilder httpClientBuilder = HttpClientBuilder.create(); + + if (hasText(username) && hasText(password)) { + CredentialsProvider provider = new BasicCredentialsProvider(); + provider.setCredentials( + AuthScope.ANY, + new UsernamePasswordCredentials(username, password) + ); + httpClientBuilder.setDefaultCredentialsProvider(provider); + } + return httpClientBuilder; + } } diff --git a/src/main/java/ru/micord/ervu/av/service/ReceiveScanReportRetryable.java b/src/main/java/ru/micord/ervu/av/service/ReceiveScanReportRetryable.java index 1b4fbc4..679b05a 100644 --- a/src/main/java/ru/micord/ervu/av/service/ReceiveScanReportRetryable.java +++ b/src/main/java/ru/micord/ervu/av/service/ReceiveScanReportRetryable.java @@ -2,16 +2,16 @@ package ru.micord.ervu.av.service; import java.io.File; import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; import java.nio.file.Path; -import java.util.concurrent.TimeUnit; import com.google.gson.Gson; -import com.google.gson.JsonSyntaxException; import org.apache.http.HttpEntity; import org.apache.http.client.ClientProtocolException; import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.utils.URIBuilder; import org.apache.http.entity.mime.MultipartEntityBuilder; import org.apache.http.entity.mime.content.FileBody; import org.apache.http.impl.client.CloseableHttpClient; @@ -24,7 +24,6 @@ import org.springframework.retry.annotation.Retryable; import org.springframework.stereotype.Service; import ru.micord.ervu.av.exception.FileUploadException; import ru.micord.ervu.av.exception.RetryableException; -import ru.micord.ervu.av.response.AvFileSendResponse; import ru.micord.ervu.av.response.AvResponse; /** @@ -35,8 +34,6 @@ public class ReceiveScanReportRetryable { @Value("${av.rest.address}") private String avRestAddress; - @Value("${av.first.timeout.milliseconds:1000}") - private Long avFirstTimeoutMilliseconds; @Retryable(retryFor = {RetryableException.class}, maxAttemptsExpression = "${av.retry.max.attempts.count}", @@ -45,7 +42,10 @@ public class ReceiveScanReportRetryable { File file = filePath.toFile(); try (CloseableHttpClient client = HttpClients.createDefault()) { - HttpPost post = new HttpPost(avRestAddress); + URI uri = new URIBuilder(avRestAddress) + .addParameter("wait", "1") + .build(); + HttpPost post = new HttpPost(uri); HttpEntity entity = MultipartEntityBuilder.create() .addPart("file", new FileBody(file)) .build(); @@ -54,84 +54,21 @@ public class ReceiveScanReportRetryable { try (CloseableHttpResponse postResponse = client.execute(post)) { int postStatusCode = postResponse.getStatusLine().getStatusCode(); String postResponseJson = EntityUtils.toString(postResponse.getEntity()); - AvFileSendResponse avFileSendResponse; - try { - avFileSendResponse = new Gson().fromJson(postResponseJson, AvFileSendResponse.class); + if (postStatusCode == HttpStatus.OK.value()) { + return new Gson().fromJson(postResponseJson, AvResponse.class); } - catch (JsonSyntaxException e) { - throw new FileUploadException("error json: " + postResponseJson, e); + else { + throw new FileUploadException( + "http status code " + postStatusCode + " for " + post.getURI() + + " get request."); } - - if (postStatusCode != HttpStatus.CREATED.value()) { - StringBuilder stringBuilder = new StringBuilder( - "http status code " + postStatusCode + " for " + avRestAddress + " post request."); - String status = avFileSendResponse.status(); - - if (status != null) { - stringBuilder.append(" Status: ").append(status).append("."); - } - - if (avFileSendResponse.error() != null) { - stringBuilder.append(" Error code: ") - .append(avFileSendResponse.error().code()) - .append(". Error message: ") - .append(avFileSendResponse.error().message()) - .append(". "); - } - throw new FileUploadException(stringBuilder.toString()); - } - - String id = avFileSendResponse.id(); - String reportRequestUri = avRestAddress + "/" + id; - HttpGet get = new HttpGet(reportRequestUri); - - // waiting for timeout time before first request - try { - TimeUnit.MILLISECONDS.sleep(avFirstTimeoutMilliseconds); - } - catch (InterruptedException e) { - throw new FileUploadException(e); - } - - return receiveScanReport(client, get); } } catch (ClientProtocolException e) { throw new RetryableException("Failed to check file"); } - catch (IOException e) { - // непредусмотренная ошибка доступа через http-клиент - throw new FileUploadException(e); - } - } - - private AvResponse receiveScanReport(CloseableHttpClient client, HttpGet get) - throws RetryableException, FileUploadException { - - try (CloseableHttpResponse getResponse = client.execute(get)) { - int getStatusCode = getResponse.getStatusLine().getStatusCode(); - - if (getStatusCode == HttpStatus.OK.value()) { - String getResponseJson = EntityUtils.toString(getResponse.getEntity()); - AvResponse avResponse = new Gson().fromJson(getResponseJson, AvResponse.class); - - if (avResponse.completed() == null) { - throw new RetryableException("Failed to complete file scan"); - } - return avResponse; - } - else { - throw new FileUploadException("http status code " + getStatusCode + " for " + get.getURI() - + " get request."); - } - } - catch (ClientProtocolException e) { - // непредусмотренная ошибка доступа через http-клиент - throw new RetryableException("Failed to receive scan report"); - } - catch (IOException e) { - // непредусмотренная ошибка доступа через http-клиент + catch (IOException | URISyntaxException e) { throw new FileUploadException(e); } } diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index fa9859a..ddf45e4 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -63,6 +63,8 @@ s3.out.secret_key=${S3_SECRET_KEY} s3.out.bucket_name=${S3_OUT_BUCKET_NAME} s3.out.path.style.access.enabled=${S3_OUT_PATH_STYLE_ACCESS_ENABLED} # +webdav.username=${WEBDAV_USERNAME} +webdav.password=${WEBDAV_PASSWORD} # endpoints management management.endpoints.web.exposure.include = health, info, metrics From 0744ca63319fdad59389ad0c037af80f072e2b6f Mon Sep 17 00:00:00 2001 From: "adel.kalimullin" Date: Wed, 11 Dec 2024 13:45:59 +0300 Subject: [PATCH 08/15] SUPPORT-8725: refactor trowing raw exception --- .../java/ru/micord/ervu/av/service/FileUploadService.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/ru/micord/ervu/av/service/FileUploadService.java b/src/main/java/ru/micord/ervu/av/service/FileUploadService.java index 0764c90..d8df0d3 100644 --- a/src/main/java/ru/micord/ervu/av/service/FileUploadService.java +++ b/src/main/java/ru/micord/ervu/av/service/FileUploadService.java @@ -83,7 +83,7 @@ public class FileUploadService { Files.createDirectories(Paths.get(fileSavingPath)); } catch (IOException e) { - throw new RuntimeException("Failed to create directory " + fileSavingPath, e); + throw new FileUploadException("Failed to create directory " + fileSavingPath, e); } LOGGER.info("working in {}", System.getProperty("user.home")); Path filePath = Paths.get(fileSavingPath, fileUrl.fileName()); @@ -131,7 +131,7 @@ public class FileUploadService { Files.delete(filePath); } catch (IOException e) { - throw new RuntimeException("Failed to delete file " + filePath.getFileName()); + throw new FileUploadException("Failed to delete file " + filePath.getFileName()); } finally { acknowledgment.acknowledge(); From c3ccfc958768d8c87701f869b774d74fadd6ec7b Mon Sep 17 00:00:00 2001 From: "adel.kalimullin" Date: Fri, 13 Dec 2024 14:31:51 +0300 Subject: [PATCH 09/15] SUPPORT-8725:fix --- .../ru/micord/ervu/av/service/FileUploadService.java | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/src/main/java/ru/micord/ervu/av/service/FileUploadService.java b/src/main/java/ru/micord/ervu/av/service/FileUploadService.java index d8df0d3..e7afe33 100644 --- a/src/main/java/ru/micord/ervu/av/service/FileUploadService.java +++ b/src/main/java/ru/micord/ervu/av/service/FileUploadService.java @@ -78,13 +78,7 @@ public class FileUploadService { try { FileUrl fileUrl = parseFileUrl(downloadRequest.fileInfo().getFileUrl()); - - try { - Files.createDirectories(Paths.get(fileSavingPath)); - } - catch (IOException e) { - throw new FileUploadException("Failed to create directory " + fileSavingPath, e); - } + Files.createDirectories(Paths.get(fileSavingPath)); LOGGER.info("working in {}", System.getProperty("user.home")); Path filePath = Paths.get(fileSavingPath, fileUrl.fileName()); String downloadUrl = fileUrl.fileUrl(); @@ -131,7 +125,7 @@ public class FileUploadService { Files.delete(filePath); } catch (IOException e) { - throw new FileUploadException("Failed to delete file " + filePath.getFileName()); + LOGGER.error("Failed to delete file " + filePath.getFileName(),e); } finally { acknowledgment.acknowledge(); @@ -146,7 +140,7 @@ public class FileUploadService { sendMessage(inStatusTopic.name(), downloadRequest, messageId, inKafkaTemplate); acknowledgment.acknowledge(); } - catch (FileUploadException e) { + catch (FileUploadException | IOException e) { // считаем, что нужно повторное считывание сообщения // ошибку логируем, сообщение оставляем непрочитанным LOGGER.error(e.getMessage(), e); From 6bc2720244244da4c4f7e2cb9a9ac2818965a16e Mon Sep 17 00:00:00 2001 From: "adel.kalimullin" Date: Fri, 13 Dec 2024 14:42:10 +0300 Subject: [PATCH 10/15] SUPPORT-8725:fix --- src/main/java/ru/micord/ervu/av/service/FileUploadService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/ru/micord/ervu/av/service/FileUploadService.java b/src/main/java/ru/micord/ervu/av/service/FileUploadService.java index e7afe33..52afb22 100644 --- a/src/main/java/ru/micord/ervu/av/service/FileUploadService.java +++ b/src/main/java/ru/micord/ervu/av/service/FileUploadService.java @@ -125,7 +125,7 @@ public class FileUploadService { Files.delete(filePath); } catch (IOException e) { - LOGGER.error("Failed to delete file " + filePath.getFileName(),e); + LOGGER.error("Failed to delete file " + filePath, e); } finally { acknowledgment.acknowledge(); From 3ba4a408fd9613ec849aebb3b7a1aff0bcb866d9 Mon Sep 17 00:00:00 2001 From: gulnaz Date: Thu, 28 Nov 2024 12:42:58 +0300 Subject: [PATCH 11/15] SUPPORT-8662: use synchronous request-reply; fix file manager --- micord.env | 3 + .../ervu/av/response/AvFileSendResponse.java | 11 --- .../micord/ervu/av/service/FileManager.java | 32 ++++++- .../service/ReceiveScanReportRetryable.java | 91 +++---------------- src/main/resources/application.properties | 2 + 5 files changed, 48 insertions(+), 91 deletions(-) delete mode 100644 src/main/java/ru/micord/ervu/av/response/AvFileSendResponse.java diff --git a/micord.env b/micord.env index 25b4ae4..a0d39b4 100644 --- a/micord.env +++ b/micord.env @@ -30,3 +30,6 @@ S3_ACCESS_KEY=rlTdTvkmSXu9FsLhfecw S3_SECRET_KEY=NUmY0wwRIEyAd98GCKd1cOgJWvLQYAcMMul5Ulu0 S3_BUCKET_NAME=default-out-bucket S3_PATH_STYLE_ACCESS_ENABLED=true + +WEBDAV_USERNAME=test +WEBDAV_PASSWORD=test diff --git a/src/main/java/ru/micord/ervu/av/response/AvFileSendResponse.java b/src/main/java/ru/micord/ervu/av/response/AvFileSendResponse.java deleted file mode 100644 index ea8dd00..0000000 --- a/src/main/java/ru/micord/ervu/av/response/AvFileSendResponse.java +++ /dev/null @@ -1,11 +0,0 @@ -package ru.micord.ervu.av.response; - -/** - * @author r.latypov - */ -public record AvFileSendResponse(String id, String location, Error error, String status) { - public static final String STATUS_ERROR = "error"; - - public record Error(String code, String message) { - } -} diff --git a/src/main/java/ru/micord/ervu/av/service/FileManager.java b/src/main/java/ru/micord/ervu/av/service/FileManager.java index 4b52afe..044cd58 100644 --- a/src/main/java/ru/micord/ervu/av/service/FileManager.java +++ b/src/main/java/ru/micord/ervu/av/service/FileManager.java @@ -6,12 +6,17 @@ import java.io.IOException; import java.nio.file.Path; import org.apache.http.HttpEntity; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.http.client.CredentialsProvider; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpDelete; import org.apache.http.client.methods.HttpGet; import org.apache.http.conn.HttpHostConnectException; +import org.apache.http.impl.client.BasicCredentialsProvider; import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClients; +import org.apache.http.impl.client.HttpClientBuilder; +import org.springframework.beans.factory.annotation.Value; import org.springframework.http.HttpStatus; import org.springframework.retry.annotation.Backoff; import org.springframework.retry.annotation.Retryable; @@ -20,12 +25,19 @@ import ru.micord.ervu.av.exception.FileUploadException; import ru.micord.ervu.av.exception.InvalidHttpFileUrlException; import ru.micord.ervu.av.exception.RetryableException; +import static org.springframework.util.StringUtils.hasText; + /** * @author gulnaz */ @Component public class FileManager { + @Value("${webdav.username}") + private String username; + @Value("${webdav.password}") + private String password; + @Retryable(retryFor = {InvalidHttpFileUrlException.class}, maxAttemptsExpression = "${av.retry.max.attempts.count}", backoff = @Backoff(delayExpression = "${av.retry.delay.milliseconds}")) @@ -34,7 +46,7 @@ public class FileManager { File file = filePath.toFile(); HttpGet request = new HttpGet(fileUrl); - try (CloseableHttpClient client = HttpClients.createDefault(); + try (CloseableHttpClient client = getClientBuilder().build(); CloseableHttpResponse response = client.execute(request)) { int statusCode = response.getStatusLine().getStatusCode(); @@ -67,7 +79,7 @@ public class FileManager { maxAttemptsExpression = "${av.retry.max.attempts.count}", backoff = @Backoff(delayExpression = "${av.retry.delay.milliseconds}")) public void deleteFile(String fileUrl) throws FileUploadException { - try (CloseableHttpClient client = HttpClients.createDefault()) { + try (CloseableHttpClient client = getClientBuilder().build()) { HttpDelete delete = new HttpDelete(fileUrl); try (CloseableHttpResponse response = client.execute(delete)) { @@ -83,4 +95,18 @@ public class FileManager { throw new FileUploadException(e); } } + + private HttpClientBuilder getClientBuilder() { + HttpClientBuilder httpClientBuilder = HttpClientBuilder.create(); + + if (hasText(username) && hasText(password)) { + CredentialsProvider provider = new BasicCredentialsProvider(); + provider.setCredentials( + AuthScope.ANY, + new UsernamePasswordCredentials(username, password) + ); + httpClientBuilder.setDefaultCredentialsProvider(provider); + } + return httpClientBuilder; + } } diff --git a/src/main/java/ru/micord/ervu/av/service/ReceiveScanReportRetryable.java b/src/main/java/ru/micord/ervu/av/service/ReceiveScanReportRetryable.java index 1b4fbc4..679b05a 100644 --- a/src/main/java/ru/micord/ervu/av/service/ReceiveScanReportRetryable.java +++ b/src/main/java/ru/micord/ervu/av/service/ReceiveScanReportRetryable.java @@ -2,16 +2,16 @@ package ru.micord.ervu.av.service; import java.io.File; import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; import java.nio.file.Path; -import java.util.concurrent.TimeUnit; import com.google.gson.Gson; -import com.google.gson.JsonSyntaxException; import org.apache.http.HttpEntity; import org.apache.http.client.ClientProtocolException; import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.utils.URIBuilder; import org.apache.http.entity.mime.MultipartEntityBuilder; import org.apache.http.entity.mime.content.FileBody; import org.apache.http.impl.client.CloseableHttpClient; @@ -24,7 +24,6 @@ import org.springframework.retry.annotation.Retryable; import org.springframework.stereotype.Service; import ru.micord.ervu.av.exception.FileUploadException; import ru.micord.ervu.av.exception.RetryableException; -import ru.micord.ervu.av.response.AvFileSendResponse; import ru.micord.ervu.av.response.AvResponse; /** @@ -35,8 +34,6 @@ public class ReceiveScanReportRetryable { @Value("${av.rest.address}") private String avRestAddress; - @Value("${av.first.timeout.milliseconds:1000}") - private Long avFirstTimeoutMilliseconds; @Retryable(retryFor = {RetryableException.class}, maxAttemptsExpression = "${av.retry.max.attempts.count}", @@ -45,7 +42,10 @@ public class ReceiveScanReportRetryable { File file = filePath.toFile(); try (CloseableHttpClient client = HttpClients.createDefault()) { - HttpPost post = new HttpPost(avRestAddress); + URI uri = new URIBuilder(avRestAddress) + .addParameter("wait", "1") + .build(); + HttpPost post = new HttpPost(uri); HttpEntity entity = MultipartEntityBuilder.create() .addPart("file", new FileBody(file)) .build(); @@ -54,84 +54,21 @@ public class ReceiveScanReportRetryable { try (CloseableHttpResponse postResponse = client.execute(post)) { int postStatusCode = postResponse.getStatusLine().getStatusCode(); String postResponseJson = EntityUtils.toString(postResponse.getEntity()); - AvFileSendResponse avFileSendResponse; - try { - avFileSendResponse = new Gson().fromJson(postResponseJson, AvFileSendResponse.class); + if (postStatusCode == HttpStatus.OK.value()) { + return new Gson().fromJson(postResponseJson, AvResponse.class); } - catch (JsonSyntaxException e) { - throw new FileUploadException("error json: " + postResponseJson, e); + else { + throw new FileUploadException( + "http status code " + postStatusCode + " for " + post.getURI() + + " get request."); } - - if (postStatusCode != HttpStatus.CREATED.value()) { - StringBuilder stringBuilder = new StringBuilder( - "http status code " + postStatusCode + " for " + avRestAddress + " post request."); - String status = avFileSendResponse.status(); - - if (status != null) { - stringBuilder.append(" Status: ").append(status).append("."); - } - - if (avFileSendResponse.error() != null) { - stringBuilder.append(" Error code: ") - .append(avFileSendResponse.error().code()) - .append(". Error message: ") - .append(avFileSendResponse.error().message()) - .append(". "); - } - throw new FileUploadException(stringBuilder.toString()); - } - - String id = avFileSendResponse.id(); - String reportRequestUri = avRestAddress + "/" + id; - HttpGet get = new HttpGet(reportRequestUri); - - // waiting for timeout time before first request - try { - TimeUnit.MILLISECONDS.sleep(avFirstTimeoutMilliseconds); - } - catch (InterruptedException e) { - throw new FileUploadException(e); - } - - return receiveScanReport(client, get); } } catch (ClientProtocolException e) { throw new RetryableException("Failed to check file"); } - catch (IOException e) { - // непредусмотренная ошибка доступа через http-клиент - throw new FileUploadException(e); - } - } - - private AvResponse receiveScanReport(CloseableHttpClient client, HttpGet get) - throws RetryableException, FileUploadException { - - try (CloseableHttpResponse getResponse = client.execute(get)) { - int getStatusCode = getResponse.getStatusLine().getStatusCode(); - - if (getStatusCode == HttpStatus.OK.value()) { - String getResponseJson = EntityUtils.toString(getResponse.getEntity()); - AvResponse avResponse = new Gson().fromJson(getResponseJson, AvResponse.class); - - if (avResponse.completed() == null) { - throw new RetryableException("Failed to complete file scan"); - } - return avResponse; - } - else { - throw new FileUploadException("http status code " + getStatusCode + " for " + get.getURI() - + " get request."); - } - } - catch (ClientProtocolException e) { - // непредусмотренная ошибка доступа через http-клиент - throw new RetryableException("Failed to receive scan report"); - } - catch (IOException e) { - // непредусмотренная ошибка доступа через http-клиент + catch (IOException | URISyntaxException e) { throw new FileUploadException(e); } } diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index fa9859a..ddf45e4 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -63,6 +63,8 @@ s3.out.secret_key=${S3_SECRET_KEY} s3.out.bucket_name=${S3_OUT_BUCKET_NAME} s3.out.path.style.access.enabled=${S3_OUT_PATH_STYLE_ACCESS_ENABLED} # +webdav.username=${WEBDAV_USERNAME} +webdav.password=${WEBDAV_PASSWORD} # endpoints management management.endpoints.web.exposure.include = health, info, metrics From 6435fc4c3391f6973ccbd35bd7e61cb140b908a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A5=D0=B0=D0=BB=D1=82=D0=BE=D0=B1=D0=B8=D0=BD=20=D0=95?= =?UTF-8?q?=D0=B2=D0=B3=D0=B5=D0=BD=D0=B8=D0=B9?= Date: Thu, 19 Dec 2024 19:57:38 +0300 Subject: [PATCH 12/15] added flags --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 3910669..b163daa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ FROM bellsoft/liberica-openjdk-alpine:17-cds COPY target/*.jar app.jar -CMD ["java", "-jar", "app.jar"] +CMD ["java", "-Dcom.sun.management.jmxremote", "-Dfile.encoding=UTF-8", "-Dcom.amazonaws.sdk.disableCertChecking", "-jar", "app.jar"] From fb36447ecb96264b13fa45cf7f908ce36328ebf0 Mon Sep 17 00:00:00 2001 From: "adel.kalimullin" Date: Fri, 27 Dec 2024 12:53:21 +0300 Subject: [PATCH 13/15] SUPPORT-8836:refactor --- .../ru/micord/ervu/av/response/AvResponse.java | 17 ----------------- .../ervu/av/service/FileUploadService.java | 14 ++++---------- .../av/service/ReceiveScanReportRetryable.java | 9 +++------ 3 files changed, 7 insertions(+), 33 deletions(-) delete mode 100644 src/main/java/ru/micord/ervu/av/response/AvResponse.java diff --git a/src/main/java/ru/micord/ervu/av/response/AvResponse.java b/src/main/java/ru/micord/ervu/av/response/AvResponse.java deleted file mode 100644 index 48e71ae..0000000 --- a/src/main/java/ru/micord/ervu/av/response/AvResponse.java +++ /dev/null @@ -1,17 +0,0 @@ -package ru.micord.ervu.av.response; - -import java.util.Map; - -/** - * @author r.latypov - */ -public record AvResponse(String completed, String created, Integer progress, - Map scan_result, String status, String[] verdicts) { - public record Scan(String started, String stopped, Threat[] threats, String verdict) { - public static final String VERDICT_CLEAN = "clean"; - public static final String VERDICT_INFECTED = "infected"; - - public record Threat(String name, String object) { - } - } -} diff --git a/src/main/java/ru/micord/ervu/av/service/FileUploadService.java b/src/main/java/ru/micord/ervu/av/service/FileUploadService.java index 52afb22..0599a63 100644 --- a/src/main/java/ru/micord/ervu/av/service/FileUploadService.java +++ b/src/main/java/ru/micord/ervu/av/service/FileUploadService.java @@ -5,7 +5,6 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.Arrays; import java.util.concurrent.CompletableFuture; import com.google.gson.Gson; @@ -30,7 +29,6 @@ import ru.micord.ervu.av.exception.RetryableException; import ru.micord.ervu.av.kafka.dto.DownloadRequest; import ru.micord.ervu.av.kafka.dto.DownloadResponse; import ru.micord.ervu.av.kafka.dto.FileStatus; -import ru.micord.ervu.av.response.AvResponse; import ru.micord.ervu.av.s3.S3Service; /** @@ -38,6 +36,7 @@ import ru.micord.ervu.av.s3.S3Service; */ @Service public class FileUploadService { + private static final int INJECTED_CODE = 72; private static final Logger LOGGER = LoggerFactory.getLogger(FileUploadService.class); @Value("${av.check.enabled}") private boolean avCheckEnabled; @@ -84,16 +83,11 @@ public class FileUploadService { String downloadUrl = fileUrl.fileUrl(); fIleManager.downloadFile(downloadUrl, filePath); boolean isAvError = false; - boolean clean = true; - boolean infected = false; + int exitCode = 0; if (avCheckEnabled) { try { - AvResponse avResponse = receiveScanReportRetryable.checkFile(filePath); - clean = Arrays.stream(avResponse.verdicts()) - .anyMatch(verdict -> verdict.equalsIgnoreCase(AvResponse.Scan.VERDICT_CLEAN)); - infected = Arrays.stream(avResponse.verdicts()) - .anyMatch(verdict -> verdict.equalsIgnoreCase(AvResponse.Scan.VERDICT_INFECTED)); + exitCode = receiveScanReportRetryable.checkFile(filePath); } catch (FileUploadException | RetryableException e) { LOGGER.error(e.getMessage(), e); @@ -101,7 +95,7 @@ public class FileUploadService { } } - if (isAvError || infected || !clean) { + if (isAvError || exitCode == INJECTED_CODE) { downloadRequest.fileInfo().setFileUrl(null); FileStatus fileStatus = isAvError ? FileStatus.FILE_STATUS_11 : FileStatus.FILE_STATUS_02; downloadRequest.fileInfo().setFileStatus(fileStatus); diff --git a/src/main/java/ru/micord/ervu/av/service/ReceiveScanReportRetryable.java b/src/main/java/ru/micord/ervu/av/service/ReceiveScanReportRetryable.java index 679b05a..c74f03e 100644 --- a/src/main/java/ru/micord/ervu/av/service/ReceiveScanReportRetryable.java +++ b/src/main/java/ru/micord/ervu/av/service/ReceiveScanReportRetryable.java @@ -6,7 +6,6 @@ import java.net.URI; import java.net.URISyntaxException; import java.nio.file.Path; -import com.google.gson.Gson; import org.apache.http.HttpEntity; import org.apache.http.client.ClientProtocolException; import org.apache.http.client.methods.CloseableHttpResponse; @@ -24,7 +23,6 @@ import org.springframework.retry.annotation.Retryable; import org.springframework.stereotype.Service; import ru.micord.ervu.av.exception.FileUploadException; import ru.micord.ervu.av.exception.RetryableException; -import ru.micord.ervu.av.response.AvResponse; /** * @author r.latypov @@ -38,12 +36,11 @@ public class ReceiveScanReportRetryable { @Retryable(retryFor = {RetryableException.class}, maxAttemptsExpression = "${av.retry.max.attempts.count}", backoff = @Backoff(delayExpression = "${av.retry.delay.milliseconds}")) - public AvResponse checkFile(Path filePath) throws RetryableException, FileUploadException { + public int checkFile(Path filePath) throws RetryableException, FileUploadException { File file = filePath.toFile(); try (CloseableHttpClient client = HttpClients.createDefault()) { URI uri = new URIBuilder(avRestAddress) - .addParameter("wait", "1") .build(); HttpPost post = new HttpPost(uri); HttpEntity entity = MultipartEntityBuilder.create() @@ -53,10 +50,10 @@ public class ReceiveScanReportRetryable { try (CloseableHttpResponse postResponse = client.execute(post)) { int postStatusCode = postResponse.getStatusLine().getStatusCode(); - String postResponseJson = EntityUtils.toString(postResponse.getEntity()); + String exitCode = EntityUtils.toString(postResponse.getEntity()); if (postStatusCode == HttpStatus.OK.value()) { - return new Gson().fromJson(postResponseJson, AvResponse.class); + return Integer.parseInt(exitCode); } else { throw new FileUploadException( From 162c6051754e34e36c665d912d1c306017b29b11 Mon Sep 17 00:00:00 2001 From: "adel.kalimullin" Date: Fri, 27 Dec 2024 13:27:07 +0300 Subject: [PATCH 14/15] SUPPORT-8836:fixes --- .../java/ru/micord/ervu/av/service/FileUploadService.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/ru/micord/ervu/av/service/FileUploadService.java b/src/main/java/ru/micord/ervu/av/service/FileUploadService.java index 0599a63..9db19bf 100644 --- a/src/main/java/ru/micord/ervu/av/service/FileUploadService.java +++ b/src/main/java/ru/micord/ervu/av/service/FileUploadService.java @@ -36,7 +36,7 @@ import ru.micord.ervu.av.s3.S3Service; */ @Service public class FileUploadService { - private static final int INJECTED_CODE = 72; + private static final int INFECTED_CODE = 72; private static final Logger LOGGER = LoggerFactory.getLogger(FileUploadService.class); @Value("${av.check.enabled}") private boolean avCheckEnabled; @@ -95,7 +95,7 @@ public class FileUploadService { } } - if (isAvError || exitCode == INJECTED_CODE) { + if (isAvError || exitCode == INFECTED_CODE) { downloadRequest.fileInfo().setFileUrl(null); FileStatus fileStatus = isAvError ? FileStatus.FILE_STATUS_11 : FileStatus.FILE_STATUS_02; downloadRequest.fileInfo().setFileStatus(fileStatus); From 6cdc3e0d5ddb92e41ea824e32109f13eae3fa7e2 Mon Sep 17 00:00:00 2001 From: kochetkov Date: Wed, 16 Apr 2025 15:54:27 +0300 Subject: [PATCH 15/15] SUPPORT-9119 update spring boot parent --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index c58ab24..47a74a8 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ org.springframework.boot spring-boot-starter-parent - 3.3.5 + 3.4.4