From 46f7eebee75c85b17017c89f9d2c03681c962e0e Mon Sep 17 00:00:00 2001 From: gulnaz Date: Thu, 26 Sep 2024 10:36:57 +0300 Subject: [PATCH 01/12] SUPPORT-8566: add and use path style access property --- micord.env | 1 + src/main/java/ru/micord/ervu/av/s3/S3Connection.java | 7 +++++-- src/main/resources/application.properties | 1 + 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/micord.env b/micord.env index 7efcce3..304585b 100644 --- a/micord.env +++ b/micord.env @@ -27,6 +27,7 @@ S3_PORT=31900 S3_ACCESS_KEY=rlTdTvkmSXu9FsLhfecw S3_SECRET_KEY=NUmY0wwRIEyAd98GCKd1cOgJWvLQYAcMMul5Ulu0 S3_OUT_BUCKET_NAME=default-out-bucket +S3_OUT_PATH_STYLE_ACCESS_ENABLED=true SPRING_DATASOURCE_URL=jdbc:postgresql://10.10.31.119:5432/ervu-lkrp-ul SPRING_DATASOURCE_USERNAME=ervu-lkrp-ul diff --git a/src/main/java/ru/micord/ervu/av/s3/S3Connection.java b/src/main/java/ru/micord/ervu/av/s3/S3Connection.java index fd6813e..ad53872 100644 --- a/src/main/java/ru/micord/ervu/av/s3/S3Connection.java +++ b/src/main/java/ru/micord/ervu/av/s3/S3Connection.java @@ -27,6 +27,8 @@ public class S3Connection { private String secretKeyOut; @Value("${s3.out.bucket_name}") private String bucketNameOut; + @Value("${s3.out.path.style.access.enabled}") + private boolean pathStyleAccessEnabled; @Bean("outBucketName") public String getBucketNameOut() { @@ -35,11 +37,11 @@ public class S3Connection { @Bean("outClient") public AmazonS3 getS3OutClient() { - return getS3Client(endpointOut, portOut, accessKeyOut, secretKeyOut); + return getS3Client(endpointOut, portOut, accessKeyOut, secretKeyOut, pathStyleAccessEnabled); } private static AmazonS3 getS3Client(String endpoint, int port, String accessKey, - String secretKey) { + String secretKey, boolean pathStyleAccessEnabled) { AWSCredentials credentials = new BasicAWSCredentials(accessKey, secretKey); String s3Endpoint = endpoint + ":" + port; String region = Region.getRegion(Regions.DEFAULT_REGION).toString(); @@ -47,6 +49,7 @@ public class S3Connection { return AmazonS3ClientBuilder.standard() .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(s3Endpoint, region)) .withCredentials(new AWSStaticCredentialsProvider(credentials)) + .withPathStyleAccessEnabled(pathStyleAccessEnabled) .build(); } } diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 05ee0fb..f0e4dd0 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -53,6 +53,7 @@ s3.out.port=${S3_PORT} s3.out.access_key=${S3_ACCESS_KEY} s3.out.secret_key=${S3_SECRET_KEY} s3.out.bucket_name=${S3_OUT_BUCKET_NAME} +s3.out.path.style.access.enabled=${S3_OUT_PATH_STYLE_ACCESS_ENABLED} # # spring jooq dsl bean properties begin -> spring.jooq.sql-dialect=Postgres From 36eebe7a8067f83bf113dd79c6a71afaf373f1d5 Mon Sep 17 00:00:00 2001 From: gulnaz Date: Thu, 26 Sep 2024 10:42:11 +0300 Subject: [PATCH 02/12] fix after merge --- src/main/java/ru/micord/ervu/av/s3/S3Connection.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/main/java/ru/micord/ervu/av/s3/S3Connection.java b/src/main/java/ru/micord/ervu/av/s3/S3Connection.java index 44a7231..f36a509 100644 --- a/src/main/java/ru/micord/ervu/av/s3/S3Connection.java +++ b/src/main/java/ru/micord/ervu/av/s3/S3Connection.java @@ -35,13 +35,10 @@ public class S3Connection { @Bean("outClient") public AmazonS3 getS3OutClient() { - return getS3Client(endpointOut, accessKeyOut, secretKeyOut); - return getS3Client(endpointOut, portOut, accessKeyOut, secretKeyOut, pathStyleAccessEnabled); + return getS3Client(endpointOut, accessKeyOut, secretKeyOut, pathStyleAccessEnabled); } private static AmazonS3 getS3Client(String endpoint, String accessKey, - String secretKey) { - private static AmazonS3 getS3Client(String endpoint, int port, String accessKey, String secretKey, boolean pathStyleAccessEnabled) { AWSCredentials credentials = new BasicAWSCredentials(accessKey, secretKey); String region = Region.getRegion(Regions.DEFAULT_REGION).toString(); From 0d8b8b2b3d0578babc17e16c94728f7a4e48a8a3 Mon Sep 17 00:00:00 2001 From: gulnaz Date: Thu, 26 Sep 2024 11:32:31 +0300 Subject: [PATCH 03/12] add group id var --- micord.env | 1 + src/main/java/ru/micord/ervu/av/s3/S3Connection.java | 2 +- src/main/resources/application.properties | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/micord.env b/micord.env index efd6a58..8f34a0a 100644 --- a/micord.env +++ b/micord.env @@ -3,6 +3,7 @@ AV_KAFKA_SECURITY_PROTOCOL=SASL_PLAINTEXT AV_KAFKA_SASL_MECHANISM=SCRAM-SHA-256 AV_KAFKA_USERNAME=user1 AV_KAFKA_PASSWORD=Blfi9d2OFG +AV_KAFKA_GROUP_ID=file-to-upload-consumers AV_KAFKA_TOPIC_NAME=file-to-upload ERVU_KAFKA_BOOTSTRAP_SERVERS=10.10.31.11:32609 diff --git a/src/main/java/ru/micord/ervu/av/s3/S3Connection.java b/src/main/java/ru/micord/ervu/av/s3/S3Connection.java index f36a509..6379384 100644 --- a/src/main/java/ru/micord/ervu/av/s3/S3Connection.java +++ b/src/main/java/ru/micord/ervu/av/s3/S3Connection.java @@ -25,7 +25,7 @@ public class S3Connection { private String secretKeyOut; @Value("${s3.out.bucket_name}") private String bucketNameOut; - @Value("${s3.out.path.style.access.enabled}") + @Value("${s3.out.path.style.access.enabled:true}") private boolean pathStyleAccessEnabled; @Bean("outBucketName") diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 6b7726e..135ce40 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -10,7 +10,7 @@ spring.kafka.consumer.properties.sasl.jaas.config=org.apache.kafka.common.securi spring.kafka.consumer.properties.sasl.mechanism=${AV_KAFKA_SASL_MECHANISM:SCRAM-SHA-256} # spring.kafka.consumer.enable.auto.commit=false -spring.kafka.consumer.group.id=file-to-upload-consumers +spring.kafka.consumer.group.id=${AV_KAFKA_GROUP_ID:file-to-upload-consumers} # kafka in listeners spring.kafka.listener.ack.mode=MANUAL_IMMEDIATE # From 10abd45fbbe08cefe5dd2b2b16d35908179e7c46 Mon Sep 17 00:00:00 2001 From: gulnaz Date: Thu, 26 Sep 2024 15:05:48 +0300 Subject: [PATCH 04/12] SUPPORT-8566: add new api for S3 --- pom.xml | 10 ++-- .../java/ru/micord/ervu/av/s3/S3Config.java | 51 ++++++++++++++++++ .../ru/micord/ervu/av/s3/S3Connection.java | 52 ------------------- .../java/ru/micord/ervu/av/s3/S3Service.java | 51 ++++++++++-------- 4 files changed, 84 insertions(+), 80 deletions(-) create mode 100644 src/main/java/ru/micord/ervu/av/s3/S3Config.java delete mode 100644 src/main/java/ru/micord/ervu/av/s3/S3Connection.java diff --git a/pom.xml b/pom.xml index aab278e..0073719 100644 --- a/pom.xml +++ b/pom.xml @@ -18,9 +18,9 @@ - com.amazonaws - aws-java-sdk-bom - 1.12.759 + software.amazon.awssdk + bom + 2.28.7 pom import @@ -59,8 +59,8 @@ - com.amazonaws - aws-java-sdk-s3 + software.amazon.awssdk + s3 diff --git a/src/main/java/ru/micord/ervu/av/s3/S3Config.java b/src/main/java/ru/micord/ervu/av/s3/S3Config.java new file mode 100644 index 0000000..4332aed --- /dev/null +++ b/src/main/java/ru/micord/ervu/av/s3/S3Config.java @@ -0,0 +1,51 @@ +package ru.micord.ervu.av.s3; + +import java.net.URI; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3Client; + +/** + * @author r.latypov + */ +@Configuration +public class S3Config { + @Value("${s3.out.endpoint}") + private String endpoint; + @Value("${s3.region:auto}") + private String regionStr; + @Value("${s3.out.access_key}") + private String accessKey; + @Value("${s3.out.secret_key}") + private String secretKey; + @Value("${s3.out.bucket_name}") + private String bucketName; + @Value("${s3.out.path.style.enabled:true}") + private boolean pathStyleAccessEnabled; + + @Bean("outBucketName") + public String getBucket() { + return bucketName; + } + + @Bean("outClient") + public S3Client getS3OutClient() { + Region region = Region.of(regionStr); + AwsBasicCredentials credentials = AwsBasicCredentials.builder() + .accessKeyId(accessKey) + .secretAccessKey(secretKey) + .build(); + + return S3Client.builder() + .region(region) + .credentialsProvider(StaticCredentialsProvider.create(credentials)) + .endpointOverride(URI.create(endpoint)) + .forcePathStyle(pathStyleAccessEnabled) + .build(); + } +} diff --git a/src/main/java/ru/micord/ervu/av/s3/S3Connection.java b/src/main/java/ru/micord/ervu/av/s3/S3Connection.java deleted file mode 100644 index 6379384..0000000 --- a/src/main/java/ru/micord/ervu/av/s3/S3Connection.java +++ /dev/null @@ -1,52 +0,0 @@ -package ru.micord.ervu.av.s3; - -import com.amazonaws.auth.AWSCredentials; -import com.amazonaws.auth.AWSStaticCredentialsProvider; -import com.amazonaws.auth.BasicAWSCredentials; -import com.amazonaws.client.builder.AwsClientBuilder; -import com.amazonaws.regions.Region; -import com.amazonaws.regions.Regions; -import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.AmazonS3ClientBuilder; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -/** - * @author r.latypov - */ -@Configuration -public class S3Connection { - @Value("${s3.out.endpoint}") - private String endpointOut; - @Value("${s3.out.access_key}") - private String accessKeyOut; - @Value("${s3.out.secret_key}") - private String secretKeyOut; - @Value("${s3.out.bucket_name}") - private String bucketNameOut; - @Value("${s3.out.path.style.access.enabled:true}") - private boolean pathStyleAccessEnabled; - - @Bean("outBucketName") - public String getBucketNameOut() { - return bucketNameOut; - } - - @Bean("outClient") - public AmazonS3 getS3OutClient() { - return getS3Client(endpointOut, accessKeyOut, secretKeyOut, pathStyleAccessEnabled); - } - - private static AmazonS3 getS3Client(String endpoint, String accessKey, - String secretKey, boolean pathStyleAccessEnabled) { - AWSCredentials credentials = new BasicAWSCredentials(accessKey, secretKey); - String region = Region.getRegion(Regions.DEFAULT_REGION).toString(); - - return AmazonS3ClientBuilder.standard() - .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(endpoint, region)) - .withCredentials(new AWSStaticCredentialsProvider(credentials)) - .withPathStyleAccessEnabled(pathStyleAccessEnabled) - .build(); - } -} diff --git a/src/main/java/ru/micord/ervu/av/s3/S3Service.java b/src/main/java/ru/micord/ervu/av/s3/S3Service.java index 9178d2c..dee9a07 100644 --- a/src/main/java/ru/micord/ervu/av/s3/S3Service.java +++ b/src/main/java/ru/micord/ervu/av/s3/S3Service.java @@ -1,48 +1,53 @@ package ru.micord.ervu.av.s3; -import java.io.File; +import java.nio.file.Paths; -import com.amazonaws.AmazonServiceException; -import com.amazonaws.services.s3.AmazonS3; import jakarta.annotation.PostConstruct; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import ru.micord.ervu.av.exception.FileUploadException; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.CreateBucketRequest; +import software.amazon.awssdk.services.s3.model.HeadBucketRequest; +import software.amazon.awssdk.services.s3.model.NoSuchBucketException; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; /** * @author r.latypov */ @Service public class S3Service { - private final String outBucketName; - private final AmazonS3 outClient; + private final String bucketName; + private final S3Client s3Client; @Autowired - public S3Service(String outBucketName, AmazonS3 outClient) { - this.outBucketName = outBucketName; - this.outClient = outClient; + public S3Service(String bucketName, S3Client s3Client) { + this.bucketName = bucketName; + this.s3Client = s3Client; } @PostConstruct private void init() { - if (!outClient.doesBucketExistV2(outBucketName)) { - outClient.createBucket(outBucketName); + HeadBucketRequest headBucketRequest = HeadBucketRequest.builder() + .bucket(bucketName) + .build(); + + try { + s3Client.headBucket(headBucketRequest); + } + catch (NoSuchBucketException e) { + CreateBucketRequest bucketRequest = CreateBucketRequest.builder() + .bucket(bucketName) + .build(); + s3Client.createBucket(bucketRequest); } } public void putFile(String filePath, String key) throws FileUploadException { - try { - outClient.putObject(outBucketName, generateResourceName(outBucketName, key), - new File(filePath) - ); - } - catch (AmazonServiceException e) { - // todo message - throw new FileUploadException(e); - } - } - - private static String generateResourceName(String bucketName, String key) { - return String.join("/", bucketName, key); + PutObjectRequest objectRequest = PutObjectRequest.builder() + .bucket(bucketName) + .key(key) + .build(); + s3Client.putObject(objectRequest, Paths.get(filePath)); } } From a08441d8b0186e2e381e2069bf7945d351ca4d02 Mon Sep 17 00:00:00 2001 From: gulnaz Date: Thu, 26 Sep 2024 15:31:39 +0300 Subject: [PATCH 05/12] SUPPORT-8566: fix PutObjectRequest key value --- src/main/java/ru/micord/ervu/av/s3/S3Service.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/java/ru/micord/ervu/av/s3/S3Service.java b/src/main/java/ru/micord/ervu/av/s3/S3Service.java index dee9a07..1cd1425 100644 --- a/src/main/java/ru/micord/ervu/av/s3/S3Service.java +++ b/src/main/java/ru/micord/ervu/av/s3/S3Service.java @@ -46,8 +46,12 @@ public class S3Service { public void putFile(String filePath, String key) throws FileUploadException { PutObjectRequest objectRequest = PutObjectRequest.builder() .bucket(bucketName) - .key(key) + .key(generateResourceName(bucketName, key)) .build(); s3Client.putObject(objectRequest, Paths.get(filePath)); } + + private static String generateResourceName(String bucketName, String key) { + return String.join("/", bucketName, key); + } } From ebc3bd4da85d917af5f4445fb8f30bb812834df8 Mon Sep 17 00:00:00 2001 From: gulnaz Date: Thu, 26 Sep 2024 15:49:25 +0300 Subject: [PATCH 06/12] SUPPORT-8566: pass Path in method --- src/main/java/ru/micord/ervu/av/s3/S3Service.java | 6 +++--- .../java/ru/micord/ervu/av/service/FileUploadService.java | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/main/java/ru/micord/ervu/av/s3/S3Service.java b/src/main/java/ru/micord/ervu/av/s3/S3Service.java index 1cd1425..09531ad 100644 --- a/src/main/java/ru/micord/ervu/av/s3/S3Service.java +++ b/src/main/java/ru/micord/ervu/av/s3/S3Service.java @@ -1,6 +1,6 @@ package ru.micord.ervu.av.s3; -import java.nio.file.Paths; +import java.nio.file.Path; import jakarta.annotation.PostConstruct; import org.springframework.beans.factory.annotation.Autowired; @@ -43,12 +43,12 @@ public class S3Service { } } - public void putFile(String filePath, String key) throws FileUploadException { + public void putFile(Path filePath, String key) throws FileUploadException { PutObjectRequest objectRequest = PutObjectRequest.builder() .bucket(bucketName) .key(generateResourceName(bucketName, key)) .build(); - s3Client.putObject(objectRequest, Paths.get(filePath)); + s3Client.putObject(objectRequest, filePath); } private static String generateResourceName(String bucketName, String key) { diff --git a/src/main/java/ru/micord/ervu/av/service/FileUploadService.java b/src/main/java/ru/micord/ervu/av/service/FileUploadService.java index 380cca8..7d81ea7 100644 --- a/src/main/java/ru/micord/ervu/av/service/FileUploadService.java +++ b/src/main/java/ru/micord/ervu/av/service/FileUploadService.java @@ -3,6 +3,7 @@ package ru.micord.ervu.av.service; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; +import java.nio.file.Paths; import java.util.Arrays; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; @@ -100,7 +101,7 @@ public class FileUploadService { fileStatusService.setStatus(fileId, FileStatus.FILE_STATUS_02.status()); } else { - s3Service.putFile(filePath, fileUrl.fileName()); + s3Service.putFile(Paths.get(filePath), fileUrl.fileName()); downloadRequest.fileInfo().setFileUrl(fileUrl.fileName()); downloadRequest.fileInfo().setFileStatus(FileStatus.FILE_STATUS_03); From 3d918f6cd7c6e63ebc96a69dd3843522a1b073c4 Mon Sep 17 00:00:00 2001 From: gulnaz Date: Thu, 26 Sep 2024 15:57:38 +0300 Subject: [PATCH 07/12] SUPPORT-8566: fix filePath type in service --- .../micord/ervu/av/service/FileUploadService.java | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/main/java/ru/micord/ervu/av/service/FileUploadService.java b/src/main/java/ru/micord/ervu/av/service/FileUploadService.java index 7d81ea7..b292513 100644 --- a/src/main/java/ru/micord/ervu/av/service/FileUploadService.java +++ b/src/main/java/ru/micord/ervu/av/service/FileUploadService.java @@ -3,6 +3,7 @@ package ru.micord.ervu.av.service; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; +import java.nio.file.Path; import java.nio.file.Paths; import java.util.Arrays; import java.util.concurrent.CompletableFuture; @@ -82,7 +83,7 @@ public class FileUploadService { try { FileUrl fileUrl = parseFileUrl(downloadRequest.fileInfo().getFileUrl()); - String filePath = fileSavingPath + fileUrl.fileName(); + Path filePath = Paths.get(fileSavingPath, fileUrl.fileName()); String downloadUrl = fileUrl.fileUrl(); downloadFile(downloadUrl, filePath); @@ -101,7 +102,7 @@ public class FileUploadService { fileStatusService.setStatus(fileId, FileStatus.FILE_STATUS_02.status()); } else { - s3Service.putFile(Paths.get(filePath), fileUrl.fileName()); + s3Service.putFile(filePath, fileUrl.fileName()); downloadRequest.fileInfo().setFileUrl(fileUrl.fileName()); downloadRequest.fileInfo().setFileStatus(FileStatus.FILE_STATUS_03); @@ -111,7 +112,7 @@ public class FileUploadService { } deleteFile(downloadUrl); - if (new File(filePath).delete()) { + if (filePath.toFile().delete()) { acknowledgment.acknowledge(); } } @@ -161,9 +162,9 @@ public class FileUploadService { } } - private void downloadFile(String fileUrl, String filePath) + private void downloadFile(String fileUrl, Path filePath) throws InvalidHttpFileUrlException, FileUploadException { - File file = new File(filePath); + File file = filePath.toFile(); HttpGet request = new HttpGet(fileUrl); try (CloseableHttpClient client = HttpClients.createDefault(); @@ -195,8 +196,8 @@ public class FileUploadService { } } - private AvResponse checkFile(String filePath) throws FileUploadException { - File file = new File(filePath); + private AvResponse checkFile(Path filePath) throws FileUploadException { + File file = filePath.toFile(); try (CloseableHttpClient client = HttpClients.createDefault()) { HttpPost post = new HttpPost(avRestAddress); From e7f8aaed29d48135456140f5c008d7073a531255 Mon Sep 17 00:00:00 2001 From: gulnaz Date: Fri, 27 Sep 2024 11:49:28 +0300 Subject: [PATCH 08/12] SUPPORT-8566: fix file removing --- .../ru/micord/ervu/av/service/FileUploadService.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/main/java/ru/micord/ervu/av/service/FileUploadService.java b/src/main/java/ru/micord/ervu/av/service/FileUploadService.java index b292513..01d7e15 100644 --- a/src/main/java/ru/micord/ervu/av/service/FileUploadService.java +++ b/src/main/java/ru/micord/ervu/av/service/FileUploadService.java @@ -3,6 +3,7 @@ package ru.micord.ervu.av.service; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; +import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.Arrays; @@ -112,7 +113,14 @@ public class FileUploadService { } deleteFile(downloadUrl); - if (filePath.toFile().delete()) { + + try { + Files.delete(filePath); + } + catch (IOException e) { + throw new RuntimeException("Failed to delete file " + filePath.getFileName()); + } + finally { acknowledgment.acknowledge(); } } From 43e0cafccafe7921db9d4fee0ba3a16e0046abf3 Mon Sep 17 00:00:00 2001 From: gulnaz Date: Fri, 27 Sep 2024 16:35:12 +0300 Subject: [PATCH 09/12] add av.check.enabled property --- micord.env | 1 + .../ervu/av/service/FileUploadService.java | 16 +++++++++++----- src/main/resources/application.properties | 1 + 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/micord.env b/micord.env index 8f34a0a..130f803 100644 --- a/micord.env +++ b/micord.env @@ -16,6 +16,7 @@ ERVU_KAFKA_ERROR_TOPIC_NAME=ervu.lkrp.download.request ERVU_KAFKA_SUCCESS_TOPIC_NAME=ervu.lkrp.download.request ERVU_KAFKA_RESPONSE_TOPIC_NAME=ervu.lkrp.download.response +AV_CHECK_ENABLED=true AV_REST_ADDRESS=http://10.10.31.118:8085/scans AV_FIRST_TIMEOUT_MILLISECONDS=1000 AV_RETRY_MAX_ATTEMPTS_COUNT=10 diff --git a/src/main/java/ru/micord/ervu/av/service/FileUploadService.java b/src/main/java/ru/micord/ervu/av/service/FileUploadService.java index 01d7e15..3f90e03 100644 --- a/src/main/java/ru/micord/ervu/av/service/FileUploadService.java +++ b/src/main/java/ru/micord/ervu/av/service/FileUploadService.java @@ -51,6 +51,8 @@ import ru.micord.ervu.av.s3.S3Service; @Service public class FileUploadService { private static final Logger logger = LoggerFactory.getLogger(FileUploadService.class); + @Value("${av.check.enabled}") + private boolean avCheckEnabled; @Value("${av.rest.address}") private String avRestAddress; @Value("${av.first.timeout.milliseconds}") @@ -87,13 +89,17 @@ public class FileUploadService { Path filePath = Paths.get(fileSavingPath, fileUrl.fileName()); String downloadUrl = fileUrl.fileUrl(); downloadFile(downloadUrl, filePath); + boolean clean = true; + boolean infected = false; - AvResponse avResponse = checkFile(filePath); + if (avCheckEnabled) { + AvResponse avResponse = checkFile(filePath); - boolean clean = Arrays.stream(avResponse.verdicts()) - .anyMatch(verdict -> verdict.equalsIgnoreCase(AvResponse.Scan.VERDICT_CLEAN)); - boolean infected = Arrays.stream(avResponse.verdicts()) - .anyMatch(verdict -> verdict.equalsIgnoreCase(AvResponse.Scan.VERDICT_INFECTED)); + clean = Arrays.stream(avResponse.verdicts()) + .anyMatch(verdict -> verdict.equalsIgnoreCase(AvResponse.Scan.VERDICT_CLEAN)); + infected = Arrays.stream(avResponse.verdicts()) + .anyMatch(verdict -> verdict.equalsIgnoreCase(AvResponse.Scan.VERDICT_INFECTED)); + } if (infected || !clean) { downloadRequest.fileInfo().setFileUrl(null); diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 135ce40..1a92a88 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -42,6 +42,7 @@ kafka.out.error.topic.name=${ERVU_KAFKA_ERROR_TOPIC_NAME} kafka.out.success.topic.name=${ERVU_KAFKA_SUCCESS_TOPIC_NAME} kafka.out.response.topic.name=${ERVU_KAFKA_RESPONSE_TOPIC_NAME} # +av.check.enabled=${AV_CHECK_ENABLED:false} av.rest.address=${AV_REST_ADDRESS} av.first.timeout.milliseconds=${AV_FIRST_TIMEOUT_MILLISECONDS:1000} av.retry.max.attempts.count=${AV_RETRY_MAX_ATTEMPTS_COUNT:10} From a66a4645a5c8479bdb955c3ee8a0440e80b94cb0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A5=D0=B0=D0=BB=D1=82=D0=BE=D0=B1=D0=B8=D0=BD=20=D0=95?= =?UTF-8?q?=D0=B2=D0=B3=D0=B5=D0=BD=D0=B8=D0=B9?= Date: Fri, 27 Sep 2024 16:40:15 +0300 Subject: [PATCH 10/12] update default value for av.check.enabled --- src/main/resources/application.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 1a92a88..077bfae 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -42,7 +42,7 @@ kafka.out.error.topic.name=${ERVU_KAFKA_ERROR_TOPIC_NAME} kafka.out.success.topic.name=${ERVU_KAFKA_SUCCESS_TOPIC_NAME} kafka.out.response.topic.name=${ERVU_KAFKA_RESPONSE_TOPIC_NAME} # -av.check.enabled=${AV_CHECK_ENABLED:false} +av.check.enabled=${AV_CHECK_ENABLED:true} av.rest.address=${AV_REST_ADDRESS} av.first.timeout.milliseconds=${AV_FIRST_TIMEOUT_MILLISECONDS:1000} av.retry.max.attempts.count=${AV_RETRY_MAX_ATTEMPTS_COUNT:10} From e896bae2d69cd457a240a9b9df054b5f3da2ee39 Mon Sep 17 00:00:00 2001 From: gulnaz Date: Fri, 27 Sep 2024 17:50:20 +0300 Subject: [PATCH 11/12] revert to old S3 api --- pom.xml | 10 ++--- .../java/ru/micord/ervu/av/s3/S3Config.java | 39 ++++++++-------- .../java/ru/micord/ervu/av/s3/S3Service.java | 44 +++++++------------ 3 files changed, 42 insertions(+), 51 deletions(-) diff --git a/pom.xml b/pom.xml index 0073719..aab278e 100644 --- a/pom.xml +++ b/pom.xml @@ -18,9 +18,9 @@ - software.amazon.awssdk - bom - 2.28.7 + com.amazonaws + aws-java-sdk-bom + 1.12.759 pom import @@ -59,8 +59,8 @@ - software.amazon.awssdk - s3 + com.amazonaws + aws-java-sdk-s3 diff --git a/src/main/java/ru/micord/ervu/av/s3/S3Config.java b/src/main/java/ru/micord/ervu/av/s3/S3Config.java index 4332aed..02791d8 100644 --- a/src/main/java/ru/micord/ervu/av/s3/S3Config.java +++ b/src/main/java/ru/micord/ervu/av/s3/S3Config.java @@ -1,14 +1,16 @@ package ru.micord.ervu.av.s3; -import java.net.URI; - +import com.amazonaws.auth.AWSCredentials; +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.client.builder.AwsClientBuilder; +import com.amazonaws.regions.Region; +import com.amazonaws.regions.Regions; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; -import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; -import software.amazon.awssdk.regions.Region; -import software.amazon.awssdk.services.s3.S3Client; /** * @author r.latypov @@ -17,8 +19,6 @@ import software.amazon.awssdk.services.s3.S3Client; public class S3Config { @Value("${s3.out.endpoint}") private String endpoint; - @Value("${s3.region:auto}") - private String regionStr; @Value("${s3.out.access_key}") private String accessKey; @Value("${s3.out.secret_key}") @@ -34,18 +34,19 @@ public class S3Config { } @Bean("outClient") - public S3Client getS3OutClient() { - Region region = Region.of(regionStr); - AwsBasicCredentials credentials = AwsBasicCredentials.builder() - .accessKeyId(accessKey) - .secretAccessKey(secretKey) - .build(); + public AmazonS3 getS3OutClient() { + return getS3Client(endpoint, accessKey, secretKey, pathStyleAccessEnabled); + } - return S3Client.builder() - .region(region) - .credentialsProvider(StaticCredentialsProvider.create(credentials)) - .endpointOverride(URI.create(endpoint)) - .forcePathStyle(pathStyleAccessEnabled) + private static AmazonS3 getS3Client(String endpoint, String accessKey, + String secretKey, boolean pathStyleAccessEnabled) { + AWSCredentials credentials = new BasicAWSCredentials(accessKey, secretKey); + String region = Region.getRegion(Regions.DEFAULT_REGION).toString(); + + return AmazonS3ClientBuilder.standard() + .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(endpoint, region)) + .withCredentials(new AWSStaticCredentialsProvider(credentials)) + .withPathStyleAccessEnabled(pathStyleAccessEnabled) .build(); } } diff --git a/src/main/java/ru/micord/ervu/av/s3/S3Service.java b/src/main/java/ru/micord/ervu/av/s3/S3Service.java index 09531ad..a1f537a 100644 --- a/src/main/java/ru/micord/ervu/av/s3/S3Service.java +++ b/src/main/java/ru/micord/ervu/av/s3/S3Service.java @@ -2,53 +2,43 @@ package ru.micord.ervu.av.s3; import java.nio.file.Path; +import com.amazonaws.AmazonServiceException; +import com.amazonaws.services.s3.AmazonS3; import jakarta.annotation.PostConstruct; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import ru.micord.ervu.av.exception.FileUploadException; -import software.amazon.awssdk.services.s3.S3Client; -import software.amazon.awssdk.services.s3.model.CreateBucketRequest; -import software.amazon.awssdk.services.s3.model.HeadBucketRequest; -import software.amazon.awssdk.services.s3.model.NoSuchBucketException; -import software.amazon.awssdk.services.s3.model.PutObjectRequest; /** * @author r.latypov */ @Service public class S3Service { - private final String bucketName; - private final S3Client s3Client; + private final String outBucketName; + private final AmazonS3 outClient; @Autowired - public S3Service(String bucketName, S3Client s3Client) { - this.bucketName = bucketName; - this.s3Client = s3Client; + public S3Service(String outBucketName, AmazonS3 outClient) { + this.outBucketName = outBucketName; + this.outClient = outClient; } @PostConstruct private void init() { - HeadBucketRequest headBucketRequest = HeadBucketRequest.builder() - .bucket(bucketName) - .build(); - - try { - s3Client.headBucket(headBucketRequest); - } - catch (NoSuchBucketException e) { - CreateBucketRequest bucketRequest = CreateBucketRequest.builder() - .bucket(bucketName) - .build(); - s3Client.createBucket(bucketRequest); + if (!outClient.doesBucketExistV2(outBucketName)) { + outClient.createBucket(outBucketName); } } public void putFile(Path filePath, String key) throws FileUploadException { - PutObjectRequest objectRequest = PutObjectRequest.builder() - .bucket(bucketName) - .key(generateResourceName(bucketName, key)) - .build(); - s3Client.putObject(objectRequest, filePath); + try { + outClient.putObject(outBucketName, generateResourceName(outBucketName, key), + filePath.toFile()); + } + catch (AmazonServiceException e) { + // todo message + throw new FileUploadException(e); + } } private static String generateResourceName(String bucketName, String key) { From 2ab00e0c5d2493eaae45408784b4d15e78efb058 Mon Sep 17 00:00:00 2001 From: gulnaz Date: Sat, 28 Sep 2024 10:47:55 +0300 Subject: [PATCH 12/12] fix props --- micord.env | 4 ++-- src/main/java/ru/micord/ervu/av/s3/S3Config.java | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/micord.env b/micord.env index 130f803..7f1c15a 100644 --- a/micord.env +++ b/micord.env @@ -27,8 +27,8 @@ FILE_SAVING_PATH=/transfer/ S3_ENDPOINT=http://ervu-minio.k8s.micord.ru:31900 S3_ACCESS_KEY=rlTdTvkmSXu9FsLhfecw S3_SECRET_KEY=NUmY0wwRIEyAd98GCKd1cOgJWvLQYAcMMul5Ulu0 -S3_OUT_BUCKET_NAME=default-out-bucket -S3_OUT_PATH_STYLE_ACCESS_ENABLED=true +S3_BUCKET_NAME=default-out-bucket +S3_PATH_STYLE_ACCESS_ENABLED=true SPRING_DATASOURCE_URL=jdbc:postgresql://10.10.31.119:5432/ervu-lkrp-ul SPRING_DATASOURCE_USERNAME=ervu-lkrp-ul diff --git a/src/main/java/ru/micord/ervu/av/s3/S3Config.java b/src/main/java/ru/micord/ervu/av/s3/S3Config.java index 02791d8..722bffb 100644 --- a/src/main/java/ru/micord/ervu/av/s3/S3Config.java +++ b/src/main/java/ru/micord/ervu/av/s3/S3Config.java @@ -17,15 +17,15 @@ import org.springframework.context.annotation.Configuration; */ @Configuration public class S3Config { - @Value("${s3.out.endpoint}") + @Value("${s3.endpoint}") private String endpoint; - @Value("${s3.out.access_key}") + @Value("${s3.access_key}") private String accessKey; - @Value("${s3.out.secret_key}") + @Value("${s3.secret_key}") private String secretKey; - @Value("${s3.out.bucket_name}") + @Value("${s3.bucket_name}") private String bucketName; - @Value("${s3.out.path.style.enabled:true}") + @Value("${s3.path.style.enabled:true}") private boolean pathStyleAccessEnabled; @Bean("outBucketName")