From 828884954d9d56956a08696abc6c37d23d35cd39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A0=D0=B0=D1=83=D1=84=20=D0=9B=D0=B0=D1=82=D1=8B=D0=BF?= =?UTF-8?q?=D0=BE=D0=B2?= Date: Wed, 24 Jul 2024 09:59:34 +0300 Subject: [PATCH 01/30] SUPPORT-8400: FileUploadService --- .gitignore | 22 ++ file-upload/pom.xml | 96 +++++++ .../cg/subproject/FileUploadApplication.java | 15 ++ .../java/ru/cg/subproject/av/AvResponse.java | 14 ++ .../exception/FileUploadException.java | 14 ++ .../InvalidHttpFileUrlException.java | 20 ++ .../kafka/config/KafkaConsumerConfig.java | 64 +++++ .../kafka/config/KafkaProducerConfig.java | 51 ++++ .../kafka/config/KafkaTopicConfig.java | 28 +++ .../ru/cg/subproject/kafka/dto/InMessage.java | 36 +++ .../subproject/kafka/dto/OutErrorMessage.java | 12 + .../subproject/service/FileUploadService.java | 235 ++++++++++++++++++ .../ru/cg/subproject/service/FileUrl.java | 7 + .../src/main/resources/application.properties | 32 +++ 14 files changed, 646 insertions(+) create mode 100644 .gitignore create mode 100644 file-upload/pom.xml create mode 100644 file-upload/src/main/java/ru/cg/subproject/FileUploadApplication.java create mode 100644 file-upload/src/main/java/ru/cg/subproject/av/AvResponse.java create mode 100644 file-upload/src/main/java/ru/cg/subproject/exception/FileUploadException.java create mode 100644 file-upload/src/main/java/ru/cg/subproject/exception/InvalidHttpFileUrlException.java create mode 100644 file-upload/src/main/java/ru/cg/subproject/kafka/config/KafkaConsumerConfig.java create mode 100644 file-upload/src/main/java/ru/cg/subproject/kafka/config/KafkaProducerConfig.java create mode 100644 file-upload/src/main/java/ru/cg/subproject/kafka/config/KafkaTopicConfig.java create mode 100644 file-upload/src/main/java/ru/cg/subproject/kafka/dto/InMessage.java create mode 100644 file-upload/src/main/java/ru/cg/subproject/kafka/dto/OutErrorMessage.java create mode 100644 file-upload/src/main/java/ru/cg/subproject/service/FileUploadService.java create mode 100644 file-upload/src/main/java/ru/cg/subproject/service/FileUrl.java create mode 100644 file-upload/src/main/resources/application.properties diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d99b869 --- /dev/null +++ b/.gitignore @@ -0,0 +1,22 @@ +#ignore target and tmp dir +target*/ +tmp*/ +debug.log + +# +# IntelliJ IDEA project files +# +.idea*/ +.classes*/ +*.ipr +*.iml +*.iws +*.ids + +# os meta files +Thumbs.db +.DS_Store + +*.orig +pom.xml.versionsBackup +.flattened-pom.xml diff --git a/file-upload/pom.xml b/file-upload/pom.xml new file mode 100644 index 0000000..816b840 --- /dev/null +++ b/file-upload/pom.xml @@ -0,0 +1,96 @@ + + + 4.0.0 + + + org.springframework.boot + spring-boot-starter-parent + 3.3.0 + + + + ru.cg.subproject + file-upload + 0.0.1-SNAPSHOT + + + 17 + + + + + + com.google.code.gson + gson + 2.11.0 + + + + org.apache.httpcomponents + httpclient + 4.5.14 + + + org.apache.httpcomponents + httpmime + 4.5.14 + + + + org.projectlombok + lombok + 1.18.34 + provided + + + + org.springframework.kafka + spring-kafka + 3.2.1 + + + + + + + com.google.code.gson + gson + + + + org.apache.httpcomponents + httpclient + + + org.apache.httpcomponents + httpmime + + + + org.projectlombok + lombok + + + + org.springframework.boot + spring-boot-starter + + + + org.springframework.kafka + spring-kafka + + + + + + + org.springframework.boot + spring-boot-maven-plugin + + + + + diff --git a/file-upload/src/main/java/ru/cg/subproject/FileUploadApplication.java b/file-upload/src/main/java/ru/cg/subproject/FileUploadApplication.java new file mode 100644 index 0000000..f9f4b06 --- /dev/null +++ b/file-upload/src/main/java/ru/cg/subproject/FileUploadApplication.java @@ -0,0 +1,15 @@ +package ru.cg.subproject; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; + +/** + * @author r.latypov + */ +@SpringBootApplication +public class FileUploadApplication { + + public static void main(String[] args) { + SpringApplication.run(FileUploadApplication.class, args); + } +} diff --git a/file-upload/src/main/java/ru/cg/subproject/av/AvResponse.java b/file-upload/src/main/java/ru/cg/subproject/av/AvResponse.java new file mode 100644 index 0000000..92a894f --- /dev/null +++ b/file-upload/src/main/java/ru/cg/subproject/av/AvResponse.java @@ -0,0 +1,14 @@ +package ru.cg.subproject.av; + +/** + * @author r.latypov + */ +public record AvResponse(String completed, String created, String progress, ScanResult scan_result, + String status, String[] verdicts) { + public record ScanResult(Scan noname) { + public record Scan(String started, String stopped, Threat[] threats, String verdict) { + public record Threat(String name, String object) { + } + } + } +} diff --git a/file-upload/src/main/java/ru/cg/subproject/exception/FileUploadException.java b/file-upload/src/main/java/ru/cg/subproject/exception/FileUploadException.java new file mode 100644 index 0000000..b11dea2 --- /dev/null +++ b/file-upload/src/main/java/ru/cg/subproject/exception/FileUploadException.java @@ -0,0 +1,14 @@ +package ru.cg.subproject.exception; + +/** + * @author r.latypov + */ +public class FileUploadException extends Exception { + public FileUploadException(String message) { + super(message); + } + + public FileUploadException(Throwable cause) { + super(cause); + } +} diff --git a/file-upload/src/main/java/ru/cg/subproject/exception/InvalidHttpFileUrlException.java b/file-upload/src/main/java/ru/cg/subproject/exception/InvalidHttpFileUrlException.java new file mode 100644 index 0000000..9ff1f80 --- /dev/null +++ b/file-upload/src/main/java/ru/cg/subproject/exception/InvalidHttpFileUrlException.java @@ -0,0 +1,20 @@ +package ru.cg.subproject.exception; + +/** + * @author r.latypov + */ +public class InvalidHttpFileUrlException extends Exception { + private static final String MESSAGE = "file url is not valid"; + + public InvalidHttpFileUrlException() { + super(MESSAGE); + } + + public InvalidHttpFileUrlException(String message) { + super(String.join(" : ", MESSAGE, message)); + } + + public InvalidHttpFileUrlException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/file-upload/src/main/java/ru/cg/subproject/kafka/config/KafkaConsumerConfig.java b/file-upload/src/main/java/ru/cg/subproject/kafka/config/KafkaConsumerConfig.java new file mode 100644 index 0000000..f428816 --- /dev/null +++ b/file-upload/src/main/java/ru/cg/subproject/kafka/config/KafkaConsumerConfig.java @@ -0,0 +1,64 @@ +package ru.cg.subproject.kafka.config; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.kafka.clients.CommonClientConfigs; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.common.config.SaslConfigs; +import org.apache.kafka.common.serialization.StringDeserializer; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.kafka.annotation.EnableKafka; +import org.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory; +import org.springframework.kafka.core.ConsumerFactory; +import org.springframework.kafka.core.DefaultKafkaConsumerFactory; +import org.springframework.kafka.listener.ContainerProperties; + +/** + * @author r.latypov + */ +@Configuration +@EnableKafka +public class KafkaConsumerConfig { + @Value("${spring.kafka.consumer.bootstrap-servers}") + private List bootstrapAddress; + @Value("${spring.kafka.consumer.security.protocol}") + private String securityProtocol; + @Value("${spring.kafka.consumer.properties.sasl.jaas.config}") + private String jaasConfig; + @Value("${spring.kafka.consumer.properties.sasl.mechanism}") + private String saslMechanism; + @Value("${spring.kafka.consumer.enable-auto-commit}") + private String enableAutoCommit; + @Value("${spring.kafka.listener.ack-mode}") + private String ackMode; + + @Bean + public ConsumerFactory consumerFactory() { + Map configs = new HashMap<>(); + + configs.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapAddress); + configs.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); + configs.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); + + configs.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, securityProtocol); + configs.put(SaslConfigs.SASL_JAAS_CONFIG, jaasConfig); + configs.put(SaslConfigs.SASL_MECHANISM, saslMechanism); + + configs.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, enableAutoCommit); + + return new DefaultKafkaConsumerFactory<>(configs); + } + + @Bean + public ConcurrentKafkaListenerContainerFactory kafkaListenerContainerFactory() { + ConcurrentKafkaListenerContainerFactory factory = + new ConcurrentKafkaListenerContainerFactory<>(); + factory.setConsumerFactory(consumerFactory()); + factory.getContainerProperties().setAckMode(ContainerProperties.AckMode.MANUAL_IMMEDIATE); + return factory; + } +} diff --git a/file-upload/src/main/java/ru/cg/subproject/kafka/config/KafkaProducerConfig.java b/file-upload/src/main/java/ru/cg/subproject/kafka/config/KafkaProducerConfig.java new file mode 100644 index 0000000..389ec9b --- /dev/null +++ b/file-upload/src/main/java/ru/cg/subproject/kafka/config/KafkaProducerConfig.java @@ -0,0 +1,51 @@ +package ru.cg.subproject.kafka.config; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.kafka.clients.CommonClientConfigs; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.common.config.SaslConfigs; +import org.apache.kafka.common.serialization.StringSerializer; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.kafka.core.DefaultKafkaProducerFactory; +import org.springframework.kafka.core.KafkaTemplate; +import org.springframework.kafka.core.ProducerFactory; + +/** + * @author r.latypov + */ +@Configuration +public class KafkaProducerConfig { + @Value("${spring.kafka.producer.bootstrap-servers}") + private List bootstrapAddress; + @Value("${spring.kafka.producer.security.protocol}") + private String securityProtocol; + @Value("${spring.kafka.producer.properties.sasl.jaas.config}") + private String jaasConfig; + @Value("${spring.kafka.producer.properties.sasl.mechanism}") + private String saslMechanism; + + @Bean + public ProducerFactory producerFactory() { + Map configs = new HashMap<>(); + + configs.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapAddress); + configs.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class); + configs.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class); + + configs.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, securityProtocol); + configs.put(SaslConfigs.SASL_JAAS_CONFIG, jaasConfig); + configs.put(SaslConfigs.SASL_MECHANISM, saslMechanism); + + return new DefaultKafkaProducerFactory<>(configs); + } + + @Bean + public KafkaTemplate kafkaTemplate() { + return new KafkaTemplate<>(producerFactory()); + } +} diff --git a/file-upload/src/main/java/ru/cg/subproject/kafka/config/KafkaTopicConfig.java b/file-upload/src/main/java/ru/cg/subproject/kafka/config/KafkaTopicConfig.java new file mode 100644 index 0000000..a44b82f --- /dev/null +++ b/file-upload/src/main/java/ru/cg/subproject/kafka/config/KafkaTopicConfig.java @@ -0,0 +1,28 @@ +package ru.cg.subproject.kafka.config; + +import org.apache.kafka.clients.admin.NewTopic; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.kafka.config.TopicBuilder; + +/** + * @author r.latypov + */ +@Configuration +public class KafkaTopicConfig { + @Value("${kafka-out.topic.error.name}") + private String kafkaOutTopicErrorName; + @Value("${kafka-out.topic.success.name}") + private String kafkaOutTopicSuccessName; + + @Bean + public NewTopic outErrorTopic() { + return TopicBuilder.name(kafkaOutTopicErrorName).build(); + } + + @Bean + public NewTopic outSuccessTopic() { + return TopicBuilder.name(kafkaOutTopicSuccessName).build(); + } +} diff --git a/file-upload/src/main/java/ru/cg/subproject/kafka/dto/InMessage.java b/file-upload/src/main/java/ru/cg/subproject/kafka/dto/InMessage.java new file mode 100644 index 0000000..1b388e1 --- /dev/null +++ b/file-upload/src/main/java/ru/cg/subproject/kafka/dto/InMessage.java @@ -0,0 +1,36 @@ +package ru.cg.subproject.kafka.dto; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.Setter; +import org.springframework.lang.NonNull; + +/** + * @author r.latypov + */ +public record InMessage(OrgInfo orgInfo, SenderInfo senderInfo, @NonNull FileInfo fileInfo) { + public record OrgInfo(String orgName, String orgTypeCode, String orgTypeName, String ogrn, + String in, String kpp) { + } + + public record SenderInfo(String senderLastName, String senderFirstName, + String senderMiddleName, String birthDate, String senderRoleCode, + String senderRoleName, String snils, String idERN, + Document document) { + + public record Document(String series, String number, String issueDate) { + } + } + + @Getter + @AllArgsConstructor + public static class FileInfo { + @NonNull + @Setter + private String fileNameBase; + private final String fileName; + private final String filePatternCode; + private final String FilePatternName; + private final String departureDateTime; + } +} diff --git a/file-upload/src/main/java/ru/cg/subproject/kafka/dto/OutErrorMessage.java b/file-upload/src/main/java/ru/cg/subproject/kafka/dto/OutErrorMessage.java new file mode 100644 index 0000000..0cba965 --- /dev/null +++ b/file-upload/src/main/java/ru/cg/subproject/kafka/dto/OutErrorMessage.java @@ -0,0 +1,12 @@ +package ru.cg.subproject.kafka.dto; + +import org.springframework.lang.NonNull; + +import ru.cg.subproject.av.AvResponse; + +/** + * @author r.latypov + */ +public record OutErrorMessage(String errorMessage, AvResponse avResponse, + @NonNull InMessage inMessage) { +} diff --git a/file-upload/src/main/java/ru/cg/subproject/service/FileUploadService.java b/file-upload/src/main/java/ru/cg/subproject/service/FileUploadService.java new file mode 100644 index 0000000..78965f4 --- /dev/null +++ b/file-upload/src/main/java/ru/cg/subproject/service/FileUploadService.java @@ -0,0 +1,235 @@ +package ru.cg.subproject.service; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.Arrays; +import java.util.concurrent.CompletableFuture; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import org.apache.http.HttpEntity; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpDelete; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.methods.HttpPut; +import org.apache.http.conn.HttpHostConnectException; +import org.apache.http.entity.mime.MultipartEntityBuilder; +import org.apache.http.entity.mime.content.FileBody; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.kafka.clients.admin.NewTopic; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.kafka.annotation.KafkaListener; +import org.springframework.kafka.core.KafkaTemplate; +import org.springframework.kafka.support.Acknowledgment; +import org.springframework.kafka.support.SendResult; +import org.springframework.lang.NonNull; +import org.springframework.stereotype.Service; + +import ru.cg.subproject.av.AvResponse; +import ru.cg.subproject.exception.FileUploadException; +import ru.cg.subproject.exception.InvalidHttpFileUrlException; +import ru.cg.subproject.kafka.dto.InMessage; +import ru.cg.subproject.kafka.dto.OutErrorMessage; + +/** + * @author r.latypov + */ +@Service +public class FileUploadService { + private static final Logger logger = LoggerFactory.getLogger(FileUploadService.class); + @Value("${av.rest.address}") + private String avRestAddress; + @Value("${file.before.av.path:}") + private String fileBeforeAvPath; + @Value("${http.file.server.out.address:http://localhost/out}") + private String httpFileServerOutAddress; + private final KafkaTemplate kafkaTemplate; + private final NewTopic outErrorTopic; + private final NewTopic outSuccessTopic; + + @Autowired + public FileUploadService(KafkaTemplate kafkaTemplate, NewTopic outErrorTopic, + NewTopic outSuccessTopic) { + this.kafkaTemplate = kafkaTemplate; + this.outErrorTopic = outErrorTopic; + this.outSuccessTopic = outSuccessTopic; + } + + @KafkaListener(id = "${spring.kafka.consumer.group-id}", topics = "${kafka-in.topic.name}") + public void listenKafkaIn(String kafkaInMessage, Acknowledgment acknowledgment) { + InMessage inMessage = new Gson().fromJson(kafkaInMessage, InMessage.class); + + try { + FileUrl fileUrl = parseFileUrl(inMessage.fileInfo().getFileNameBase()); + String filePath = fileBeforeAvPath + fileUrl.fileName(); + String downloadUrl = fileUrl.fileUrl(); + downloadFileByHttp(downloadUrl, filePath); + + AvResponse avResponse = sendFileToAvScan(filePath); + + boolean infected = Arrays.stream(avResponse.verdicts()) + .anyMatch(verdict -> verdict.equalsIgnoreCase("infected")); + + if (infected) { + sendKafkaMessage(outErrorTopic.name(), + new OutErrorMessage("file is infected", avResponse, inMessage) + ); + } + else { + String uploadUrl = httpFileServerOutAddress + "/" + fileUrl.fileName(); + uploadFileByHttp(filePath, uploadUrl); + + inMessage.fileInfo().setFileNameBase(uploadUrl); + sendKafkaMessage(outSuccessTopic.name(), inMessage); + } + + deleteFileByHttp(downloadUrl); + if (new File(filePath).delete()) { + acknowledgment.acknowledge(); + } + } + catch (InvalidHttpFileUrlException e) { + logger.error(e.getMessage() + ": " + kafkaInMessage); + + sendKafkaMessage(outErrorTopic.name(), + new OutErrorMessage(e.getMessage(), null, inMessage) + ); + acknowledgment.acknowledge(); + } + catch (FileUploadException e) { + logger.error(e.getMessage(), e); + } + } + + private static FileUrl parseFileUrl(@NonNull String fileUrl) throws InvalidHttpFileUrlException { + String[] substrings = fileUrl.split("/"); + String fileName = substrings[substrings.length - 1]; + if (substrings.length == 1 || fileName.isBlank()) { + throw new InvalidHttpFileUrlException(fileUrl); + } + else { + return new FileUrl(fileName, fileUrl); + } + } + + private void downloadFileByHttp(String fileUrl, String filePath) + throws InvalidHttpFileUrlException, FileUploadException { + File file = new File(filePath); + HttpGet request = new HttpGet(fileUrl); + + try (CloseableHttpClient client = HttpClients.createDefault(); + CloseableHttpResponse response = client.execute(request)) { + + int statusCode = response.getStatusLine().getStatusCode(); + if (statusCode == 200) { + HttpEntity entity = response.getEntity(); + if (entity != null) { + try (FileOutputStream outputStream = new FileOutputStream(file)) { + entity.writeTo(outputStream); + } + } + } + else { + String message = "http status code " + statusCode + " : " + fileUrl; + throw new InvalidHttpFileUrlException(message); + } + } + catch (HttpHostConnectException e) { + throw new FileUploadException(e); + } + catch (IOException e) { + String message = + (e.getMessage() == null ? e.getCause().getMessage() : e.getMessage()) + " : " + fileUrl; + throw new InvalidHttpFileUrlException(message, e); + } + } + + private AvResponse sendFileToAvScan(String filePath) throws FileUploadException { + File file = new File(filePath); + + try (CloseableHttpClient client = HttpClients.createDefault()) { + HttpPost post = new HttpPost(avRestAddress); + post.addHeader("Content-type", "application/json"); + HttpEntity entity = MultipartEntityBuilder.create() + .addPart("file", new FileBody(file)) + .build(); + post.setEntity(entity); + + try (CloseableHttpResponse response = client.execute(post)) { + int statusCode = response.getStatusLine().getStatusCode(); + if (statusCode != 200) { + String message = "http status code " + statusCode + " for " + avRestAddress + " request"; + throw new FileUploadException(message); + } + return new Gson().fromJson(response.getEntity().toString(), AvResponse.class); + } + } + catch (IOException e) { + throw new FileUploadException(e); + } + } + + private void uploadFileByHttp(String filePath, String uploadUrl) throws FileUploadException { + File file = new File(filePath); + + try (CloseableHttpClient client = HttpClients.createDefault()) { + HttpPut put = new HttpPut(uploadUrl); + HttpEntity entity = MultipartEntityBuilder.create() + .addPart("file", new FileBody(file)) + .build(); + put.setEntity(entity); + + try (CloseableHttpResponse response = client.execute(put)) { + int statusCode = response.getStatusLine().getStatusCode(); + if (statusCode == 204) { + logger.warn("file already exists : " + uploadUrl); + } + else if (statusCode != 201) { + String message = "http status code " + statusCode + " : " + uploadUrl; + throw new RuntimeException(message); + } + } + } + catch (IOException e) { + throw new FileUploadException(e); + } + } + + private void deleteFileByHttp(String fileUrl) throws FileUploadException { + try (CloseableHttpClient client = HttpClients.createDefault()) { + HttpDelete delete = new HttpDelete(fileUrl); + + try (CloseableHttpResponse response = client.execute(delete)) { + int statusCode = response.getStatusLine().getStatusCode(); + if (statusCode != 204) { + String message = "http status code " + statusCode + " : " + fileUrl; + throw new RuntimeException(message); + } + } + } + catch (IOException e) { + throw new FileUploadException(e); + } + } + + private void sendKafkaMessage(@NonNull String topicName, Object object) { + CompletableFuture> future = kafkaTemplate.send(topicName, + new GsonBuilder().setPrettyPrinting().create().toJson(object) + ); + + future.whenComplete((result, e) -> { + if (e != null) { + String message = + "Unable to send message [" + result.getProducerRecord().value() + "] into kafka topic '" + + topicName + "' due to : " + e.getMessage(); + throw new RuntimeException(message, e); + } + }); + } +} diff --git a/file-upload/src/main/java/ru/cg/subproject/service/FileUrl.java b/file-upload/src/main/java/ru/cg/subproject/service/FileUrl.java new file mode 100644 index 0000000..910181d --- /dev/null +++ b/file-upload/src/main/java/ru/cg/subproject/service/FileUrl.java @@ -0,0 +1,7 @@ +package ru.cg.subproject.service; + +/** + * @author r.latypov + */ +public record FileUrl(String fileName, String fileUrl) { +} diff --git a/file-upload/src/main/resources/application.properties b/file-upload/src/main/resources/application.properties new file mode 100644 index 0000000..35d1ad6 --- /dev/null +++ b/file-upload/src/main/resources/application.properties @@ -0,0 +1,32 @@ +spring.kafka.admin.security.protocol=SASL_PLAINTEXT +spring.kafka.admin.properties.sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required username="user1" password="Blfi9d2OFG"; +spring.kafka.admin.properties.sasl.mechanism=SCRAM-SHA-256 +# +spring.kafka.bootstrap-servers=10.10.31.11:32609 +# +spring.kafka.consumer.bootstrap-servers=10.10.31.11:32609 +spring.kafka.consumer.security.protocol=SASL_PLAINTEXT +spring.kafka.consumer.properties.sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required username="user1" password="Blfi9d2OFG"; +spring.kafka.consumer.properties.sasl.mechanism=SCRAM-SHA-256 +# +spring.kafka.consumer.enable-auto-commit=false +spring.kafka.consumer.group-id=file-to-upload-consumers +# +spring.kafka.listener.ack-mode=MANUAL_IMMEDIATE +# +spring.kafka.producer.bootstrap-servers=10.10.31.11:32609 +spring.kafka.producer.security.protocol=SASL_PLAINTEXT +spring.kafka.producer.properties.sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required username="user1" password="Blfi9d2OFG"; +spring.kafka.producer.properties.sasl.mechanism=SCRAM-SHA-256 +# +spring.kafka.properties.security.protocol=SASL_PLAINTEXT +spring.kafka.properties.sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required username="user1" password="Blfi9d2OFG"; +spring.kafka.properties.sasl.mechanism=SCRAM-SHA-256 +# +kafka-in.topic.name=file-to-upload +kafka-out.topic.error.name=error +kafka-out.topic.success.name=success +# +av.rest.address=http://:/scans +file.before.av.path=/nginx/transfer/ +http.file.server.out.address=http://localhost/out From bca11f37a64dab90a9b5a7f6c34409baf46aa54f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A0=D0=B0=D1=83=D1=84=20=D0=9B=D0=B0=D1=82=D1=8B=D0=BF?= =?UTF-8?q?=D0=BE=D0=B2?= Date: Thu, 25 Jul 2024 10:22:20 +0300 Subject: [PATCH 02/30] SUPPORT-8400: fix for review (1) --- .../cg/subproject/kafka/config/KafkaTopicConfig.java | 12 ++++++------ .../ru/cg/subproject/service/FileUploadService.java | 9 ++++++--- .../main/java/ru/cg/subproject/service/FileUrl.java | 7 ------- .../src/main/resources/application.properties | 6 +++--- 4 files changed, 15 insertions(+), 19 deletions(-) delete mode 100644 file-upload/src/main/java/ru/cg/subproject/service/FileUrl.java diff --git a/file-upload/src/main/java/ru/cg/subproject/kafka/config/KafkaTopicConfig.java b/file-upload/src/main/java/ru/cg/subproject/kafka/config/KafkaTopicConfig.java index a44b82f..0795327 100644 --- a/file-upload/src/main/java/ru/cg/subproject/kafka/config/KafkaTopicConfig.java +++ b/file-upload/src/main/java/ru/cg/subproject/kafka/config/KafkaTopicConfig.java @@ -11,18 +11,18 @@ import org.springframework.kafka.config.TopicBuilder; */ @Configuration public class KafkaTopicConfig { - @Value("${kafka-out.topic.error.name}") - private String kafkaOutTopicErrorName; - @Value("${kafka-out.topic.success.name}") - private String kafkaOutTopicSuccessName; + @Value("${kafka-out.error.topic.name}") + private String kafkaOutErrorTopicName; + @Value("${kafka-out.success.topic.name}") + private String kafkaOutSuccessTopicName; @Bean public NewTopic outErrorTopic() { - return TopicBuilder.name(kafkaOutTopicErrorName).build(); + return TopicBuilder.name(kafkaOutErrorTopicName).build(); } @Bean public NewTopic outSuccessTopic() { - return TopicBuilder.name(kafkaOutTopicSuccessName).build(); + return TopicBuilder.name(kafkaOutSuccessTopicName).build(); } } diff --git a/file-upload/src/main/java/ru/cg/subproject/service/FileUploadService.java b/file-upload/src/main/java/ru/cg/subproject/service/FileUploadService.java index 78965f4..121aa90 100644 --- a/file-upload/src/main/java/ru/cg/subproject/service/FileUploadService.java +++ b/file-upload/src/main/java/ru/cg/subproject/service/FileUploadService.java @@ -45,8 +45,8 @@ public class FileUploadService { private static final Logger logger = LoggerFactory.getLogger(FileUploadService.class); @Value("${av.rest.address}") private String avRestAddress; - @Value("${file.before.av.path:}") - private String fileBeforeAvPath; + @Value("${file.saving.path}") + private String fileSavingPath; @Value("${http.file.server.out.address:http://localhost/out}") private String httpFileServerOutAddress; private final KafkaTemplate kafkaTemplate; @@ -67,7 +67,7 @@ public class FileUploadService { try { FileUrl fileUrl = parseFileUrl(inMessage.fileInfo().getFileNameBase()); - String filePath = fileBeforeAvPath + fileUrl.fileName(); + String filePath = fileSavingPath + fileUrl.fileName(); String downloadUrl = fileUrl.fileUrl(); downloadFileByHttp(downloadUrl, filePath); @@ -232,4 +232,7 @@ public class FileUploadService { } }); } + + private record FileUrl(String fileName, String fileUrl) { + } } diff --git a/file-upload/src/main/java/ru/cg/subproject/service/FileUrl.java b/file-upload/src/main/java/ru/cg/subproject/service/FileUrl.java deleted file mode 100644 index 910181d..0000000 --- a/file-upload/src/main/java/ru/cg/subproject/service/FileUrl.java +++ /dev/null @@ -1,7 +0,0 @@ -package ru.cg.subproject.service; - -/** - * @author r.latypov - */ -public record FileUrl(String fileName, String fileUrl) { -} diff --git a/file-upload/src/main/resources/application.properties b/file-upload/src/main/resources/application.properties index 35d1ad6..f279a02 100644 --- a/file-upload/src/main/resources/application.properties +++ b/file-upload/src/main/resources/application.properties @@ -24,9 +24,9 @@ spring.kafka.properties.sasl.jaas.config=org.apache.kafka.common.security.scram. spring.kafka.properties.sasl.mechanism=SCRAM-SHA-256 # kafka-in.topic.name=file-to-upload -kafka-out.topic.error.name=error -kafka-out.topic.success.name=success +kafka-out.error.topic.name=error +kafka-out.success.topic.name=success # av.rest.address=http://:/scans -file.before.av.path=/nginx/transfer/ +file.saving.path=/nginx/transfer/ http.file.server.out.address=http://localhost/out From 3b6f9ca57f6f21050aa145776b4c75f8d1831c51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A0=D0=B0=D1=83=D1=84=20=D0=9B=D0=B0=D1=82=D1=8B=D0=BF?= =?UTF-8?q?=D0=BE=D0=B2?= Date: Thu, 25 Jul 2024 14:45:54 +0300 Subject: [PATCH 03/30] =?UTF-8?q?SUPPORT-8400:=20=D1=80=D0=B0=D0=B7=D0=BB?= =?UTF-8?q?=D0=BE=D0=B6=D0=B8=D0=BB=20=D0=BF=D0=BE=20=D0=BF=D0=B0=D0=BA?= =?UTF-8?q?=D0=B5=D1=82=D0=B0=D0=BC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../kafka/config/{ => input}/KafkaConsumerConfig.java | 2 +- .../kafka/config/{ => output}/KafkaProducerConfig.java | 2 +- .../subproject/kafka/config/{ => output}/KafkaTopicConfig.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) rename file-upload/src/main/java/ru/cg/subproject/kafka/config/{ => input}/KafkaConsumerConfig.java (98%) rename file-upload/src/main/java/ru/cg/subproject/kafka/config/{ => output}/KafkaProducerConfig.java (97%) rename file-upload/src/main/java/ru/cg/subproject/kafka/config/{ => output}/KafkaTopicConfig.java (94%) diff --git a/file-upload/src/main/java/ru/cg/subproject/kafka/config/KafkaConsumerConfig.java b/file-upload/src/main/java/ru/cg/subproject/kafka/config/input/KafkaConsumerConfig.java similarity index 98% rename from file-upload/src/main/java/ru/cg/subproject/kafka/config/KafkaConsumerConfig.java rename to file-upload/src/main/java/ru/cg/subproject/kafka/config/input/KafkaConsumerConfig.java index f428816..107fe20 100644 --- a/file-upload/src/main/java/ru/cg/subproject/kafka/config/KafkaConsumerConfig.java +++ b/file-upload/src/main/java/ru/cg/subproject/kafka/config/input/KafkaConsumerConfig.java @@ -1,4 +1,4 @@ -package ru.cg.subproject.kafka.config; +package ru.cg.subproject.kafka.config.input; import java.util.HashMap; import java.util.List; diff --git a/file-upload/src/main/java/ru/cg/subproject/kafka/config/KafkaProducerConfig.java b/file-upload/src/main/java/ru/cg/subproject/kafka/config/output/KafkaProducerConfig.java similarity index 97% rename from file-upload/src/main/java/ru/cg/subproject/kafka/config/KafkaProducerConfig.java rename to file-upload/src/main/java/ru/cg/subproject/kafka/config/output/KafkaProducerConfig.java index 389ec9b..a898965 100644 --- a/file-upload/src/main/java/ru/cg/subproject/kafka/config/KafkaProducerConfig.java +++ b/file-upload/src/main/java/ru/cg/subproject/kafka/config/output/KafkaProducerConfig.java @@ -1,4 +1,4 @@ -package ru.cg.subproject.kafka.config; +package ru.cg.subproject.kafka.config.output; import java.util.HashMap; import java.util.List; diff --git a/file-upload/src/main/java/ru/cg/subproject/kafka/config/KafkaTopicConfig.java b/file-upload/src/main/java/ru/cg/subproject/kafka/config/output/KafkaTopicConfig.java similarity index 94% rename from file-upload/src/main/java/ru/cg/subproject/kafka/config/KafkaTopicConfig.java rename to file-upload/src/main/java/ru/cg/subproject/kafka/config/output/KafkaTopicConfig.java index 0795327..feaf43d 100644 --- a/file-upload/src/main/java/ru/cg/subproject/kafka/config/KafkaTopicConfig.java +++ b/file-upload/src/main/java/ru/cg/subproject/kafka/config/output/KafkaTopicConfig.java @@ -1,4 +1,4 @@ -package ru.cg.subproject.kafka.config; +package ru.cg.subproject.kafka.config.output; import org.apache.kafka.clients.admin.NewTopic; import org.springframework.beans.factory.annotation.Value; From bd89e15590e8e2111cdfb080404955668690deeb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A0=D0=B0=D1=83=D1=84=20=D0=9B=D0=B0=D1=82=D1=8B=D0=BF?= =?UTF-8?q?=D0=BE=D0=B2?= Date: Thu, 25 Jul 2024 14:54:22 +0300 Subject: [PATCH 04/30] =?UTF-8?q?SUPPORT-8400:=20=D1=80=D0=B0=D0=B7=D0=BB?= =?UTF-8?q?=D0=BE=D0=B6=D0=B8=D0=BB=20=D0=BF=D0=BE=20=D0=BF=D0=B0=D0=BA?= =?UTF-8?q?=D0=B5=D1=82=D0=B0=D0=BC=20(2)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../{KafkaConsumerConfig.java => InputKafkaConsumerConfig.java} | 2 +- ...{KafkaProducerConfig.java => OutputKafkaProducerConfig.java} | 2 +- .../{KafkaTopicConfig.java => OutputKafkaTopicConfig.java} | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) rename file-upload/src/main/java/ru/cg/subproject/kafka/config/input/{KafkaConsumerConfig.java => InputKafkaConsumerConfig.java} (98%) rename file-upload/src/main/java/ru/cg/subproject/kafka/config/output/{KafkaProducerConfig.java => OutputKafkaProducerConfig.java} (97%) rename file-upload/src/main/java/ru/cg/subproject/kafka/config/output/{KafkaTopicConfig.java => OutputKafkaTopicConfig.java} (95%) diff --git a/file-upload/src/main/java/ru/cg/subproject/kafka/config/input/KafkaConsumerConfig.java b/file-upload/src/main/java/ru/cg/subproject/kafka/config/input/InputKafkaConsumerConfig.java similarity index 98% rename from file-upload/src/main/java/ru/cg/subproject/kafka/config/input/KafkaConsumerConfig.java rename to file-upload/src/main/java/ru/cg/subproject/kafka/config/input/InputKafkaConsumerConfig.java index 107fe20..aa4def0 100644 --- a/file-upload/src/main/java/ru/cg/subproject/kafka/config/input/KafkaConsumerConfig.java +++ b/file-upload/src/main/java/ru/cg/subproject/kafka/config/input/InputKafkaConsumerConfig.java @@ -22,7 +22,7 @@ import org.springframework.kafka.listener.ContainerProperties; */ @Configuration @EnableKafka -public class KafkaConsumerConfig { +public class InputKafkaConsumerConfig { @Value("${spring.kafka.consumer.bootstrap-servers}") private List bootstrapAddress; @Value("${spring.kafka.consumer.security.protocol}") diff --git a/file-upload/src/main/java/ru/cg/subproject/kafka/config/output/KafkaProducerConfig.java b/file-upload/src/main/java/ru/cg/subproject/kafka/config/output/OutputKafkaProducerConfig.java similarity index 97% rename from file-upload/src/main/java/ru/cg/subproject/kafka/config/output/KafkaProducerConfig.java rename to file-upload/src/main/java/ru/cg/subproject/kafka/config/output/OutputKafkaProducerConfig.java index a898965..d20be6c 100644 --- a/file-upload/src/main/java/ru/cg/subproject/kafka/config/output/KafkaProducerConfig.java +++ b/file-upload/src/main/java/ru/cg/subproject/kafka/config/output/OutputKafkaProducerConfig.java @@ -19,7 +19,7 @@ import org.springframework.kafka.core.ProducerFactory; * @author r.latypov */ @Configuration -public class KafkaProducerConfig { +public class OutputKafkaProducerConfig { @Value("${spring.kafka.producer.bootstrap-servers}") private List bootstrapAddress; @Value("${spring.kafka.producer.security.protocol}") diff --git a/file-upload/src/main/java/ru/cg/subproject/kafka/config/output/KafkaTopicConfig.java b/file-upload/src/main/java/ru/cg/subproject/kafka/config/output/OutputKafkaTopicConfig.java similarity index 95% rename from file-upload/src/main/java/ru/cg/subproject/kafka/config/output/KafkaTopicConfig.java rename to file-upload/src/main/java/ru/cg/subproject/kafka/config/output/OutputKafkaTopicConfig.java index feaf43d..accae7f 100644 --- a/file-upload/src/main/java/ru/cg/subproject/kafka/config/output/KafkaTopicConfig.java +++ b/file-upload/src/main/java/ru/cg/subproject/kafka/config/output/OutputKafkaTopicConfig.java @@ -10,7 +10,7 @@ import org.springframework.kafka.config.TopicBuilder; * @author r.latypov */ @Configuration -public class KafkaTopicConfig { +public class OutputKafkaTopicConfig { @Value("${kafka-out.error.topic.name}") private String kafkaOutErrorTopicName; @Value("${kafka-out.success.topic.name}") From fb99f534cc9bf84f43b7783a3111cc9240e3e459 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A0=D0=B0=D1=83=D1=84=20=D0=9B=D0=B0=D1=82=D1=8B=D0=BF?= =?UTF-8?q?=D0=BE=D0=B2?= Date: Mon, 29 Jul 2024 21:15:00 +0300 Subject: [PATCH 05/30] SUPPORT-8400: fix for review (2) --- file-upload/pom.xml => pom.xml | 4 ---- .../java/ru/micord}/FileUploadApplication.java | 2 +- .../main/java/ru/micord}/av/AvResponse.java | 2 +- .../ru/micord}/exception/FileUploadException.java | 2 +- .../exception/InvalidHttpFileUrlException.java | 2 +- .../config/input/InputKafkaConsumerConfig.java | 2 +- .../config/output/OutputKafkaProducerConfig.java | 2 +- .../config/output/OutputKafkaTopicConfig.java | 2 +- .../main/java/ru/micord}/kafka/dto/InMessage.java | 2 +- .../ru/micord}/kafka/dto/OutErrorMessage.java | 5 ++--- .../ru/micord}/service/FileUploadService.java | 15 +++++++-------- .../main/resources/application.properties | 0 12 files changed, 17 insertions(+), 23 deletions(-) rename file-upload/pom.xml => pom.xml (97%) rename {file-upload/src/main/java/ru/cg/subproject => src/main/java/ru/micord}/FileUploadApplication.java (92%) rename {file-upload/src/main/java/ru/cg/subproject => src/main/java/ru/micord}/av/AvResponse.java (93%) rename {file-upload/src/main/java/ru/cg/subproject => src/main/java/ru/micord}/exception/FileUploadException.java (86%) rename {file-upload/src/main/java/ru/cg/subproject => src/main/java/ru/micord}/exception/InvalidHttpFileUrlException.java (92%) rename {file-upload/src/main/java/ru/cg/subproject => src/main/java/ru/micord}/kafka/config/input/InputKafkaConsumerConfig.java (98%) rename {file-upload/src/main/java/ru/cg/subproject => src/main/java/ru/micord}/kafka/config/output/OutputKafkaProducerConfig.java (97%) rename {file-upload/src/main/java/ru/cg/subproject => src/main/java/ru/micord}/kafka/config/output/OutputKafkaTopicConfig.java (94%) rename {file-upload/src/main/java/ru/cg/subproject => src/main/java/ru/micord}/kafka/dto/InMessage.java (96%) rename {file-upload/src/main/java/ru/cg/subproject => src/main/java/ru/micord}/kafka/dto/OutErrorMessage.java (73%) rename {file-upload/src/main/java/ru/cg/subproject => src/main/java/ru/micord}/service/FileUploadService.java (94%) rename {file-upload/src => src}/main/resources/application.properties (100%) diff --git a/file-upload/pom.xml b/pom.xml similarity index 97% rename from file-upload/pom.xml rename to pom.xml index 816b840..a4e0805 100644 --- a/file-upload/pom.xml +++ b/pom.xml @@ -15,10 +15,6 @@ file-upload 0.0.1-SNAPSHOT - - 17 - - diff --git a/file-upload/src/main/java/ru/cg/subproject/FileUploadApplication.java b/src/main/java/ru/micord/FileUploadApplication.java similarity index 92% rename from file-upload/src/main/java/ru/cg/subproject/FileUploadApplication.java rename to src/main/java/ru/micord/FileUploadApplication.java index f9f4b06..5ee86d3 100644 --- a/file-upload/src/main/java/ru/cg/subproject/FileUploadApplication.java +++ b/src/main/java/ru/micord/FileUploadApplication.java @@ -1,4 +1,4 @@ -package ru.cg.subproject; +package ru.micord; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; diff --git a/file-upload/src/main/java/ru/cg/subproject/av/AvResponse.java b/src/main/java/ru/micord/av/AvResponse.java similarity index 93% rename from file-upload/src/main/java/ru/cg/subproject/av/AvResponse.java rename to src/main/java/ru/micord/av/AvResponse.java index 92a894f..d4f6829 100644 --- a/file-upload/src/main/java/ru/cg/subproject/av/AvResponse.java +++ b/src/main/java/ru/micord/av/AvResponse.java @@ -1,4 +1,4 @@ -package ru.cg.subproject.av; +package ru.micord.av; /** * @author r.latypov diff --git a/file-upload/src/main/java/ru/cg/subproject/exception/FileUploadException.java b/src/main/java/ru/micord/exception/FileUploadException.java similarity index 86% rename from file-upload/src/main/java/ru/cg/subproject/exception/FileUploadException.java rename to src/main/java/ru/micord/exception/FileUploadException.java index b11dea2..0d66769 100644 --- a/file-upload/src/main/java/ru/cg/subproject/exception/FileUploadException.java +++ b/src/main/java/ru/micord/exception/FileUploadException.java @@ -1,4 +1,4 @@ -package ru.cg.subproject.exception; +package ru.micord.exception; /** * @author r.latypov diff --git a/file-upload/src/main/java/ru/cg/subproject/exception/InvalidHttpFileUrlException.java b/src/main/java/ru/micord/exception/InvalidHttpFileUrlException.java similarity index 92% rename from file-upload/src/main/java/ru/cg/subproject/exception/InvalidHttpFileUrlException.java rename to src/main/java/ru/micord/exception/InvalidHttpFileUrlException.java index 9ff1f80..043ecce 100644 --- a/file-upload/src/main/java/ru/cg/subproject/exception/InvalidHttpFileUrlException.java +++ b/src/main/java/ru/micord/exception/InvalidHttpFileUrlException.java @@ -1,4 +1,4 @@ -package ru.cg.subproject.exception; +package ru.micord.exception; /** * @author r.latypov diff --git a/file-upload/src/main/java/ru/cg/subproject/kafka/config/input/InputKafkaConsumerConfig.java b/src/main/java/ru/micord/kafka/config/input/InputKafkaConsumerConfig.java similarity index 98% rename from file-upload/src/main/java/ru/cg/subproject/kafka/config/input/InputKafkaConsumerConfig.java rename to src/main/java/ru/micord/kafka/config/input/InputKafkaConsumerConfig.java index aa4def0..891a0a0 100644 --- a/file-upload/src/main/java/ru/cg/subproject/kafka/config/input/InputKafkaConsumerConfig.java +++ b/src/main/java/ru/micord/kafka/config/input/InputKafkaConsumerConfig.java @@ -1,4 +1,4 @@ -package ru.cg.subproject.kafka.config.input; +package ru.micord.kafka.config.input; import java.util.HashMap; import java.util.List; diff --git a/file-upload/src/main/java/ru/cg/subproject/kafka/config/output/OutputKafkaProducerConfig.java b/src/main/java/ru/micord/kafka/config/output/OutputKafkaProducerConfig.java similarity index 97% rename from file-upload/src/main/java/ru/cg/subproject/kafka/config/output/OutputKafkaProducerConfig.java rename to src/main/java/ru/micord/kafka/config/output/OutputKafkaProducerConfig.java index d20be6c..ef64cde 100644 --- a/file-upload/src/main/java/ru/cg/subproject/kafka/config/output/OutputKafkaProducerConfig.java +++ b/src/main/java/ru/micord/kafka/config/output/OutputKafkaProducerConfig.java @@ -1,4 +1,4 @@ -package ru.cg.subproject.kafka.config.output; +package ru.micord.kafka.config.output; import java.util.HashMap; import java.util.List; diff --git a/file-upload/src/main/java/ru/cg/subproject/kafka/config/output/OutputKafkaTopicConfig.java b/src/main/java/ru/micord/kafka/config/output/OutputKafkaTopicConfig.java similarity index 94% rename from file-upload/src/main/java/ru/cg/subproject/kafka/config/output/OutputKafkaTopicConfig.java rename to src/main/java/ru/micord/kafka/config/output/OutputKafkaTopicConfig.java index accae7f..ea6b95c 100644 --- a/file-upload/src/main/java/ru/cg/subproject/kafka/config/output/OutputKafkaTopicConfig.java +++ b/src/main/java/ru/micord/kafka/config/output/OutputKafkaTopicConfig.java @@ -1,4 +1,4 @@ -package ru.cg.subproject.kafka.config.output; +package ru.micord.kafka.config.output; import org.apache.kafka.clients.admin.NewTopic; import org.springframework.beans.factory.annotation.Value; diff --git a/file-upload/src/main/java/ru/cg/subproject/kafka/dto/InMessage.java b/src/main/java/ru/micord/kafka/dto/InMessage.java similarity index 96% rename from file-upload/src/main/java/ru/cg/subproject/kafka/dto/InMessage.java rename to src/main/java/ru/micord/kafka/dto/InMessage.java index 1b388e1..131bc67 100644 --- a/file-upload/src/main/java/ru/cg/subproject/kafka/dto/InMessage.java +++ b/src/main/java/ru/micord/kafka/dto/InMessage.java @@ -1,4 +1,4 @@ -package ru.cg.subproject.kafka.dto; +package ru.micord.kafka.dto; import lombok.AllArgsConstructor; import lombok.Getter; diff --git a/file-upload/src/main/java/ru/cg/subproject/kafka/dto/OutErrorMessage.java b/src/main/java/ru/micord/kafka/dto/OutErrorMessage.java similarity index 73% rename from file-upload/src/main/java/ru/cg/subproject/kafka/dto/OutErrorMessage.java rename to src/main/java/ru/micord/kafka/dto/OutErrorMessage.java index 0cba965..ee5641b 100644 --- a/file-upload/src/main/java/ru/cg/subproject/kafka/dto/OutErrorMessage.java +++ b/src/main/java/ru/micord/kafka/dto/OutErrorMessage.java @@ -1,8 +1,7 @@ -package ru.cg.subproject.kafka.dto; +package ru.micord.kafka.dto; import org.springframework.lang.NonNull; - -import ru.cg.subproject.av.AvResponse; +import ru.micord.av.AvResponse; /** * @author r.latypov diff --git a/file-upload/src/main/java/ru/cg/subproject/service/FileUploadService.java b/src/main/java/ru/micord/service/FileUploadService.java similarity index 94% rename from file-upload/src/main/java/ru/cg/subproject/service/FileUploadService.java rename to src/main/java/ru/micord/service/FileUploadService.java index 121aa90..05e7dc9 100644 --- a/file-upload/src/main/java/ru/cg/subproject/service/FileUploadService.java +++ b/src/main/java/ru/micord/service/FileUploadService.java @@ -1,4 +1,4 @@ -package ru.cg.subproject.service; +package ru.micord.service; import java.io.File; import java.io.FileOutputStream; @@ -24,18 +24,17 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; -import org.springframework.kafka.annotation.KafkaListener; import org.springframework.kafka.core.KafkaTemplate; import org.springframework.kafka.support.Acknowledgment; import org.springframework.kafka.support.SendResult; import org.springframework.lang.NonNull; import org.springframework.stereotype.Service; +import ru.micord.av.AvResponse; +import ru.micord.exception.InvalidHttpFileUrlException; +import ru.micord.kafka.dto.InMessage; +import ru.micord.kafka.dto.OutErrorMessage; -import ru.cg.subproject.av.AvResponse; -import ru.cg.subproject.exception.FileUploadException; -import ru.cg.subproject.exception.InvalidHttpFileUrlException; -import ru.cg.subproject.kafka.dto.InMessage; -import ru.cg.subproject.kafka.dto.OutErrorMessage; +import ru.micord.exception.FileUploadException; /** * @author r.latypov @@ -61,7 +60,7 @@ public class FileUploadService { this.outSuccessTopic = outSuccessTopic; } - @KafkaListener(id = "${spring.kafka.consumer.group-id}", topics = "${kafka-in.topic.name}") + //@KafkaListener(id = "${spring.kafka.consumer.group-id}", topics = "${kafka-in.topic.name}") public void listenKafkaIn(String kafkaInMessage, Acknowledgment acknowledgment) { InMessage inMessage = new Gson().fromJson(kafkaInMessage, InMessage.class); diff --git a/file-upload/src/main/resources/application.properties b/src/main/resources/application.properties similarity index 100% rename from file-upload/src/main/resources/application.properties rename to src/main/resources/application.properties From 3b8a978a05014f591d4f3fa00fa1d44ece42fb1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A0=D0=B0=D1=83=D1=84=20=D0=9B=D0=B0=D1=82=D1=8B=D0=BF?= =?UTF-8?q?=D0=BE=D0=B2?= Date: Wed, 31 Jul 2024 19:51:47 +0300 Subject: [PATCH 06/30] SAPPORT-8400: fix for review (3) --- .../{ => ervu/av}/FileUploadApplication.java | 2 +- .../av}/exception/FileUploadException.java | 2 +- .../InvalidHttpFileUrlException.java | 6 +- .../input/InputKafkaConsumerConfig.java | 2 +- .../output/OutputKafkaProducerConfig.java | 2 +- .../config/output/OutputKafkaTopicConfig.java | 2 +- .../{ => ervu/av}/kafka/dto/InMessage.java | 2 +- .../av}/kafka/dto/OutErrorMessage.java | 4 +- .../{av => ervu/av/response}/AvResponse.java | 2 +- .../av}/service/FileUploadService.java | 56 ++++++++++++------- src/main/resources/application.properties | 21 ++++--- 11 files changed, 60 insertions(+), 41 deletions(-) rename src/main/java/ru/micord/{ => ervu/av}/FileUploadApplication.java (92%) rename src/main/java/ru/micord/{ => ervu/av}/exception/FileUploadException.java (85%) rename src/main/java/ru/micord/{ => ervu/av}/exception/InvalidHttpFileUrlException.java (79%) rename src/main/java/ru/micord/{ => ervu/av}/kafka/config/input/InputKafkaConsumerConfig.java (98%) rename src/main/java/ru/micord/{ => ervu/av}/kafka/config/output/OutputKafkaProducerConfig.java (97%) rename src/main/java/ru/micord/{ => ervu/av}/kafka/config/output/OutputKafkaTopicConfig.java (94%) rename src/main/java/ru/micord/{ => ervu/av}/kafka/dto/InMessage.java (96%) rename src/main/java/ru/micord/{ => ervu/av}/kafka/dto/OutErrorMessage.java (71%) rename src/main/java/ru/micord/{av => ervu/av/response}/AvResponse.java (91%) rename src/main/java/ru/micord/{ => ervu/av}/service/FileUploadService.java (73%) diff --git a/src/main/java/ru/micord/FileUploadApplication.java b/src/main/java/ru/micord/ervu/av/FileUploadApplication.java similarity index 92% rename from src/main/java/ru/micord/FileUploadApplication.java rename to src/main/java/ru/micord/ervu/av/FileUploadApplication.java index 5ee86d3..017e664 100644 --- a/src/main/java/ru/micord/FileUploadApplication.java +++ b/src/main/java/ru/micord/ervu/av/FileUploadApplication.java @@ -1,4 +1,4 @@ -package ru.micord; +package ru.micord.ervu.av; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; diff --git a/src/main/java/ru/micord/exception/FileUploadException.java b/src/main/java/ru/micord/ervu/av/exception/FileUploadException.java similarity index 85% rename from src/main/java/ru/micord/exception/FileUploadException.java rename to src/main/java/ru/micord/ervu/av/exception/FileUploadException.java index 0d66769..79e7555 100644 --- a/src/main/java/ru/micord/exception/FileUploadException.java +++ b/src/main/java/ru/micord/ervu/av/exception/FileUploadException.java @@ -1,4 +1,4 @@ -package ru.micord.exception; +package ru.micord.ervu.av.exception; /** * @author r.latypov diff --git a/src/main/java/ru/micord/exception/InvalidHttpFileUrlException.java b/src/main/java/ru/micord/ervu/av/exception/InvalidHttpFileUrlException.java similarity index 79% rename from src/main/java/ru/micord/exception/InvalidHttpFileUrlException.java rename to src/main/java/ru/micord/ervu/av/exception/InvalidHttpFileUrlException.java index 043ecce..aaa4a81 100644 --- a/src/main/java/ru/micord/exception/InvalidHttpFileUrlException.java +++ b/src/main/java/ru/micord/ervu/av/exception/InvalidHttpFileUrlException.java @@ -1,4 +1,4 @@ -package ru.micord.exception; +package ru.micord.ervu.av.exception; /** * @author r.latypov @@ -6,10 +6,6 @@ package ru.micord.exception; public class InvalidHttpFileUrlException extends Exception { private static final String MESSAGE = "file url is not valid"; - public InvalidHttpFileUrlException() { - super(MESSAGE); - } - public InvalidHttpFileUrlException(String message) { super(String.join(" : ", MESSAGE, message)); } diff --git a/src/main/java/ru/micord/kafka/config/input/InputKafkaConsumerConfig.java b/src/main/java/ru/micord/ervu/av/kafka/config/input/InputKafkaConsumerConfig.java similarity index 98% rename from src/main/java/ru/micord/kafka/config/input/InputKafkaConsumerConfig.java rename to src/main/java/ru/micord/ervu/av/kafka/config/input/InputKafkaConsumerConfig.java index 891a0a0..672a8f3 100644 --- a/src/main/java/ru/micord/kafka/config/input/InputKafkaConsumerConfig.java +++ b/src/main/java/ru/micord/ervu/av/kafka/config/input/InputKafkaConsumerConfig.java @@ -1,4 +1,4 @@ -package ru.micord.kafka.config.input; +package ru.micord.ervu.av.kafka.config.input; import java.util.HashMap; import java.util.List; diff --git a/src/main/java/ru/micord/kafka/config/output/OutputKafkaProducerConfig.java b/src/main/java/ru/micord/ervu/av/kafka/config/output/OutputKafkaProducerConfig.java similarity index 97% rename from src/main/java/ru/micord/kafka/config/output/OutputKafkaProducerConfig.java rename to src/main/java/ru/micord/ervu/av/kafka/config/output/OutputKafkaProducerConfig.java index ef64cde..340fa3d 100644 --- a/src/main/java/ru/micord/kafka/config/output/OutputKafkaProducerConfig.java +++ b/src/main/java/ru/micord/ervu/av/kafka/config/output/OutputKafkaProducerConfig.java @@ -1,4 +1,4 @@ -package ru.micord.kafka.config.output; +package ru.micord.ervu.av.kafka.config.output; import java.util.HashMap; import java.util.List; diff --git a/src/main/java/ru/micord/kafka/config/output/OutputKafkaTopicConfig.java b/src/main/java/ru/micord/ervu/av/kafka/config/output/OutputKafkaTopicConfig.java similarity index 94% rename from src/main/java/ru/micord/kafka/config/output/OutputKafkaTopicConfig.java rename to src/main/java/ru/micord/ervu/av/kafka/config/output/OutputKafkaTopicConfig.java index ea6b95c..8fb5ef9 100644 --- a/src/main/java/ru/micord/kafka/config/output/OutputKafkaTopicConfig.java +++ b/src/main/java/ru/micord/ervu/av/kafka/config/output/OutputKafkaTopicConfig.java @@ -1,4 +1,4 @@ -package ru.micord.kafka.config.output; +package ru.micord.ervu.av.kafka.config.output; import org.apache.kafka.clients.admin.NewTopic; import org.springframework.beans.factory.annotation.Value; diff --git a/src/main/java/ru/micord/kafka/dto/InMessage.java b/src/main/java/ru/micord/ervu/av/kafka/dto/InMessage.java similarity index 96% rename from src/main/java/ru/micord/kafka/dto/InMessage.java rename to src/main/java/ru/micord/ervu/av/kafka/dto/InMessage.java index 131bc67..98f997e 100644 --- a/src/main/java/ru/micord/kafka/dto/InMessage.java +++ b/src/main/java/ru/micord/ervu/av/kafka/dto/InMessage.java @@ -1,4 +1,4 @@ -package ru.micord.kafka.dto; +package ru.micord.ervu.av.kafka.dto; import lombok.AllArgsConstructor; import lombok.Getter; diff --git a/src/main/java/ru/micord/kafka/dto/OutErrorMessage.java b/src/main/java/ru/micord/ervu/av/kafka/dto/OutErrorMessage.java similarity index 71% rename from src/main/java/ru/micord/kafka/dto/OutErrorMessage.java rename to src/main/java/ru/micord/ervu/av/kafka/dto/OutErrorMessage.java index ee5641b..eca7a3a 100644 --- a/src/main/java/ru/micord/kafka/dto/OutErrorMessage.java +++ b/src/main/java/ru/micord/ervu/av/kafka/dto/OutErrorMessage.java @@ -1,7 +1,7 @@ -package ru.micord.kafka.dto; +package ru.micord.ervu.av.kafka.dto; import org.springframework.lang.NonNull; -import ru.micord.av.AvResponse; +import ru.micord.ervu.av.response.AvResponse; /** * @author r.latypov diff --git a/src/main/java/ru/micord/av/AvResponse.java b/src/main/java/ru/micord/ervu/av/response/AvResponse.java similarity index 91% rename from src/main/java/ru/micord/av/AvResponse.java rename to src/main/java/ru/micord/ervu/av/response/AvResponse.java index d4f6829..8b74329 100644 --- a/src/main/java/ru/micord/av/AvResponse.java +++ b/src/main/java/ru/micord/ervu/av/response/AvResponse.java @@ -1,4 +1,4 @@ -package ru.micord.av; +package ru.micord.ervu.av.response; /** * @author r.latypov diff --git a/src/main/java/ru/micord/service/FileUploadService.java b/src/main/java/ru/micord/ervu/av/service/FileUploadService.java similarity index 73% rename from src/main/java/ru/micord/service/FileUploadService.java rename to src/main/java/ru/micord/ervu/av/service/FileUploadService.java index 05e7dc9..ca1ca59 100644 --- a/src/main/java/ru/micord/service/FileUploadService.java +++ b/src/main/java/ru/micord/ervu/av/service/FileUploadService.java @@ -1,9 +1,10 @@ -package ru.micord.service; +package ru.micord.ervu.av.service; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.util.Arrays; +import java.util.UUID; import java.util.concurrent.CompletableFuture; import com.google.gson.Gson; @@ -24,17 +25,17 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; +import org.springframework.kafka.annotation.KafkaListener; import org.springframework.kafka.core.KafkaTemplate; import org.springframework.kafka.support.Acknowledgment; import org.springframework.kafka.support.SendResult; import org.springframework.lang.NonNull; import org.springframework.stereotype.Service; -import ru.micord.av.AvResponse; -import ru.micord.exception.InvalidHttpFileUrlException; -import ru.micord.kafka.dto.InMessage; -import ru.micord.kafka.dto.OutErrorMessage; - -import ru.micord.exception.FileUploadException; +import ru.micord.ervu.av.exception.FileUploadException; +import ru.micord.ervu.av.exception.InvalidHttpFileUrlException; +import ru.micord.ervu.av.kafka.dto.InMessage; +import ru.micord.ervu.av.kafka.dto.OutErrorMessage; +import ru.micord.ervu.av.response.AvResponse; /** * @author r.latypov @@ -60,7 +61,7 @@ public class FileUploadService { this.outSuccessTopic = outSuccessTopic; } - //@KafkaListener(id = "${spring.kafka.consumer.group-id}", topics = "${kafka-in.topic.name}") + @KafkaListener(id = "${spring.kafka.consumer.group-id}", topics = "${kafka-in.topic.name}") public void listenKafkaIn(String kafkaInMessage, Acknowledgment acknowledgment) { InMessage inMessage = new Gson().fromJson(kafkaInMessage, InMessage.class); @@ -68,48 +69,56 @@ public class FileUploadService { FileUrl fileUrl = parseFileUrl(inMessage.fileInfo().getFileNameBase()); String filePath = fileSavingPath + fileUrl.fileName(); String downloadUrl = fileUrl.fileUrl(); - downloadFileByHttp(downloadUrl, filePath); + downloadFile(downloadUrl, filePath); - AvResponse avResponse = sendFileToAvScan(filePath); + AvResponse avResponse = checkFile(filePath); boolean infected = Arrays.stream(avResponse.verdicts()) .anyMatch(verdict -> verdict.equalsIgnoreCase("infected")); if (infected) { - sendKafkaMessage(outErrorTopic.name(), + sendMessage(outErrorTopic.name(), new OutErrorMessage("file is infected", avResponse, inMessage) ); } else { String uploadUrl = httpFileServerOutAddress + "/" + fileUrl.fileName(); - uploadFileByHttp(filePath, uploadUrl); + uploadFile(filePath, uploadUrl); inMessage.fileInfo().setFileNameBase(uploadUrl); - sendKafkaMessage(outSuccessTopic.name(), inMessage); + sendMessage(outSuccessTopic.name(), inMessage); } - deleteFileByHttp(downloadUrl); + deleteFile(downloadUrl); if (new File(filePath).delete()) { acknowledgment.acknowledge(); } } catch (InvalidHttpFileUrlException e) { + // считаем, что повторная обработка сообщения не нужна + // ошибку логируем, сообщаем об ошибке, помечаем прочтение сообщения logger.error(e.getMessage() + ": " + kafkaInMessage); - sendKafkaMessage(outErrorTopic.name(), + sendMessage(outErrorTopic.name(), new OutErrorMessage(e.getMessage(), null, inMessage) ); acknowledgment.acknowledge(); } catch (FileUploadException e) { + // считаем, что нужно повторное считывание сообщения + // ошибку логируем, сообщение оставляем непрочитанным logger.error(e.getMessage(), e); } } + /* метод для выделения UUID файла из ссылки на файл + сохраняем на диске и отправляем файл в хранилище под тем же UUID, сохраняя расширение файла + */ private static FileUrl parseFileUrl(@NonNull String fileUrl) throws InvalidHttpFileUrlException { String[] substrings = fileUrl.split("/"); String fileName = substrings[substrings.length - 1]; if (substrings.length == 1 || fileName.isBlank()) { + // ошибка данных; сообщение некорректно throw new InvalidHttpFileUrlException(fileUrl); } else { @@ -117,7 +126,7 @@ public class FileUploadService { } } - private void downloadFileByHttp(String fileUrl, String filePath) + private void downloadFile(String fileUrl, String filePath) throws InvalidHttpFileUrlException, FileUploadException { File file = new File(filePath); HttpGet request = new HttpGet(fileUrl); @@ -135,6 +144,7 @@ public class FileUploadService { } } else { + // в хранилище не обнаружено файла; сообщение некорректно String message = "http status code " + statusCode + " : " + fileUrl; throw new InvalidHttpFileUrlException(message); } @@ -143,13 +153,14 @@ public class FileUploadService { throw new FileUploadException(e); } catch (IOException e) { + // хранилище недоступно; сообщение некорректно String message = (e.getMessage() == null ? e.getCause().getMessage() : e.getMessage()) + " : " + fileUrl; throw new InvalidHttpFileUrlException(message, e); } } - private AvResponse sendFileToAvScan(String filePath) throws FileUploadException { + private AvResponse checkFile(String filePath) throws FileUploadException { File file = new File(filePath); try (CloseableHttpClient client = HttpClients.createDefault()) { @@ -170,11 +181,12 @@ public class FileUploadService { } } catch (IOException e) { + // непредусмотренная ошибка доступа через http-клиент throw new FileUploadException(e); } } - private void uploadFileByHttp(String filePath, String uploadUrl) throws FileUploadException { + private void uploadFile(String filePath, String uploadUrl) throws FileUploadException { File file = new File(filePath); try (CloseableHttpClient client = HttpClients.createDefault()) { @@ -187,6 +199,8 @@ public class FileUploadService { try (CloseableHttpResponse response = client.execute(put)) { int statusCode = response.getStatusLine().getStatusCode(); if (statusCode == 204) { + // считаем, что происходит повторная обработка + // этот же файл доставлен при предыдущей обработке и не удален из входного хранилища logger.warn("file already exists : " + uploadUrl); } else if (statusCode != 201) { @@ -196,11 +210,12 @@ public class FileUploadService { } } catch (IOException e) { + // непредусмотренная ошибка доступа через http-клиент throw new FileUploadException(e); } } - private void deleteFileByHttp(String fileUrl) throws FileUploadException { + private void deleteFile(String fileUrl) throws FileUploadException { try (CloseableHttpClient client = HttpClients.createDefault()) { HttpDelete delete = new HttpDelete(fileUrl); @@ -213,11 +228,12 @@ public class FileUploadService { } } catch (IOException e) { + // непредусмотренная ошибка доступа через http-клиент throw new FileUploadException(e); } } - private void sendKafkaMessage(@NonNull String topicName, Object object) { + private void sendMessage(@NonNull String topicName, Object object) { CompletableFuture> future = kafkaTemplate.send(topicName, new GsonBuilder().setPrettyPrinting().create().toJson(object) ); diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index f279a02..265a55b 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -1,12 +1,16 @@ spring.kafka.admin.security.protocol=SASL_PLAINTEXT -spring.kafka.admin.properties.sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required username="user1" password="Blfi9d2OFG"; +#login password to set +spring.kafka.admin.properties.sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required username="" password=""; spring.kafka.admin.properties.sasl.mechanism=SCRAM-SHA-256 # -spring.kafka.bootstrap-servers=10.10.31.11:32609 +#host1:port1, host2:port2 +spring.kafka.bootstrap-servers= # -spring.kafka.consumer.bootstrap-servers=10.10.31.11:32609 +#host1:port1, host2:port2 +spring.kafka.consumer.bootstrap-servers= spring.kafka.consumer.security.protocol=SASL_PLAINTEXT -spring.kafka.consumer.properties.sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required username="user1" password="Blfi9d2OFG"; +#login password to set +spring.kafka.consumer.properties.sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required username="" password=""; spring.kafka.consumer.properties.sasl.mechanism=SCRAM-SHA-256 # spring.kafka.consumer.enable-auto-commit=false @@ -14,13 +18,16 @@ spring.kafka.consumer.group-id=file-to-upload-consumers # spring.kafka.listener.ack-mode=MANUAL_IMMEDIATE # -spring.kafka.producer.bootstrap-servers=10.10.31.11:32609 +#host1:port1, host2:port2 +spring.kafka.producer.bootstrap-servers= spring.kafka.producer.security.protocol=SASL_PLAINTEXT -spring.kafka.producer.properties.sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required username="user1" password="Blfi9d2OFG"; +#login password to set +spring.kafka.producer.properties.sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required username="" password=""; spring.kafka.producer.properties.sasl.mechanism=SCRAM-SHA-256 # spring.kafka.properties.security.protocol=SASL_PLAINTEXT -spring.kafka.properties.sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required username="user1" password="Blfi9d2OFG"; +#login password to set +spring.kafka.properties.sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required username="" password=""; spring.kafka.properties.sasl.mechanism=SCRAM-SHA-256 # kafka-in.topic.name=file-to-upload From d6e0c353ddf87290b8009142e05dc0ddb2e36b72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A5=D0=B0=D0=BB=D1=82=D0=BE=D0=B1=D0=B8=D0=BD=20=D0=95?= =?UTF-8?q?=D0=B2=D0=B3=D0=B5=D0=BD=D0=B8=D0=B9?= Date: Fri, 2 Aug 2024 14:57:41 +0300 Subject: [PATCH 07/30] added Dockerfile --- Dockerfile | 4 ++++ docker-compose.yaml | 13 ------------- 2 files changed, 4 insertions(+), 13 deletions(-) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..0aaea2c --- /dev/null +++ b/Dockerfile @@ -0,0 +1,4 @@ +FROM bellsoft/liberica-openjdk-alpine:17-cds +COPY target/*.jar app.jar + +CMD ["java", "-jar", "app.jar"] \ No newline at end of file diff --git a/docker-compose.yaml b/docker-compose.yaml index e2a2b70..20e91ad 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -8,19 +8,6 @@ services: networks: - lkrp_av - minio: - image: minio/minio - restart: always - env_file: - - test.env - volumes: - - minio_data:/data - ports: - - '${MINIO_HOST_PORT}:9000' - command: server /data - networks: - - lkrp_av - networks: lkrp_av: From 3928f6fd3ee1c678e518a5f9bf189a155a1432a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A0=D0=B0=D1=83=D1=84=20=D0=9B=D0=B0=D1=82=D1=8B=D0=BF?= =?UTF-8?q?=D0=BE=D0=B2?= Date: Mon, 5 Aug 2024 11:17:48 +0300 Subject: [PATCH 08/30] SUPPORT-8400: fix for review (4) --- pom.xml | 5 +++++ .../ru/micord/ervu/av/service/FileUploadService.java | 11 ++++++----- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/pom.xml b/pom.xml index a4e0805..59c1f1a 100644 --- a/pom.xml +++ b/pom.xml @@ -69,6 +69,11 @@ lombok + + org.springframework + spring-web + + org.springframework.boot spring-boot-starter diff --git a/src/main/java/ru/micord/ervu/av/service/FileUploadService.java b/src/main/java/ru/micord/ervu/av/service/FileUploadService.java index ca1ca59..53bbc0b 100644 --- a/src/main/java/ru/micord/ervu/av/service/FileUploadService.java +++ b/src/main/java/ru/micord/ervu/av/service/FileUploadService.java @@ -25,6 +25,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; +import org.springframework.http.HttpStatus; import org.springframework.kafka.annotation.KafkaListener; import org.springframework.kafka.core.KafkaTemplate; import org.springframework.kafka.support.Acknowledgment; @@ -135,7 +136,7 @@ public class FileUploadService { CloseableHttpResponse response = client.execute(request)) { int statusCode = response.getStatusLine().getStatusCode(); - if (statusCode == 200) { + if (statusCode == HttpStatus.OK.value()) { HttpEntity entity = response.getEntity(); if (entity != null) { try (FileOutputStream outputStream = new FileOutputStream(file)) { @@ -173,7 +174,7 @@ public class FileUploadService { try (CloseableHttpResponse response = client.execute(post)) { int statusCode = response.getStatusLine().getStatusCode(); - if (statusCode != 200) { + if (statusCode != HttpStatus.OK.value()) { String message = "http status code " + statusCode + " for " + avRestAddress + " request"; throw new FileUploadException(message); } @@ -198,12 +199,12 @@ public class FileUploadService { try (CloseableHttpResponse response = client.execute(put)) { int statusCode = response.getStatusLine().getStatusCode(); - if (statusCode == 204) { + if (statusCode == HttpStatus.NO_CONTENT.value()) { // считаем, что происходит повторная обработка // этот же файл доставлен при предыдущей обработке и не удален из входного хранилища logger.warn("file already exists : " + uploadUrl); } - else if (statusCode != 201) { + else if (statusCode != HttpStatus.CREATED.value()) { String message = "http status code " + statusCode + " : " + uploadUrl; throw new RuntimeException(message); } @@ -221,7 +222,7 @@ public class FileUploadService { try (CloseableHttpResponse response = client.execute(delete)) { int statusCode = response.getStatusLine().getStatusCode(); - if (statusCode != 204) { + if (statusCode != HttpStatus.NO_CONTENT.value()) { String message = "http status code " + statusCode + " : " + fileUrl; throw new RuntimeException(message); } From c57929b00214715145469e8596422079347374d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A0=D0=B0=D1=83=D1=84=20=D0=9B=D0=B0=D1=82=D1=8B=D0=BF?= =?UTF-8?q?=D0=BE=D0=B2?= Date: Mon, 5 Aug 2024 11:50:31 +0300 Subject: [PATCH 09/30] SUPPORT-8400: fix for review (4.1) --- pom.xml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/pom.xml b/pom.xml index 59c1f1a..8ec923b 100644 --- a/pom.xml +++ b/pom.xml @@ -40,12 +40,6 @@ 1.18.34 provided - - - org.springframework.kafka - spring-kafka - 3.2.1 - From 44b4231a2a4c272d4b0ef67607d421b57d4af1a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A0=D0=B0=D1=83=D1=84=20=D0=9B=D0=B0=D1=82=D1=8B=D0=BF?= =?UTF-8?q?=D0=BE=D0=B2?= Date: Wed, 21 Aug 2024 13:01:22 +0300 Subject: [PATCH 10/30] =?UTF-8?q?=D0=BF=D0=BE=D0=B4=D1=81=D1=82=D0=B0?= =?UTF-8?q?=D0=BD=D0=BE=D0=B2=D0=BA=D0=B0=20=D0=BF=D0=B5=D1=80=D0=B5=D0=BC?= =?UTF-8?q?=D0=B5=D0=BD=D0=BD=D1=8B=D1=85=20=D1=81=D1=80=D0=B5=D0=B4=D1=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/main/resources/application.properties | 26 +++++++++++------------ 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 265a55b..e2cf0ef 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -1,16 +1,16 @@ spring.kafka.admin.security.protocol=SASL_PLAINTEXT #login password to set -spring.kafka.admin.properties.sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required username="" password=""; +spring.kafka.admin.properties.sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required username="${OUT_KAFKA_USERNAME}" password="${OUT_KAFKA_PASSWORD}"; spring.kafka.admin.properties.sasl.mechanism=SCRAM-SHA-256 # #host1:port1, host2:port2 -spring.kafka.bootstrap-servers= +spring.kafka.bootstrap-servers=${OUT_KAFKA_SERVERS} # #host1:port1, host2:port2 -spring.kafka.consumer.bootstrap-servers= +spring.kafka.consumer.bootstrap-servers=${IN_KAFKA_SERVERS} spring.kafka.consumer.security.protocol=SASL_PLAINTEXT #login password to set -spring.kafka.consumer.properties.sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required username="" password=""; +spring.kafka.consumer.properties.sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required username="${IN_KAFKA_USERNAME}" password="${IN_KAFKA_PASSWORD}"; spring.kafka.consumer.properties.sasl.mechanism=SCRAM-SHA-256 # spring.kafka.consumer.enable-auto-commit=false @@ -19,21 +19,21 @@ spring.kafka.consumer.group-id=file-to-upload-consumers spring.kafka.listener.ack-mode=MANUAL_IMMEDIATE # #host1:port1, host2:port2 -spring.kafka.producer.bootstrap-servers= +spring.kafka.producer.bootstrap-servers=${OUT_KAFKA_SERVERS} spring.kafka.producer.security.protocol=SASL_PLAINTEXT #login password to set -spring.kafka.producer.properties.sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required username="" password=""; +spring.kafka.producer.properties.sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required username="${OUT_KAFKA_USERNAME}" password="${OUT_KAFKA_PASSWORD}"; spring.kafka.producer.properties.sasl.mechanism=SCRAM-SHA-256 # spring.kafka.properties.security.protocol=SASL_PLAINTEXT #login password to set -spring.kafka.properties.sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required username="" password=""; +spring.kafka.properties.sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required username="${OUT_KAFKA_USERNAME}" password="${OUT_KAFKA_PASSWORD}"; spring.kafka.properties.sasl.mechanism=SCRAM-SHA-256 # -kafka-in.topic.name=file-to-upload -kafka-out.error.topic.name=error -kafka-out.success.topic.name=success +kafka-in.topic.name=${IN_KAFKA_TOPIC_NAME} +kafka-out.error.topic.name=${OUT_KAFKA_ERROR_TOPIC_NAME} +kafka-out.success.topic.name=${OUT_KAFKA_SUCCESS_TOPIC_NAME} # -av.rest.address=http://:/scans -file.saving.path=/nginx/transfer/ -http.file.server.out.address=http://localhost/out +av.rest.address=${AV_REST_ADDRESS} +file.saving.path=/transfer/ +http.file.server.out.address=${HTTP_FILE_SERVER_OUT_ADDRESS} From 0dddcef837df67c85d81d10e4fc8c6892f76e2c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A0=D0=B0=D1=83=D1=84=20=D0=9B=D0=B0=D1=82=D1=8B=D0=BF?= =?UTF-8?q?=D0=BE=D0=B2?= Date: Wed, 21 Aug 2024 13:12:52 +0300 Subject: [PATCH 11/30] =?UTF-8?q?=D0=B7=D0=B0=D0=B3=D0=BB=D1=83=D1=88?= =?UTF-8?q?=D0=BA=D0=B0=20=D0=BD=D0=B0=20=D0=B0=D0=BD=D1=82=D0=B8=D0=B2?= =?UTF-8?q?=D0=B8=D1=80=D1=83=D1=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/ru/micord/ervu/av/service/FileUploadService.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/main/java/ru/micord/ervu/av/service/FileUploadService.java b/src/main/java/ru/micord/ervu/av/service/FileUploadService.java index 53bbc0b..416f0a4 100644 --- a/src/main/java/ru/micord/ervu/av/service/FileUploadService.java +++ b/src/main/java/ru/micord/ervu/av/service/FileUploadService.java @@ -72,15 +72,20 @@ public class FileUploadService { String downloadUrl = fileUrl.fileUrl(); downloadFile(downloadUrl, filePath); + // todo активировать код в комментариях и отладить запросы к антивирусу SUPPORT-8507 +/* AvResponse avResponse = checkFile(filePath); boolean infected = Arrays.stream(avResponse.verdicts()) .anyMatch(verdict -> verdict.equalsIgnoreCase("infected")); - +*/ + boolean infected = false; if (infected) { +/* sendMessage(outErrorTopic.name(), new OutErrorMessage("file is infected", avResponse, inMessage) ); +*/ } else { String uploadUrl = httpFileServerOutAddress + "/" + fileUrl.fileName(); From c620d6a0ad4ed201478f08a71d369d44af14392e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A0=D0=B0=D1=83=D1=84=20=D0=9B=D0=B0=D1=82=D1=8B=D0=BF?= =?UTF-8?q?=D0=BE=D0=B2?= Date: Wed, 4 Sep 2024 16:02:03 +0300 Subject: [PATCH 12/30] SUPPORT-8507: adding response treatment --- .../av/exception/FileUploadException.java | 4 ++ .../micord/ervu/av/response/AvResponse.java | 15 ++-- .../ervu/av/response/AvScanResponse.java | 11 +++ .../ervu/av/service/FileUploadService.java | 70 ++++++++++++++++--- 4 files changed, 86 insertions(+), 14 deletions(-) create mode 100644 src/main/java/ru/micord/ervu/av/response/AvScanResponse.java diff --git a/src/main/java/ru/micord/ervu/av/exception/FileUploadException.java b/src/main/java/ru/micord/ervu/av/exception/FileUploadException.java index 79e7555..5c2b06a 100644 --- a/src/main/java/ru/micord/ervu/av/exception/FileUploadException.java +++ b/src/main/java/ru/micord/ervu/av/exception/FileUploadException.java @@ -11,4 +11,8 @@ public class FileUploadException extends Exception { public FileUploadException(Throwable cause) { super(cause); } + + public FileUploadException(String message, Throwable cause) { + super(message, cause); + } } diff --git a/src/main/java/ru/micord/ervu/av/response/AvResponse.java b/src/main/java/ru/micord/ervu/av/response/AvResponse.java index 8b74329..48e71ae 100644 --- a/src/main/java/ru/micord/ervu/av/response/AvResponse.java +++ b/src/main/java/ru/micord/ervu/av/response/AvResponse.java @@ -1,14 +1,17 @@ package ru.micord.ervu.av.response; +import java.util.Map; + /** * @author r.latypov */ -public record AvResponse(String completed, String created, String progress, ScanResult scan_result, - String status, String[] verdicts) { - public record ScanResult(Scan noname) { - public record Scan(String started, String stopped, Threat[] threats, String verdict) { - public record Threat(String name, String object) { - } +public record AvResponse(String completed, String created, Integer progress, + Map scan_result, String status, String[] verdicts) { + public record Scan(String started, String stopped, Threat[] threats, String verdict) { + public static final String VERDICT_CLEAN = "clean"; + public static final String VERDICT_INFECTED = "infected"; + + public record Threat(String name, String object) { } } } diff --git a/src/main/java/ru/micord/ervu/av/response/AvScanResponse.java b/src/main/java/ru/micord/ervu/av/response/AvScanResponse.java new file mode 100644 index 0000000..35616f0 --- /dev/null +++ b/src/main/java/ru/micord/ervu/av/response/AvScanResponse.java @@ -0,0 +1,11 @@ +package ru.micord.ervu.av.response; + +/** + * @author r.latypov + */ +public record AvScanResponse(String id, String location, Error error, String status) { + public static final String STATUS_ERROR = "error"; + + public record Error(String code, String message) { + } +} diff --git a/src/main/java/ru/micord/ervu/av/service/FileUploadService.java b/src/main/java/ru/micord/ervu/av/service/FileUploadService.java index 416f0a4..5013438 100644 --- a/src/main/java/ru/micord/ervu/av/service/FileUploadService.java +++ b/src/main/java/ru/micord/ervu/av/service/FileUploadService.java @@ -4,11 +4,12 @@ import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.util.Arrays; -import java.util.UUID; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; import com.google.gson.Gson; import com.google.gson.GsonBuilder; +import com.google.gson.JsonSyntaxException; import org.apache.http.HttpEntity; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpDelete; @@ -20,6 +21,7 @@ import org.apache.http.entity.mime.MultipartEntityBuilder; import org.apache.http.entity.mime.content.FileBody; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; +import org.apache.http.util.EntityUtils; import org.apache.kafka.clients.admin.NewTopic; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -37,6 +39,7 @@ import ru.micord.ervu.av.exception.InvalidHttpFileUrlException; import ru.micord.ervu.av.kafka.dto.InMessage; import ru.micord.ervu.av.kafka.dto.OutErrorMessage; import ru.micord.ervu.av.response.AvResponse; +import ru.micord.ervu.av.response.AvScanResponse; /** * @author r.latypov @@ -171,19 +174,70 @@ public class FileUploadService { try (CloseableHttpClient client = HttpClients.createDefault()) { HttpPost post = new HttpPost(avRestAddress); - post.addHeader("Content-type", "application/json"); HttpEntity entity = MultipartEntityBuilder.create() .addPart("file", new FileBody(file)) .build(); post.setEntity(entity); - try (CloseableHttpResponse response = client.execute(post)) { - int statusCode = response.getStatusLine().getStatusCode(); - if (statusCode != HttpStatus.OK.value()) { - String message = "http status code " + statusCode + " for " + avRestAddress + " request"; - throw new FileUploadException(message); + try (CloseableHttpResponse postResponse = client.execute(post)) { + int postStatusCode = postResponse.getStatusLine().getStatusCode(); + String postResponseJson = EntityUtils.toString(postResponse.getEntity()); + + AvScanResponse avScanResponse = null; + try { + avScanResponse = new Gson().fromJson(postResponseJson, AvScanResponse.class); } - return new Gson().fromJson(response.getEntity().toString(), AvResponse.class); + catch (JsonSyntaxException e) { + throw new FileUploadException("error json: " + postResponseJson, e); + } + + if (postStatusCode != HttpStatus.CREATED.value()) { + StringBuilder stringBuilder = new StringBuilder( + "http status code " + postStatusCode + " for " + avRestAddress + " post request."); + + String status = avScanResponse.status(); + if (status != null) { + stringBuilder.append(" Status: ").append(status).append("."); + } + if (avScanResponse.error() != null) { + stringBuilder.append(" Error code: ").append(avScanResponse.error().code()) + .append(". Error message: ").append(avScanResponse.error().message()).append(". "); + } + throw new FileUploadException(stringBuilder.toString()); + } + + String id = avScanResponse.id(); + String reportRequestUri = avRestAddress + "/" + id; + HttpGet get = new HttpGet(reportRequestUri); + + int countdown = 10; + long timeout = 1L; + AvResponse avResponse = null; + + do { + try { + TimeUnit.SECONDS.sleep(timeout); + } + catch (InterruptedException e) { + throw new FileUploadException(e); + } + + try (CloseableHttpResponse getResponse = client.execute(get)) { + int getStatusCode = getResponse.getStatusLine().getStatusCode(); + if (getStatusCode == HttpStatus.OK.value()) { + String getResponseJson = EntityUtils.toString(getResponse.getEntity()); + avResponse = new Gson().fromJson(getResponseJson, AvResponse.class); + } + else { + throw new FileUploadException("http status code " + getStatusCode + " for " + + reportRequestUri + " get request."); + } + } + countdown--; + } + while (countdown > 0 && (avResponse != null && avResponse.completed() == null)); + + return avResponse; } } catch (IOException e) { From d09a96e24d8e8ce04eb9208b286e0b6c18563418 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A0=D0=B0=D1=83=D1=84=20=D0=9B=D0=B0=D1=82=D1=8B=D0=BF?= =?UTF-8?q?=D0=BE=D0=B2?= Date: Fri, 6 Sep 2024 00:27:21 +0300 Subject: [PATCH 13/30] SUPPORT-8507: refactoring --- ...nResponse.java => AvFileSendResponse.java} | 2 +- .../ervu/av/service/FileUploadService.java | 31 +++++++++---------- 2 files changed, 15 insertions(+), 18 deletions(-) rename src/main/java/ru/micord/ervu/av/response/{AvScanResponse.java => AvFileSendResponse.java} (66%) diff --git a/src/main/java/ru/micord/ervu/av/response/AvScanResponse.java b/src/main/java/ru/micord/ervu/av/response/AvFileSendResponse.java similarity index 66% rename from src/main/java/ru/micord/ervu/av/response/AvScanResponse.java rename to src/main/java/ru/micord/ervu/av/response/AvFileSendResponse.java index 35616f0..ea8dd00 100644 --- a/src/main/java/ru/micord/ervu/av/response/AvScanResponse.java +++ b/src/main/java/ru/micord/ervu/av/response/AvFileSendResponse.java @@ -3,7 +3,7 @@ package ru.micord.ervu.av.response; /** * @author r.latypov */ -public record AvScanResponse(String id, String location, Error error, String status) { +public record AvFileSendResponse(String id, String location, Error error, String status) { public static final String STATUS_ERROR = "error"; public record Error(String code, String message) { diff --git a/src/main/java/ru/micord/ervu/av/service/FileUploadService.java b/src/main/java/ru/micord/ervu/av/service/FileUploadService.java index 5013438..e99b172 100644 --- a/src/main/java/ru/micord/ervu/av/service/FileUploadService.java +++ b/src/main/java/ru/micord/ervu/av/service/FileUploadService.java @@ -39,7 +39,7 @@ import ru.micord.ervu.av.exception.InvalidHttpFileUrlException; import ru.micord.ervu.av.kafka.dto.InMessage; import ru.micord.ervu.av.kafka.dto.OutErrorMessage; import ru.micord.ervu.av.response.AvResponse; -import ru.micord.ervu.av.response.AvScanResponse; +import ru.micord.ervu.av.response.AvFileSendResponse; /** * @author r.latypov @@ -75,20 +75,17 @@ public class FileUploadService { String downloadUrl = fileUrl.fileUrl(); downloadFile(downloadUrl, filePath); - // todo активировать код в комментариях и отладить запросы к антивирусу SUPPORT-8507 -/* AvResponse avResponse = checkFile(filePath); + boolean clean = Arrays.stream(avResponse.verdicts()) + .anyMatch(verdict -> verdict.equalsIgnoreCase(AvResponse.Scan.VERDICT_CLEAN)); boolean infected = Arrays.stream(avResponse.verdicts()) - .anyMatch(verdict -> verdict.equalsIgnoreCase("infected")); -*/ - boolean infected = false; - if (infected) { -/* + .anyMatch(verdict -> verdict.equalsIgnoreCase(AvResponse.Scan.VERDICT_INFECTED)); + + if (infected || !clean) { sendMessage(outErrorTopic.name(), new OutErrorMessage("file is infected", avResponse, inMessage) ); -*/ } else { String uploadUrl = httpFileServerOutAddress + "/" + fileUrl.fileName(); @@ -183,9 +180,9 @@ public class FileUploadService { int postStatusCode = postResponse.getStatusLine().getStatusCode(); String postResponseJson = EntityUtils.toString(postResponse.getEntity()); - AvScanResponse avScanResponse = null; + AvFileSendResponse avFileSendResponse; try { - avScanResponse = new Gson().fromJson(postResponseJson, AvScanResponse.class); + avFileSendResponse = new Gson().fromJson(postResponseJson, AvFileSendResponse.class); } catch (JsonSyntaxException e) { throw new FileUploadException("error json: " + postResponseJson, e); @@ -195,24 +192,24 @@ public class FileUploadService { StringBuilder stringBuilder = new StringBuilder( "http status code " + postStatusCode + " for " + avRestAddress + " post request."); - String status = avScanResponse.status(); + String status = avFileSendResponse.status(); if (status != null) { stringBuilder.append(" Status: ").append(status).append("."); } - if (avScanResponse.error() != null) { - stringBuilder.append(" Error code: ").append(avScanResponse.error().code()) - .append(". Error message: ").append(avScanResponse.error().message()).append(". "); + if (avFileSendResponse.error() != null) { + stringBuilder.append(" Error code: ").append(avFileSendResponse.error().code()) + .append(". Error message: ").append(avFileSendResponse.error().message()).append(". "); } throw new FileUploadException(stringBuilder.toString()); } - String id = avScanResponse.id(); + String id = avFileSendResponse.id(); String reportRequestUri = avRestAddress + "/" + id; HttpGet get = new HttpGet(reportRequestUri); int countdown = 10; long timeout = 1L; - AvResponse avResponse = null; + AvResponse avResponse; do { try { From 22b8665ca524bec65ff701c53be41eaa336da0bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A0=D0=B0=D1=83=D1=84=20=D0=9B=D0=B0=D1=82=D1=8B=D0=BF?= =?UTF-8?q?=D0=BE=D0=B2?= Date: Fri, 6 Sep 2024 01:00:05 +0300 Subject: [PATCH 14/30] SUPPORT-8507: using spring retry --- pom.xml | 9 +++ .../micord/ervu/av/FileUploadApplication.java | 2 + .../ervu/av/exception/RetryableException.java | 7 ++ .../ervu/av/service/FileUploadService.java | 65 ++++++++++++------- 4 files changed, 58 insertions(+), 25 deletions(-) create mode 100644 src/main/java/ru/micord/ervu/av/exception/RetryableException.java diff --git a/pom.xml b/pom.xml index 8ec923b..c76ad50 100644 --- a/pom.xml +++ b/pom.xml @@ -63,6 +63,10 @@ lombok + + org.springframework + spring-aspects + org.springframework spring-web @@ -77,6 +81,11 @@ org.springframework.kafka spring-kafka + + + org.springframework.retry + spring-retry + diff --git a/src/main/java/ru/micord/ervu/av/FileUploadApplication.java b/src/main/java/ru/micord/ervu/av/FileUploadApplication.java index 017e664..4ef2923 100644 --- a/src/main/java/ru/micord/ervu/av/FileUploadApplication.java +++ b/src/main/java/ru/micord/ervu/av/FileUploadApplication.java @@ -2,10 +2,12 @@ package ru.micord.ervu.av; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.retry.annotation.EnableRetry; /** * @author r.latypov */ +@EnableRetry @SpringBootApplication public class FileUploadApplication { diff --git a/src/main/java/ru/micord/ervu/av/exception/RetryableException.java b/src/main/java/ru/micord/ervu/av/exception/RetryableException.java new file mode 100644 index 0000000..3cc7f82 --- /dev/null +++ b/src/main/java/ru/micord/ervu/av/exception/RetryableException.java @@ -0,0 +1,7 @@ +package ru.micord.ervu.av.exception; + +/** + * @author r.latypov + */ +public class RetryableException extends RuntimeException { +} diff --git a/src/main/java/ru/micord/ervu/av/service/FileUploadService.java b/src/main/java/ru/micord/ervu/av/service/FileUploadService.java index e99b172..18ab489 100644 --- a/src/main/java/ru/micord/ervu/av/service/FileUploadService.java +++ b/src/main/java/ru/micord/ervu/av/service/FileUploadService.java @@ -11,6 +11,7 @@ import com.google.gson.Gson; import com.google.gson.GsonBuilder; import com.google.gson.JsonSyntaxException; import org.apache.http.HttpEntity; +import org.apache.http.client.ClientProtocolException; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpDelete; import org.apache.http.client.methods.HttpGet; @@ -33,9 +34,12 @@ import org.springframework.kafka.core.KafkaTemplate; import org.springframework.kafka.support.Acknowledgment; import org.springframework.kafka.support.SendResult; import org.springframework.lang.NonNull; +import org.springframework.retry.annotation.Backoff; +import org.springframework.retry.annotation.Retryable; import org.springframework.stereotype.Service; import ru.micord.ervu.av.exception.FileUploadException; import ru.micord.ervu.av.exception.InvalidHttpFileUrlException; +import ru.micord.ervu.av.exception.RetryableException; import ru.micord.ervu.av.kafka.dto.InMessage; import ru.micord.ervu.av.kafka.dto.OutErrorMessage; import ru.micord.ervu.av.response.AvResponse; @@ -84,7 +88,7 @@ public class FileUploadService { if (infected || !clean) { sendMessage(outErrorTopic.name(), - new OutErrorMessage("file is infected", avResponse, inMessage) + new OutErrorMessage("file is infected or not clean", avResponse, inMessage) ); } else { @@ -207,35 +211,46 @@ public class FileUploadService { String reportRequestUri = avRestAddress + "/" + id; HttpGet get = new HttpGet(reportRequestUri); - int countdown = 10; + // waiting for timeout time before first request long timeout = 1L; - AvResponse avResponse; - - do { - try { - TimeUnit.SECONDS.sleep(timeout); - } - catch (InterruptedException e) { - throw new FileUploadException(e); - } - - try (CloseableHttpResponse getResponse = client.execute(get)) { - int getStatusCode = getResponse.getStatusLine().getStatusCode(); - if (getStatusCode == HttpStatus.OK.value()) { - String getResponseJson = EntityUtils.toString(getResponse.getEntity()); - avResponse = new Gson().fromJson(getResponseJson, AvResponse.class); - } - else { - throw new FileUploadException("http status code " + getStatusCode + " for " - + reportRequestUri + " get request."); - } - } - countdown--; + try { + TimeUnit.SECONDS.sleep(timeout); + } + catch (InterruptedException e) { + throw new FileUploadException(e); } - while (countdown > 0 && (avResponse != null && avResponse.completed() == null)); + return receiveScanReport(client, get); + } + } + catch (IOException e) { + // непредусмотренная ошибка доступа через http-клиент + throw new FileUploadException(e); + } + } + + @Retryable(retryFor = {RetryableException.class}, maxAttempts = 10, + backoff = @Backoff(delay = 1000)) + public AvResponse receiveScanReport(CloseableHttpClient client, HttpGet get) + throws FileUploadException { + + try (CloseableHttpResponse getResponse = client.execute(get)) { + int getStatusCode = getResponse.getStatusLine().getStatusCode(); + if (getStatusCode == HttpStatus.OK.value()) { + String getResponseJson = EntityUtils.toString(getResponse.getEntity()); + AvResponse avResponse = new Gson().fromJson(getResponseJson, AvResponse.class); + if (avResponse.completed() == null) { + throw new RetryableException(); + } return avResponse; } + else { + throw new FileUploadException("http status code " + getStatusCode + " for " + get.getURI() + + " get request."); + } + } + catch (ClientProtocolException e) { + throw new RuntimeException(e); } catch (IOException e) { // непредусмотренная ошибка доступа через http-клиент From d9470861c10d0e53c316190f63f74b35189d319c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A0=D0=B0=D1=83=D1=84=20=D0=9B=D0=B0=D1=82=D1=8B=D0=BF?= =?UTF-8?q?=D0=BE=D0=B2?= Date: Fri, 6 Sep 2024 22:42:46 +0300 Subject: [PATCH 15/30] SUPPORT-8507: moving retryable methode to the separated class --- .../ervu/av/service/FileUploadService.java | 48 ++++------------- .../service/ReceiveScanReportRetryable.java | 52 +++++++++++++++++++ 2 files changed, 62 insertions(+), 38 deletions(-) create mode 100644 src/main/java/ru/micord/ervu/av/service/ReceiveScanReportRetryable.java diff --git a/src/main/java/ru/micord/ervu/av/service/FileUploadService.java b/src/main/java/ru/micord/ervu/av/service/FileUploadService.java index 18ab489..680244d 100644 --- a/src/main/java/ru/micord/ervu/av/service/FileUploadService.java +++ b/src/main/java/ru/micord/ervu/av/service/FileUploadService.java @@ -11,7 +11,6 @@ import com.google.gson.Gson; import com.google.gson.GsonBuilder; import com.google.gson.JsonSyntaxException; import org.apache.http.HttpEntity; -import org.apache.http.client.ClientProtocolException; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpDelete; import org.apache.http.client.methods.HttpGet; @@ -34,16 +33,13 @@ import org.springframework.kafka.core.KafkaTemplate; import org.springframework.kafka.support.Acknowledgment; import org.springframework.kafka.support.SendResult; import org.springframework.lang.NonNull; -import org.springframework.retry.annotation.Backoff; -import org.springframework.retry.annotation.Retryable; import org.springframework.stereotype.Service; import ru.micord.ervu.av.exception.FileUploadException; import ru.micord.ervu.av.exception.InvalidHttpFileUrlException; -import ru.micord.ervu.av.exception.RetryableException; import ru.micord.ervu.av.kafka.dto.InMessage; import ru.micord.ervu.av.kafka.dto.OutErrorMessage; -import ru.micord.ervu.av.response.AvResponse; import ru.micord.ervu.av.response.AvFileSendResponse; +import ru.micord.ervu.av.response.AvResponse; /** * @author r.latypov @@ -60,13 +56,15 @@ public class FileUploadService { private final KafkaTemplate kafkaTemplate; private final NewTopic outErrorTopic; private final NewTopic outSuccessTopic; + private final ReceiveScanReportRetryable receiveScanReportRetryable; @Autowired public FileUploadService(KafkaTemplate kafkaTemplate, NewTopic outErrorTopic, - NewTopic outSuccessTopic) { + NewTopic outSuccessTopic, ReceiveScanReportRetryable receiveScanReportRetryable) { this.kafkaTemplate = kafkaTemplate; this.outErrorTopic = outErrorTopic; this.outSuccessTopic = outSuccessTopic; + this.receiveScanReportRetryable = receiveScanReportRetryable; } @KafkaListener(id = "${spring.kafka.consumer.group-id}", topics = "${kafka-in.topic.name}") @@ -201,8 +199,11 @@ public class FileUploadService { stringBuilder.append(" Status: ").append(status).append("."); } if (avFileSendResponse.error() != null) { - stringBuilder.append(" Error code: ").append(avFileSendResponse.error().code()) - .append(". Error message: ").append(avFileSendResponse.error().message()).append(". "); + stringBuilder.append(" Error code: ") + .append(avFileSendResponse.error().code()) + .append(". Error message: ") + .append(avFileSendResponse.error().message()) + .append(". "); } throw new FileUploadException(stringBuilder.toString()); } @@ -220,7 +221,7 @@ public class FileUploadService { throw new FileUploadException(e); } - return receiveScanReport(client, get); + return receiveScanReportRetryable.receiveScanReport(client, get); } } catch (IOException e) { @@ -229,35 +230,6 @@ public class FileUploadService { } } - @Retryable(retryFor = {RetryableException.class}, maxAttempts = 10, - backoff = @Backoff(delay = 1000)) - public AvResponse receiveScanReport(CloseableHttpClient client, HttpGet get) - throws FileUploadException { - - try (CloseableHttpResponse getResponse = client.execute(get)) { - int getStatusCode = getResponse.getStatusLine().getStatusCode(); - if (getStatusCode == HttpStatus.OK.value()) { - String getResponseJson = EntityUtils.toString(getResponse.getEntity()); - AvResponse avResponse = new Gson().fromJson(getResponseJson, AvResponse.class); - if (avResponse.completed() == null) { - throw new RetryableException(); - } - return avResponse; - } - else { - throw new FileUploadException("http status code " + getStatusCode + " for " + get.getURI() - + " get request."); - } - } - catch (ClientProtocolException e) { - throw new RuntimeException(e); - } - catch (IOException e) { - // непредусмотренная ошибка доступа через http-клиент - throw new FileUploadException(e); - } - } - private void uploadFile(String filePath, String uploadUrl) throws FileUploadException { File file = new File(filePath); diff --git a/src/main/java/ru/micord/ervu/av/service/ReceiveScanReportRetryable.java b/src/main/java/ru/micord/ervu/av/service/ReceiveScanReportRetryable.java new file mode 100644 index 0000000..35e1d46 --- /dev/null +++ b/src/main/java/ru/micord/ervu/av/service/ReceiveScanReportRetryable.java @@ -0,0 +1,52 @@ +package ru.micord.ervu.av.service; + +import java.io.IOException; + +import com.google.gson.Gson; +import org.apache.http.client.ClientProtocolException; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.util.EntityUtils; +import org.springframework.http.HttpStatus; +import org.springframework.retry.annotation.Backoff; +import org.springframework.retry.annotation.Retryable; +import org.springframework.stereotype.Service; +import ru.micord.ervu.av.exception.FileUploadException; +import ru.micord.ervu.av.exception.RetryableException; +import ru.micord.ervu.av.response.AvResponse; + +/** + * @author r.latypov + */ +@Service +public class ReceiveScanReportRetryable { + @Retryable(retryFor = {RetryableException.class}, maxAttempts = 10, + backoff = @Backoff(delay = 1000)) + public AvResponse receiveScanReport(CloseableHttpClient client, HttpGet get) + throws FileUploadException { + + try (CloseableHttpResponse getResponse = client.execute(get)) { + int getStatusCode = getResponse.getStatusLine().getStatusCode(); + if (getStatusCode == HttpStatus.OK.value()) { + String getResponseJson = EntityUtils.toString(getResponse.getEntity()); + AvResponse avResponse = new Gson().fromJson(getResponseJson, AvResponse.class); + if (avResponse.completed() == null) { + throw new RetryableException(); + } + return avResponse; + } + else { + throw new FileUploadException("http status code " + getStatusCode + " for " + get.getURI() + + " get request."); + } + } + catch (ClientProtocolException e) { + throw new RuntimeException(e); + } + catch (IOException e) { + // непредусмотренная ошибка доступа через http-клиент + throw new FileUploadException(e); + } + } +} From 9d6f9a8ce76f390c0ee0582e65eef2c37e91c927 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A0=D0=B0=D1=83=D1=84=20=D0=9B=D0=B0=D1=82=D1=8B=D0=BF?= =?UTF-8?q?=D0=BE=D0=B2?= Date: Fri, 6 Sep 2024 23:03:06 +0300 Subject: [PATCH 16/30] =?UTF-8?q?SUPPORT-8507:=20=D0=B2=D1=8B=D0=BD=D0=B5?= =?UTF-8?q?=D1=81=20=D0=BF=D0=B0=D1=80=D0=B0=D0=BC=D0=B5=D1=82=D1=80=20?= =?UTF-8?q?=D0=B7=D0=B0=D0=B4=D0=B5=D1=80=D0=B6=D0=BA=D0=B8=20=D0=BE=D0=BF?= =?UTF-8?q?=D1=80=D0=BE=D1=81=D0=B0=20=D0=B2=20=D0=BD=D0=B0=D1=81=D1=82?= =?UTF-8?q?=D1=80=D0=BE=D0=B9=D0=BA=D0=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/ru/micord/ervu/av/service/FileUploadService.java | 5 +++-- src/main/resources/application.properties | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/main/java/ru/micord/ervu/av/service/FileUploadService.java b/src/main/java/ru/micord/ervu/av/service/FileUploadService.java index 680244d..7a0403b 100644 --- a/src/main/java/ru/micord/ervu/av/service/FileUploadService.java +++ b/src/main/java/ru/micord/ervu/av/service/FileUploadService.java @@ -49,6 +49,8 @@ public class FileUploadService { private static final Logger logger = LoggerFactory.getLogger(FileUploadService.class); @Value("${av.rest.address}") private String avRestAddress; + @Value("${av.first.timeout.milliseconds}") + private Long avFirstTimeoutMilliseconds; @Value("${file.saving.path}") private String fileSavingPath; @Value("${http.file.server.out.address:http://localhost/out}") @@ -213,9 +215,8 @@ public class FileUploadService { HttpGet get = new HttpGet(reportRequestUri); // waiting for timeout time before first request - long timeout = 1L; try { - TimeUnit.SECONDS.sleep(timeout); + TimeUnit.MILLISECONDS.sleep(avFirstTimeoutMilliseconds); } catch (InterruptedException e) { throw new FileUploadException(e); diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index e2cf0ef..aa2fd2a 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -35,5 +35,6 @@ kafka-out.error.topic.name=${OUT_KAFKA_ERROR_TOPIC_NAME} kafka-out.success.topic.name=${OUT_KAFKA_SUCCESS_TOPIC_NAME} # av.rest.address=${AV_REST_ADDRESS} -file.saving.path=/transfer/ +av.first.timeout.milliseconds=${AV_FIRST_TIMEOUT_MILLISECONDS} +file.saving.path=${FILE_SAVING_PATH} http.file.server.out.address=${HTTP_FILE_SERVER_OUT_ADDRESS} From a4b7a3b124d3af38f5fff69cca5f91edb130d8f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A0=D0=B0=D1=83=D1=84=20=D0=9B=D0=B0=D1=82=D1=8B=D0=BF?= =?UTF-8?q?=D0=BE=D0=B2?= Date: Fri, 6 Sep 2024 23:19:54 +0300 Subject: [PATCH 17/30] =?UTF-8?q?SUPPORT-8507:=20fix=20=D0=B7=D0=B0=D0=B4?= =?UTF-8?q?=D0=B0=D0=BD=D0=B8=D1=8F=20=D0=BF=D0=B0=D1=83=D0=B7=D1=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/main/java/ru/micord/ervu/av/service/FileUploadService.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/ru/micord/ervu/av/service/FileUploadService.java b/src/main/java/ru/micord/ervu/av/service/FileUploadService.java index 7a0403b..cd9d2ff 100644 --- a/src/main/java/ru/micord/ervu/av/service/FileUploadService.java +++ b/src/main/java/ru/micord/ervu/av/service/FileUploadService.java @@ -216,7 +216,8 @@ public class FileUploadService { // waiting for timeout time before first request try { - TimeUnit.MILLISECONDS.sleep(avFirstTimeoutMilliseconds); + TimeUnit.MILLISECONDS.sleep( + avFirstTimeoutMilliseconds == null ? 1000L : avFirstTimeoutMilliseconds); } catch (InterruptedException e) { throw new FileUploadException(e); From 280f6b55a009dbf506bc03cf419c8010a47cbd77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A0=D0=B0=D1=83=D1=84=20=D0=9B=D0=B0=D1=82=D1=8B=D0=BF?= =?UTF-8?q?=D0=BE=D0=B2?= Date: Sat, 7 Sep 2024 00:55:36 +0300 Subject: [PATCH 18/30] SUPPORT-8507: s3 service --- pom.xml | 13 +++++ .../ru/micord/ervu/av/s3/S3Connection.java | 52 +++++++++++++++++++ .../java/ru/micord/ervu/av/s3/S3Service.java | 48 +++++++++++++++++ .../ervu/av/service/FileUploadService.java | 11 ++-- src/main/resources/application.properties | 6 +++ 5 files changed, 126 insertions(+), 4 deletions(-) create mode 100644 src/main/java/ru/micord/ervu/av/s3/S3Connection.java create mode 100644 src/main/java/ru/micord/ervu/av/s3/S3Service.java diff --git a/pom.xml b/pom.xml index c76ad50..b2c3c2f 100644 --- a/pom.xml +++ b/pom.xml @@ -17,6 +17,14 @@ + + com.amazonaws + aws-java-sdk-bom + 1.12.759 + pom + import + + com.google.code.gson gson @@ -44,6 +52,11 @@ + + com.amazonaws + aws-java-sdk-s3 + + com.google.code.gson gson diff --git a/src/main/java/ru/micord/ervu/av/s3/S3Connection.java b/src/main/java/ru/micord/ervu/av/s3/S3Connection.java new file mode 100644 index 0000000..fd6813e --- /dev/null +++ b/src/main/java/ru/micord/ervu/av/s3/S3Connection.java @@ -0,0 +1,52 @@ +package ru.micord.ervu.av.s3; + +import com.amazonaws.auth.AWSCredentials; +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.client.builder.AwsClientBuilder; +import com.amazonaws.regions.Region; +import com.amazonaws.regions.Regions; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +/** + * @author r.latypov + */ +@Configuration +public class S3Connection { + @Value("${s3.out.endpoint}") + private String endpointOut; + @Value("${s3.out.port:9000}") + private int portOut; + @Value("${s3.out.access_key}") + private String accessKeyOut; + @Value("${s3.out.secret_key}") + private String secretKeyOut; + @Value("${s3.out.bucket_name}") + private String bucketNameOut; + + @Bean("outBucketName") + public String getBucketNameOut() { + return bucketNameOut; + } + + @Bean("outClient") + public AmazonS3 getS3OutClient() { + return getS3Client(endpointOut, portOut, accessKeyOut, secretKeyOut); + } + + private static AmazonS3 getS3Client(String endpoint, int port, String accessKey, + String secretKey) { + AWSCredentials credentials = new BasicAWSCredentials(accessKey, secretKey); + String s3Endpoint = endpoint + ":" + port; + String region = Region.getRegion(Regions.DEFAULT_REGION).toString(); + + return AmazonS3ClientBuilder.standard() + .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(s3Endpoint, region)) + .withCredentials(new AWSStaticCredentialsProvider(credentials)) + .build(); + } +} diff --git a/src/main/java/ru/micord/ervu/av/s3/S3Service.java b/src/main/java/ru/micord/ervu/av/s3/S3Service.java new file mode 100644 index 0000000..9178d2c --- /dev/null +++ b/src/main/java/ru/micord/ervu/av/s3/S3Service.java @@ -0,0 +1,48 @@ +package ru.micord.ervu.av.s3; + +import java.io.File; + +import com.amazonaws.AmazonServiceException; +import com.amazonaws.services.s3.AmazonS3; +import jakarta.annotation.PostConstruct; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import ru.micord.ervu.av.exception.FileUploadException; + +/** + * @author r.latypov + */ +@Service +public class S3Service { + private final String outBucketName; + private final AmazonS3 outClient; + + @Autowired + public S3Service(String outBucketName, AmazonS3 outClient) { + this.outBucketName = outBucketName; + this.outClient = outClient; + } + + @PostConstruct + private void init() { + if (!outClient.doesBucketExistV2(outBucketName)) { + outClient.createBucket(outBucketName); + } + } + + public void putFile(String filePath, String key) throws FileUploadException { + try { + outClient.putObject(outBucketName, generateResourceName(outBucketName, key), + new File(filePath) + ); + } + catch (AmazonServiceException e) { + // todo message + throw new FileUploadException(e); + } + } + + private static String generateResourceName(String bucketName, String key) { + return String.join("/", bucketName, key); + } +} diff --git a/src/main/java/ru/micord/ervu/av/service/FileUploadService.java b/src/main/java/ru/micord/ervu/av/service/FileUploadService.java index cd9d2ff..aa64868 100644 --- a/src/main/java/ru/micord/ervu/av/service/FileUploadService.java +++ b/src/main/java/ru/micord/ervu/av/service/FileUploadService.java @@ -40,6 +40,7 @@ import ru.micord.ervu.av.kafka.dto.InMessage; import ru.micord.ervu.av.kafka.dto.OutErrorMessage; import ru.micord.ervu.av.response.AvFileSendResponse; import ru.micord.ervu.av.response.AvResponse; +import ru.micord.ervu.av.s3.S3Service; /** * @author r.latypov @@ -59,14 +60,17 @@ public class FileUploadService { private final NewTopic outErrorTopic; private final NewTopic outSuccessTopic; private final ReceiveScanReportRetryable receiveScanReportRetryable; + private final S3Service s3Service; @Autowired public FileUploadService(KafkaTemplate kafkaTemplate, NewTopic outErrorTopic, - NewTopic outSuccessTopic, ReceiveScanReportRetryable receiveScanReportRetryable) { + NewTopic outSuccessTopic, ReceiveScanReportRetryable receiveScanReportRetryable, + S3Service s3Service) { this.kafkaTemplate = kafkaTemplate; this.outErrorTopic = outErrorTopic; this.outSuccessTopic = outSuccessTopic; this.receiveScanReportRetryable = receiveScanReportRetryable; + this.s3Service = s3Service; } @KafkaListener(id = "${spring.kafka.consumer.group-id}", topics = "${kafka-in.topic.name}") @@ -92,10 +96,9 @@ public class FileUploadService { ); } else { - String uploadUrl = httpFileServerOutAddress + "/" + fileUrl.fileName(); - uploadFile(filePath, uploadUrl); + s3Service.putFile(filePath, fileUrl.fileName()); - inMessage.fileInfo().setFileNameBase(uploadUrl); + inMessage.fileInfo().setFileNameBase(fileUrl.fileName()); sendMessage(outSuccessTopic.name(), inMessage); } diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index aa2fd2a..73d430a 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -38,3 +38,9 @@ av.rest.address=${AV_REST_ADDRESS} av.first.timeout.milliseconds=${AV_FIRST_TIMEOUT_MILLISECONDS} file.saving.path=${FILE_SAVING_PATH} http.file.server.out.address=${HTTP_FILE_SERVER_OUT_ADDRESS} +# +s3.out.endpoint=${S3_ENDPOINT} +s3.out.port=${S3_PORT} +s3.out.access_key=${S3_ACCESS_KEY} +s3.out.secret_key=${S3_SECRET_KEY} +s3.out.bucket_name=${S3_OUT_BUCKET_NAME} From b866c025370e69812d46acfd18e689946421eb53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A0=D0=B0=D1=83=D1=84=20=D0=9B=D0=B0=D1=82=D1=8B=D0=BF?= =?UTF-8?q?=D0=BE=D0=B2?= Date: Sat, 7 Sep 2024 01:36:57 +0300 Subject: [PATCH 19/30] SUPPORT-8507: unused methode and parameter are removed --- .../ervu/av/service/FileUploadService.java | 32 ------------------- .../service/ReceiveScanReportRetryable.java | 1 + src/main/resources/application.properties | 1 - 3 files changed, 1 insertion(+), 33 deletions(-) diff --git a/src/main/java/ru/micord/ervu/av/service/FileUploadService.java b/src/main/java/ru/micord/ervu/av/service/FileUploadService.java index aa64868..f180a03 100644 --- a/src/main/java/ru/micord/ervu/av/service/FileUploadService.java +++ b/src/main/java/ru/micord/ervu/av/service/FileUploadService.java @@ -15,7 +15,6 @@ import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpDelete; import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpPost; -import org.apache.http.client.methods.HttpPut; import org.apache.http.conn.HttpHostConnectException; import org.apache.http.entity.mime.MultipartEntityBuilder; import org.apache.http.entity.mime.content.FileBody; @@ -54,8 +53,6 @@ public class FileUploadService { private Long avFirstTimeoutMilliseconds; @Value("${file.saving.path}") private String fileSavingPath; - @Value("${http.file.server.out.address:http://localhost/out}") - private String httpFileServerOutAddress; private final KafkaTemplate kafkaTemplate; private final NewTopic outErrorTopic; private final NewTopic outSuccessTopic; @@ -235,35 +232,6 @@ public class FileUploadService { } } - private void uploadFile(String filePath, String uploadUrl) throws FileUploadException { - File file = new File(filePath); - - try (CloseableHttpClient client = HttpClients.createDefault()) { - HttpPut put = new HttpPut(uploadUrl); - HttpEntity entity = MultipartEntityBuilder.create() - .addPart("file", new FileBody(file)) - .build(); - put.setEntity(entity); - - try (CloseableHttpResponse response = client.execute(put)) { - int statusCode = response.getStatusLine().getStatusCode(); - if (statusCode == HttpStatus.NO_CONTENT.value()) { - // считаем, что происходит повторная обработка - // этот же файл доставлен при предыдущей обработке и не удален из входного хранилища - logger.warn("file already exists : " + uploadUrl); - } - else if (statusCode != HttpStatus.CREATED.value()) { - String message = "http status code " + statusCode + " : " + uploadUrl; - throw new RuntimeException(message); - } - } - } - catch (IOException e) { - // непредусмотренная ошибка доступа через http-клиент - throw new FileUploadException(e); - } - } - private void deleteFile(String fileUrl) throws FileUploadException { try (CloseableHttpClient client = HttpClients.createDefault()) { HttpDelete delete = new HttpDelete(fileUrl); diff --git a/src/main/java/ru/micord/ervu/av/service/ReceiveScanReportRetryable.java b/src/main/java/ru/micord/ervu/av/service/ReceiveScanReportRetryable.java index 35e1d46..1cb8027 100644 --- a/src/main/java/ru/micord/ervu/av/service/ReceiveScanReportRetryable.java +++ b/src/main/java/ru/micord/ervu/av/service/ReceiveScanReportRetryable.java @@ -42,6 +42,7 @@ public class ReceiveScanReportRetryable { } } catch (ClientProtocolException e) { + // непредусмотренная ошибка доступа через http-клиент throw new RuntimeException(e); } catch (IOException e) { diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 73d430a..43f5667 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -37,7 +37,6 @@ kafka-out.success.topic.name=${OUT_KAFKA_SUCCESS_TOPIC_NAME} av.rest.address=${AV_REST_ADDRESS} av.first.timeout.milliseconds=${AV_FIRST_TIMEOUT_MILLISECONDS} file.saving.path=${FILE_SAVING_PATH} -http.file.server.out.address=${HTTP_FILE_SERVER_OUT_ADDRESS} # s3.out.endpoint=${S3_ENDPOINT} s3.out.port=${S3_PORT} From 9380f5958788bde00f5a4fbf4d64415ea3541e71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A0=D0=B0=D1=83=D1=84=20=D0=9B=D0=B0=D1=82=D1=8B=D0=BF?= =?UTF-8?q?=D0=BE=D0=B2?= Date: Sat, 7 Sep 2024 12:23:11 +0300 Subject: [PATCH 20/30] =?UTF-8?q?SUPPORT-8507:=20=D0=B4=D0=BE=D0=B1=D0=B0?= =?UTF-8?q?=D0=B2=D0=BB=D0=B5=D0=BD=D1=8B=20=D1=82=D0=B5=D1=81=D1=82=D0=BE?= =?UTF-8?q?=D0=B2=D1=8B=D0=B5=20=D0=BF=D0=B5=D1=80=D0=B5=D0=BC=D0=B5=D0=BD?= =?UTF-8?q?=D0=BD=D1=8B=D0=B5=20=D1=81=D1=80=D0=B5=D0=B4=D1=8B=20=D0=B4?= =?UTF-8?q?=D0=BB=D1=8F=20=D0=B4=D0=BE=D0=BA=D0=B5=D1=80=D0=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- test.env | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/test.env b/test.env index f0a194e..aece32d 100644 --- a/test.env +++ b/test.env @@ -1,2 +1,19 @@ MINIO_ROOT_USER=changeIt123 -MINIO_ROOT_PASSWORD=changeIt123 \ No newline at end of file +MINIO_ROOT_PASSWORD=changeIt123 +IN_KAFKA_SERVERS=10.10.31.11:32609 +IN_KAFKA_USERNAME=user1 +IN_KAFKA_PASSWORD=Blfi9d2OFG +IN_KAFKA_TOPIC_NAME=file-to-upload +OUT_KAFKA_SERVERS=10.10.31.11:32609 +OUT_KAFKA_USERNAME=user1 +OUT_KAFKA_PASSWORD=Blfi9d2OFG +OUT_KAFKA_ERROR_TOPIC_NAME=error +OUT_KAFKA_SUCCESS_TOPIC_NAME=success +AV_REST_ADDRESS=http://10.10.31.118:8085/scans +AV_FIRST_TIMEOUT_MILLISECONDS=1000 +FILE_SAVING_PATH=/transfer/ +S3_ENDPOINT=http://ervu-minio.k8s.micord.ru +S3_PORT=31900 +S3_ACCESS_KEY=Keyq0l8IRarEf5GmpvEO +S3_SECRET_KEY=8A2epSoI6OjdHHwA5F6tHxeYRThv47GdGwcBrV7a +S3_OUT_BUCKET_NAME=default-out-bucket From 4317a82f50b014f132443a0bf277bc3af604f02c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A0=D0=B0=D1=83=D1=84=20=D0=9B=D0=B0=D1=82=D1=8B=D0=BF?= =?UTF-8?q?=D0=BE=D0=B2?= Date: Fri, 13 Sep 2024 07:28:38 +0300 Subject: [PATCH 21/30] =?UTF-8?q?SUPPORT-8539:=20SUPPORT-8507:=20=D0=B4?= =?UTF-8?q?=D0=BE=D1=80=D0=B0=D0=B1=D0=BE=D1=82=D0=BA=D0=B0=20=D0=BF=D1=80?= =?UTF-8?q?=D0=B8=D0=BB=D0=BE=D0=B6=D0=B5=D0=BD=D0=B8=D1=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pom.xml | 15 +++++ .../micord/ervu/av/FileUploadApplication.java | 2 + .../input/InputKafkaConsumerConfig.java | 6 +- .../output/OutputKafkaConsumerConfig.java | 64 +++++++++++++++++++ .../ervu/av/kafka/dto/DownloadRequest.java | 27 ++++++++ .../ervu/av/kafka/dto/DownloadResponse.java | 11 ++++ .../micord/ervu/av/kafka/dto/FileStatus.java | 19 ++++++ .../micord/ervu/av/kafka/dto/InMessage.java | 36 ----------- .../ru/micord/ervu/av/kafka/dto/OrgInfo.java | 7 ++ .../ervu/av/kafka/dto/OutErrorMessage.java | 2 +- .../ervu/av/service/FileStatusService.java | 40 ++++++++++++ .../ervu/av/service/FileUploadService.java | 54 +++++++++++----- .../service/ReceiveScanReportRetryable.java | 5 +- src/main/resources/application.properties | 11 ++++ test.env | 10 ++- 15 files changed, 250 insertions(+), 59 deletions(-) create mode 100644 src/main/java/ru/micord/ervu/av/kafka/config/output/OutputKafkaConsumerConfig.java create mode 100644 src/main/java/ru/micord/ervu/av/kafka/dto/DownloadRequest.java create mode 100644 src/main/java/ru/micord/ervu/av/kafka/dto/DownloadResponse.java create mode 100644 src/main/java/ru/micord/ervu/av/kafka/dto/FileStatus.java delete mode 100644 src/main/java/ru/micord/ervu/av/kafka/dto/InMessage.java create mode 100644 src/main/java/ru/micord/ervu/av/kafka/dto/OrgInfo.java create mode 100644 src/main/java/ru/micord/ervu/av/service/FileStatusService.java diff --git a/pom.xml b/pom.xml index b2c3c2f..e0d11d0 100644 --- a/pom.xml +++ b/pom.xml @@ -42,6 +42,12 @@ 4.5.14 + + org.postgresql + postgresql + 42.7.3 + + org.projectlombok lombok @@ -71,6 +77,11 @@ httpmime + + org.postgresql + postgresql + + org.projectlombok lombok @@ -89,6 +100,10 @@ org.springframework.boot spring-boot-starter + + org.springframework.boot + spring-boot-starter-jooq + org.springframework.kafka diff --git a/src/main/java/ru/micord/ervu/av/FileUploadApplication.java b/src/main/java/ru/micord/ervu/av/FileUploadApplication.java index 4ef2923..e1e7979 100644 --- a/src/main/java/ru/micord/ervu/av/FileUploadApplication.java +++ b/src/main/java/ru/micord/ervu/av/FileUploadApplication.java @@ -3,11 +3,13 @@ package ru.micord.ervu.av; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.retry.annotation.EnableRetry; +import org.springframework.transaction.annotation.EnableTransactionManagement; /** * @author r.latypov */ @EnableRetry +@EnableTransactionManagement @SpringBootApplication public class FileUploadApplication { diff --git a/src/main/java/ru/micord/ervu/av/kafka/config/input/InputKafkaConsumerConfig.java b/src/main/java/ru/micord/ervu/av/kafka/config/input/InputKafkaConsumerConfig.java index 672a8f3..7d1f8cc 100644 --- a/src/main/java/ru/micord/ervu/av/kafka/config/input/InputKafkaConsumerConfig.java +++ b/src/main/java/ru/micord/ervu/av/kafka/config/input/InputKafkaConsumerConfig.java @@ -37,7 +37,7 @@ public class InputKafkaConsumerConfig { private String ackMode; @Bean - public ConsumerFactory consumerFactory() { + public ConsumerFactory inputConsumerFactory() { Map configs = new HashMap<>(); configs.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapAddress); @@ -54,10 +54,10 @@ public class InputKafkaConsumerConfig { } @Bean - public ConcurrentKafkaListenerContainerFactory kafkaListenerContainerFactory() { + public ConcurrentKafkaListenerContainerFactory inputKafkaListenerContainerFactory() { ConcurrentKafkaListenerContainerFactory factory = new ConcurrentKafkaListenerContainerFactory<>(); - factory.setConsumerFactory(consumerFactory()); + factory.setConsumerFactory(inputConsumerFactory()); factory.getContainerProperties().setAckMode(ContainerProperties.AckMode.MANUAL_IMMEDIATE); return factory; } diff --git a/src/main/java/ru/micord/ervu/av/kafka/config/output/OutputKafkaConsumerConfig.java b/src/main/java/ru/micord/ervu/av/kafka/config/output/OutputKafkaConsumerConfig.java new file mode 100644 index 0000000..5588680 --- /dev/null +++ b/src/main/java/ru/micord/ervu/av/kafka/config/output/OutputKafkaConsumerConfig.java @@ -0,0 +1,64 @@ +package ru.micord.ervu.av.kafka.config.output; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.kafka.clients.CommonClientConfigs; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.common.config.SaslConfigs; +import org.apache.kafka.common.serialization.StringDeserializer; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.kafka.annotation.EnableKafka; +import org.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory; +import org.springframework.kafka.core.ConsumerFactory; +import org.springframework.kafka.core.DefaultKafkaConsumerFactory; +import org.springframework.kafka.listener.ContainerProperties; + +/** + * @author r.latypov + */ +@Configuration +@EnableKafka +public class OutputKafkaConsumerConfig { + @Value("${spring.kafka.bootstrap-servers}") + private List bootstrapAddress; + @Value("${spring.kafka.consumer.security.protocol}") + private String securityProtocol; + @Value("${spring.kafka.consumer.properties.sasl.jaas.config}") + private String jaasConfig; + @Value("${spring.kafka.consumer.properties.sasl.mechanism}") + private String saslMechanism; + @Value("${spring.kafka.consumer.enable-auto-commit}") + private String enableAutoCommit; + @Value("${spring.kafka.listener.ack-mode}") + private String ackMode; + + @Bean + public ConsumerFactory outputConsumerFactory() { + Map configs = new HashMap<>(); + + configs.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapAddress); + configs.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); + configs.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); + + configs.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, securityProtocol); + configs.put(SaslConfigs.SASL_JAAS_CONFIG, jaasConfig); + configs.put(SaslConfigs.SASL_MECHANISM, saslMechanism); + + configs.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, enableAutoCommit); + + return new DefaultKafkaConsumerFactory<>(configs); + } + + @Bean + public ConcurrentKafkaListenerContainerFactory outputKafkaListenerContainerFactory() { + ConcurrentKafkaListenerContainerFactory factory = + new ConcurrentKafkaListenerContainerFactory<>(); + factory.setConsumerFactory(outputConsumerFactory()); + factory.getContainerProperties().setAckMode(ContainerProperties.AckMode.MANUAL_IMMEDIATE); + return factory; + } +} diff --git a/src/main/java/ru/micord/ervu/av/kafka/dto/DownloadRequest.java b/src/main/java/ru/micord/ervu/av/kafka/dto/DownloadRequest.java new file mode 100644 index 0000000..eb49660 --- /dev/null +++ b/src/main/java/ru/micord/ervu/av/kafka/dto/DownloadRequest.java @@ -0,0 +1,27 @@ +package ru.micord.ervu.av.kafka.dto; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.Setter; +import org.springframework.lang.NonNull; + +/** + * @author r.latypov + */ +public record DownloadRequest(OrgInfo orgInfo, @NonNull FileInfo fileInfo) { + + @Getter + @AllArgsConstructor + public static class FileInfo { + @Setter + private String fileUrl; + private final String fileId; + private final String fileName; + private final String filePatternCode; + private final String FilePatternName; + private final String departureDateTime; + private final String timeZone; + @Setter + private FileStatus fileStatus; + } +} diff --git a/src/main/java/ru/micord/ervu/av/kafka/dto/DownloadResponse.java b/src/main/java/ru/micord/ervu/av/kafka/dto/DownloadResponse.java new file mode 100644 index 0000000..56d7b7e --- /dev/null +++ b/src/main/java/ru/micord/ervu/av/kafka/dto/DownloadResponse.java @@ -0,0 +1,11 @@ +package ru.micord.ervu.av.kafka.dto; + +import org.springframework.lang.NonNull; + +/** + * @author r.latypov + */ +public record DownloadResponse(OrgInfo orgInfo, @NonNull FileInfo fileInfo) { + public record FileInfo(String fileId, FileStatus fileStatus) { + } +} diff --git a/src/main/java/ru/micord/ervu/av/kafka/dto/FileStatus.java b/src/main/java/ru/micord/ervu/av/kafka/dto/FileStatus.java new file mode 100644 index 0000000..95729f0 --- /dev/null +++ b/src/main/java/ru/micord/ervu/av/kafka/dto/FileStatus.java @@ -0,0 +1,19 @@ +package ru.micord.ervu.av.kafka.dto; + +/** + * @author r.latypov + */ +public record FileStatus(String code, String status, String description) { + public static final FileStatus FILE_STATUS_01 = new FileStatus("01", "Загрузка", + "Файл принят до проверки на вирусы" + ); + public static final FileStatus FILE_STATUS_02 = new FileStatus("02", "Проверка не пройдена", + "Проверка на вирусы не пройдена" + ); + public static final FileStatus FILE_STATUS_03 = new FileStatus("03", "Направлено в ЕРВУ", + "Проверка на вирусы пройдена успешно, файл направлен в очередь" + ); + public static final FileStatus FILE_STATUS_04 = new FileStatus("04", "Получен ЕРВУ", + "Файл был принят в обработку" + ); +} diff --git a/src/main/java/ru/micord/ervu/av/kafka/dto/InMessage.java b/src/main/java/ru/micord/ervu/av/kafka/dto/InMessage.java deleted file mode 100644 index 98f997e..0000000 --- a/src/main/java/ru/micord/ervu/av/kafka/dto/InMessage.java +++ /dev/null @@ -1,36 +0,0 @@ -package ru.micord.ervu.av.kafka.dto; - -import lombok.AllArgsConstructor; -import lombok.Getter; -import lombok.Setter; -import org.springframework.lang.NonNull; - -/** - * @author r.latypov - */ -public record InMessage(OrgInfo orgInfo, SenderInfo senderInfo, @NonNull FileInfo fileInfo) { - public record OrgInfo(String orgName, String orgTypeCode, String orgTypeName, String ogrn, - String in, String kpp) { - } - - public record SenderInfo(String senderLastName, String senderFirstName, - String senderMiddleName, String birthDate, String senderRoleCode, - String senderRoleName, String snils, String idERN, - Document document) { - - public record Document(String series, String number, String issueDate) { - } - } - - @Getter - @AllArgsConstructor - public static class FileInfo { - @NonNull - @Setter - private String fileNameBase; - private final String fileName; - private final String filePatternCode; - private final String FilePatternName; - private final String departureDateTime; - } -} diff --git a/src/main/java/ru/micord/ervu/av/kafka/dto/OrgInfo.java b/src/main/java/ru/micord/ervu/av/kafka/dto/OrgInfo.java new file mode 100644 index 0000000..12fcbbf --- /dev/null +++ b/src/main/java/ru/micord/ervu/av/kafka/dto/OrgInfo.java @@ -0,0 +1,7 @@ +package ru.micord.ervu.av.kafka.dto; + +/** + * @author r.latypov + */ +public record OrgInfo(String orgId, String orgName, String prnOid) { +} diff --git a/src/main/java/ru/micord/ervu/av/kafka/dto/OutErrorMessage.java b/src/main/java/ru/micord/ervu/av/kafka/dto/OutErrorMessage.java index eca7a3a..7a9add0 100644 --- a/src/main/java/ru/micord/ervu/av/kafka/dto/OutErrorMessage.java +++ b/src/main/java/ru/micord/ervu/av/kafka/dto/OutErrorMessage.java @@ -7,5 +7,5 @@ import ru.micord.ervu.av.response.AvResponse; * @author r.latypov */ public record OutErrorMessage(String errorMessage, AvResponse avResponse, - @NonNull InMessage inMessage) { + @NonNull DownloadRequest inMessage) { } diff --git a/src/main/java/ru/micord/ervu/av/service/FileStatusService.java b/src/main/java/ru/micord/ervu/av/service/FileStatusService.java new file mode 100644 index 0000000..82e2c7c --- /dev/null +++ b/src/main/java/ru/micord/ervu/av/service/FileStatusService.java @@ -0,0 +1,40 @@ +package ru.micord.ervu.av.service; + +import org.jooq.DSLContext; +import org.jooq.Field; +import org.jooq.Record; +import org.jooq.Table; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import static org.jooq.impl.DSL.field; +import static org.jooq.impl.DSL.name; +import static org.jooq.impl.DSL.table; + +/** + * @author r.latypov + */ +@Service +public class FileStatusService { + public static final Table INTERACTION_LOG = table(name("public", "interaction_log")); + public static final Field INTERACTION_LOG_ID = field(name("id"), Long.class); + public static final Field INTERACTION_LOG_FILE_ID = field(name("file_id"), String.class); + public static final Field INTERACTION_LOG_STATUS = field(name("status"), String.class); + + @Autowired + private DSLContext dslContext; + + public void setStatus(Long id, String status) { + dslContext.update(INTERACTION_LOG) + .set(INTERACTION_LOG_STATUS, status) + .where(INTERACTION_LOG_ID.eq(id)) + .execute(); + } + + public void setStatus(String fileId, String status) { + dslContext.update(INTERACTION_LOG) + .set(INTERACTION_LOG_STATUS, status) + .where(INTERACTION_LOG_FILE_ID.eq(fileId)) + .execute(); + } +} diff --git a/src/main/java/ru/micord/ervu/av/service/FileUploadService.java b/src/main/java/ru/micord/ervu/av/service/FileUploadService.java index f180a03..bd00a74 100644 --- a/src/main/java/ru/micord/ervu/av/service/FileUploadService.java +++ b/src/main/java/ru/micord/ervu/av/service/FileUploadService.java @@ -35,8 +35,9 @@ import org.springframework.lang.NonNull; import org.springframework.stereotype.Service; import ru.micord.ervu.av.exception.FileUploadException; import ru.micord.ervu.av.exception.InvalidHttpFileUrlException; -import ru.micord.ervu.av.kafka.dto.InMessage; -import ru.micord.ervu.av.kafka.dto.OutErrorMessage; +import ru.micord.ervu.av.kafka.dto.DownloadRequest; +import ru.micord.ervu.av.kafka.dto.DownloadResponse; +import ru.micord.ervu.av.kafka.dto.FileStatus; import ru.micord.ervu.av.response.AvFileSendResponse; import ru.micord.ervu.av.response.AvResponse; import ru.micord.ervu.av.s3.S3Service; @@ -57,25 +58,29 @@ public class FileUploadService { private final NewTopic outErrorTopic; private final NewTopic outSuccessTopic; private final ReceiveScanReportRetryable receiveScanReportRetryable; + private final FileStatusService fileStatusService; private final S3Service s3Service; @Autowired public FileUploadService(KafkaTemplate kafkaTemplate, NewTopic outErrorTopic, NewTopic outSuccessTopic, ReceiveScanReportRetryable receiveScanReportRetryable, - S3Service s3Service) { + FileStatusService fileStatusService, S3Service s3Service) { this.kafkaTemplate = kafkaTemplate; this.outErrorTopic = outErrorTopic; this.outSuccessTopic = outSuccessTopic; this.receiveScanReportRetryable = receiveScanReportRetryable; + this.fileStatusService = fileStatusService; this.s3Service = s3Service; } - @KafkaListener(id = "${spring.kafka.consumer.group-id}", topics = "${kafka-in.topic.name}") + @KafkaListener(id = "${spring.kafka.consumer.group-id}", topics = "${kafka-in.topic.name}", + containerFactory = "inputKafkaListenerContainerFactory") public void listenKafkaIn(String kafkaInMessage, Acknowledgment acknowledgment) { - InMessage inMessage = new Gson().fromJson(kafkaInMessage, InMessage.class); + DownloadRequest downloadRequest = new Gson().fromJson(kafkaInMessage, DownloadRequest.class); + String fileId = downloadRequest.fileInfo().getFileId(); try { - FileUrl fileUrl = parseFileUrl(inMessage.fileInfo().getFileNameBase()); + FileUrl fileUrl = parseFileUrl(downloadRequest.fileInfo().getFileUrl()); String filePath = fileSavingPath + fileUrl.fileName(); String downloadUrl = fileUrl.fileUrl(); downloadFile(downloadUrl, filePath); @@ -88,15 +93,20 @@ public class FileUploadService { .anyMatch(verdict -> verdict.equalsIgnoreCase(AvResponse.Scan.VERDICT_INFECTED)); if (infected || !clean) { - sendMessage(outErrorTopic.name(), - new OutErrorMessage("file is infected or not clean", avResponse, inMessage) - ); + downloadRequest.fileInfo().setFileUrl(null); + downloadRequest.fileInfo().setFileStatus(FileStatus.FILE_STATUS_02); + sendMessage(outErrorTopic.name(), downloadRequest); + + fileStatusService.setStatus(fileId, FileStatus.FILE_STATUS_02.status()); } else { s3Service.putFile(filePath, fileUrl.fileName()); - inMessage.fileInfo().setFileNameBase(fileUrl.fileName()); - sendMessage(outSuccessTopic.name(), inMessage); + downloadRequest.fileInfo().setFileUrl(fileUrl.fileName()); + downloadRequest.fileInfo().setFileStatus(FileStatus.FILE_STATUS_03); + sendMessage(outSuccessTopic.name(), downloadRequest); + + fileStatusService.setStatus(fileId, FileStatus.FILE_STATUS_03.status()); } deleteFile(downloadUrl); @@ -108,11 +118,8 @@ public class FileUploadService { // считаем, что повторная обработка сообщения не нужна // ошибку логируем, сообщаем об ошибке, помечаем прочтение сообщения logger.error(e.getMessage() + ": " + kafkaInMessage); - - sendMessage(outErrorTopic.name(), - new OutErrorMessage(e.getMessage(), null, inMessage) - ); acknowledgment.acknowledge(); + throw new RuntimeException(kafkaInMessage, e); } catch (FileUploadException e) { // считаем, что нужно повторное считывание сообщения @@ -121,6 +128,23 @@ public class FileUploadService { } } + @KafkaListener(id = "${spring.kafka.out.consumer.group-id}", + topics = "${kafka-out.response.topic.name}", + containerFactory = "outputKafkaListenerContainerFactory") + public void listenKafkaOut(String kafkaOutResponseMessage, Acknowledgment acknowledgment) { + DownloadResponse downloadResponse = new Gson().fromJson(kafkaOutResponseMessage, + DownloadResponse.class + ); + + String fileId = downloadResponse.fileInfo().fileId(); + FileStatus fileStatus = downloadResponse.fileInfo().fileStatus(); + if (fileStatus.code().equalsIgnoreCase(FileStatus.FILE_STATUS_04.code())) { + fileStatusService.setStatus(fileId, FileStatus.FILE_STATUS_04.status()); + } + + acknowledgment.acknowledge(); + } + /* метод для выделения UUID файла из ссылки на файл сохраняем на диске и отправляем файл в хранилище под тем же UUID, сохраняя расширение файла */ diff --git a/src/main/java/ru/micord/ervu/av/service/ReceiveScanReportRetryable.java b/src/main/java/ru/micord/ervu/av/service/ReceiveScanReportRetryable.java index 1cb8027..3c4f2cf 100644 --- a/src/main/java/ru/micord/ervu/av/service/ReceiveScanReportRetryable.java +++ b/src/main/java/ru/micord/ervu/av/service/ReceiveScanReportRetryable.java @@ -21,8 +21,9 @@ import ru.micord.ervu.av.response.AvResponse; */ @Service public class ReceiveScanReportRetryable { - @Retryable(retryFor = {RetryableException.class}, maxAttempts = 10, - backoff = @Backoff(delay = 1000)) + @Retryable(retryFor = {RetryableException.class}, + maxAttemptsExpression = "${av.retry.max-attempts.count}", + backoff = @Backoff(delayExpression = "${av.retry.delay.milliseconds}")) public AvResponse receiveScanReport(CloseableHttpClient client, HttpGet get) throws FileUploadException { diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 43f5667..2238b29 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -15,6 +15,7 @@ spring.kafka.consumer.properties.sasl.mechanism=SCRAM-SHA-256 # spring.kafka.consumer.enable-auto-commit=false spring.kafka.consumer.group-id=file-to-upload-consumers +spring.kafka.out.consumer.group-id=response-consumers # spring.kafka.listener.ack-mode=MANUAL_IMMEDIATE # @@ -33,9 +34,12 @@ spring.kafka.properties.sasl.mechanism=SCRAM-SHA-256 kafka-in.topic.name=${IN_KAFKA_TOPIC_NAME} kafka-out.error.topic.name=${OUT_KAFKA_ERROR_TOPIC_NAME} kafka-out.success.topic.name=${OUT_KAFKA_SUCCESS_TOPIC_NAME} +kafka-out.response.topic.name=${OUT_KAFKA_RESPONSE_TOPIC_NAME} # av.rest.address=${AV_REST_ADDRESS} av.first.timeout.milliseconds=${AV_FIRST_TIMEOUT_MILLISECONDS} +av.retry.max-attempts.count=${AV_RETRY_MAX_ATTEMPTS_COUNT} +av.retry.delay.milliseconds=${AV_RETRY_DELAY_MILLISECONDS} file.saving.path=${FILE_SAVING_PATH} # s3.out.endpoint=${S3_ENDPOINT} @@ -43,3 +47,10 @@ s3.out.port=${S3_PORT} s3.out.access_key=${S3_ACCESS_KEY} s3.out.secret_key=${S3_SECRET_KEY} s3.out.bucket_name=${S3_OUT_BUCKET_NAME} +# +spring.jooq.sql-dialect=Postgres +spring.datasource.driver-class-name=org.postgresql.Driver +#host:port/database_name +spring.datasource.url=jdbc:postgresql://${SPRING_DATASOURCE_URL} +spring.datasource.username=${SPRING_DATASOURCE_USERNAME} +spring.datasource.password=${SPRING_DATASOURCE_PASSWORD} diff --git a/test.env b/test.env index aece32d..01c5d43 100644 --- a/test.env +++ b/test.env @@ -7,13 +7,19 @@ IN_KAFKA_TOPIC_NAME=file-to-upload OUT_KAFKA_SERVERS=10.10.31.11:32609 OUT_KAFKA_USERNAME=user1 OUT_KAFKA_PASSWORD=Blfi9d2OFG -OUT_KAFKA_ERROR_TOPIC_NAME=error -OUT_KAFKA_SUCCESS_TOPIC_NAME=success +OUT_KAFKA_ERROR_TOPIC_NAME=ervu.lkrp.download.request +OUT_KAFKA_SUCCESS_TOPIC_NAME=ervu.lkrp.download.request +OUT_KAFKA_RESPONSE_TOPIC_NAME=ervu.lkrp.download.response AV_REST_ADDRESS=http://10.10.31.118:8085/scans AV_FIRST_TIMEOUT_MILLISECONDS=1000 +AV_RETRY_MAX_ATTEMPTS_COUNT=10 +AV_RETRY_DELAY_MILLISECONDS=1000 FILE_SAVING_PATH=/transfer/ S3_ENDPOINT=http://ervu-minio.k8s.micord.ru S3_PORT=31900 S3_ACCESS_KEY=Keyq0l8IRarEf5GmpvEO S3_SECRET_KEY=8A2epSoI6OjdHHwA5F6tHxeYRThv47GdGwcBrV7a S3_OUT_BUCKET_NAME=default-out-bucket +SPRING_DATASOURCE_URL=10.10.31.119:5432/ervu-lkrp-ul +SPRING_DATASOURCE_USERNAME=ervu-lkrp-ul +SPRING_DATASOURCE_PASSWORD=ervu-lkrp-ul From ff3bdffdee31db86010aee0bf09bccac27cd84ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A0=D0=B0=D1=83=D1=84=20=D0=9B=D0=B0=D1=82=D1=8B=D0=BF?= =?UTF-8?q?=D0=BE=D0=B2?= Date: Fri, 13 Sep 2024 23:48:13 +0300 Subject: [PATCH 22/30] SUPPORT-8507: removing unused class --- .../ru/micord/ervu/av/kafka/dto/OutErrorMessage.java | 11 ----------- 1 file changed, 11 deletions(-) delete mode 100644 src/main/java/ru/micord/ervu/av/kafka/dto/OutErrorMessage.java diff --git a/src/main/java/ru/micord/ervu/av/kafka/dto/OutErrorMessage.java b/src/main/java/ru/micord/ervu/av/kafka/dto/OutErrorMessage.java deleted file mode 100644 index 7a9add0..0000000 --- a/src/main/java/ru/micord/ervu/av/kafka/dto/OutErrorMessage.java +++ /dev/null @@ -1,11 +0,0 @@ -package ru.micord.ervu.av.kafka.dto; - -import org.springframework.lang.NonNull; -import ru.micord.ervu.av.response.AvResponse; - -/** - * @author r.latypov - */ -public record OutErrorMessage(String errorMessage, AvResponse avResponse, - @NonNull DownloadRequest inMessage) { -} From 4fafc87547861485ef27b1e67c050c86f0bebdc9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A0=D0=B0=D1=83=D1=84=20=D0=9B=D0=B0=D1=82=D1=8B=D0=BF?= =?UTF-8?q?=D0=BE=D0=B2?= Date: Tue, 17 Sep 2024 09:03:53 +0300 Subject: [PATCH 23/30] SUPPORT-8507: fix for review (1) --- .../config/output/OutputKafkaConsumerConfig.java | 8 ++++---- src/main/resources/application.properties | 15 ++++++++++----- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/src/main/java/ru/micord/ervu/av/kafka/config/output/OutputKafkaConsumerConfig.java b/src/main/java/ru/micord/ervu/av/kafka/config/output/OutputKafkaConsumerConfig.java index 5588680..0204148 100644 --- a/src/main/java/ru/micord/ervu/av/kafka/config/output/OutputKafkaConsumerConfig.java +++ b/src/main/java/ru/micord/ervu/av/kafka/config/output/OutputKafkaConsumerConfig.java @@ -23,13 +23,13 @@ import org.springframework.kafka.listener.ContainerProperties; @Configuration @EnableKafka public class OutputKafkaConsumerConfig { - @Value("${spring.kafka.bootstrap-servers}") + @Value("${spring.kafka.out.bootstrap-servers}") private List bootstrapAddress; - @Value("${spring.kafka.consumer.security.protocol}") + @Value("${spring.kafka.properties.security.protocol}") private String securityProtocol; - @Value("${spring.kafka.consumer.properties.sasl.jaas.config}") + @Value("${spring.kafka.properties.sasl.jaas.config}") private String jaasConfig; - @Value("${spring.kafka.consumer.properties.sasl.mechanism}") + @Value("${spring.kafka.properties.sasl.mechanism}") private String saslMechanism; @Value("${spring.kafka.consumer.enable-auto-commit}") private String enableAutoCommit; diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 2238b29..0641b64 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -1,11 +1,12 @@ +# kafka out admin bean settings spring.kafka.admin.security.protocol=SASL_PLAINTEXT #login password to set spring.kafka.admin.properties.sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required username="${OUT_KAFKA_USERNAME}" password="${OUT_KAFKA_PASSWORD}"; spring.kafka.admin.properties.sasl.mechanism=SCRAM-SHA-256 -# +# kafka out servers for admin bean #host1:port1, host2:port2 spring.kafka.bootstrap-servers=${OUT_KAFKA_SERVERS} -# +# kafka in consumer #host1:port1, host2:port2 spring.kafka.consumer.bootstrap-servers=${IN_KAFKA_SERVERS} spring.kafka.consumer.security.protocol=SASL_PLAINTEXT @@ -15,17 +16,21 @@ spring.kafka.consumer.properties.sasl.mechanism=SCRAM-SHA-256 # spring.kafka.consumer.enable-auto-commit=false spring.kafka.consumer.group-id=file-to-upload-consumers +# kafka out consumer spring.kafka.out.consumer.group-id=response-consumers -# +# kafka out servers for consumer +#host1:port1, host2:port2 +spring.kafka.out.bootstrap-servers=${OUT_KAFKA_SERVERS} +# listeners spring.kafka.listener.ack-mode=MANUAL_IMMEDIATE -# +# kafka out producer #host1:port1, host2:port2 spring.kafka.producer.bootstrap-servers=${OUT_KAFKA_SERVERS} spring.kafka.producer.security.protocol=SASL_PLAINTEXT #login password to set spring.kafka.producer.properties.sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required username="${OUT_KAFKA_USERNAME}" password="${OUT_KAFKA_PASSWORD}"; spring.kafka.producer.properties.sasl.mechanism=SCRAM-SHA-256 -# +# kafka general spring.kafka.properties.security.protocol=SASL_PLAINTEXT #login password to set spring.kafka.properties.sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required username="${OUT_KAFKA_USERNAME}" password="${OUT_KAFKA_PASSWORD}"; From 31f674332b0fd4e0303c356626e87fd5308ee973 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A0=D0=B0=D1=83=D1=84=20=D0=9B=D0=B0=D1=82=D1=8B=D0=BF?= =?UTF-8?q?=D0=BE=D0=B2?= Date: Tue, 17 Sep 2024 09:24:39 +0300 Subject: [PATCH 24/30] SUPPORT-8507: fix for review (1.2) --- .../config/output/OutputKafkaConsumerConfig.java | 4 ++-- src/main/resources/application.properties | 16 ++++++++++++---- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/src/main/java/ru/micord/ervu/av/kafka/config/output/OutputKafkaConsumerConfig.java b/src/main/java/ru/micord/ervu/av/kafka/config/output/OutputKafkaConsumerConfig.java index 0204148..fd1c008 100644 --- a/src/main/java/ru/micord/ervu/av/kafka/config/output/OutputKafkaConsumerConfig.java +++ b/src/main/java/ru/micord/ervu/av/kafka/config/output/OutputKafkaConsumerConfig.java @@ -31,9 +31,9 @@ public class OutputKafkaConsumerConfig { private String jaasConfig; @Value("${spring.kafka.properties.sasl.mechanism}") private String saslMechanism; - @Value("${spring.kafka.consumer.enable-auto-commit}") + @Value("${spring.kafka.out.consumer.enable-auto-commit}") private String enableAutoCommit; - @Value("${spring.kafka.listener.ack-mode}") + @Value("${spring.kafka.out.listener.ack-mode}") private String ackMode; @Bean diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 0641b64..cc0dac4 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -6,6 +6,7 @@ spring.kafka.admin.properties.sasl.mechanism=SCRAM-SHA-256 # kafka out servers for admin bean #host1:port1, host2:port2 spring.kafka.bootstrap-servers=${OUT_KAFKA_SERVERS} +# # kafka in consumer #host1:port1, host2:port2 spring.kafka.consumer.bootstrap-servers=${IN_KAFKA_SERVERS} @@ -16,13 +17,19 @@ spring.kafka.consumer.properties.sasl.mechanism=SCRAM-SHA-256 # spring.kafka.consumer.enable-auto-commit=false spring.kafka.consumer.group-id=file-to-upload-consumers +# kafka in listeners +spring.kafka.listener.ack-mode=MANUAL_IMMEDIATE +# # kafka out consumer -spring.kafka.out.consumer.group-id=response-consumers # kafka out servers for consumer #host1:port1, host2:port2 spring.kafka.out.bootstrap-servers=${OUT_KAFKA_SERVERS} -# listeners -spring.kafka.listener.ack-mode=MANUAL_IMMEDIATE +# +spring.kafka.out.consumer.enable-auto-commit=false +spring.kafka.out.consumer.group-id=response-consumers +# kafka out listeners +spring.kafka.out.listener.ack-mode=MANUAL_IMMEDIATE +# # kafka out producer #host1:port1, host2:port2 spring.kafka.producer.bootstrap-servers=${OUT_KAFKA_SERVERS} @@ -30,7 +37,8 @@ spring.kafka.producer.security.protocol=SASL_PLAINTEXT #login password to set spring.kafka.producer.properties.sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required username="${OUT_KAFKA_USERNAME}" password="${OUT_KAFKA_PASSWORD}"; spring.kafka.producer.properties.sasl.mechanism=SCRAM-SHA-256 -# kafka general +# +# kafka out general spring.kafka.properties.security.protocol=SASL_PLAINTEXT #login password to set spring.kafka.properties.sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required username="${OUT_KAFKA_USERNAME}" password="${OUT_KAFKA_PASSWORD}"; From 4bf2cb09968cd3050f7c94b1a5960a465e6e35ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A0=D0=B0=D1=83=D1=84=20=D0=9B=D0=B0=D1=82=D1=8B=D0=BF?= =?UTF-8?q?=D0=BE=D0=B2?= Date: Wed, 18 Sep 2024 07:29:58 +0300 Subject: [PATCH 25/30] SUPPORT-8507: fix for review (1.3) --- .../output/OutputKafkaConsumerConfig.java | 8 ++--- src/main/resources/application.properties | 32 ++++++++++++------- 2 files changed, 24 insertions(+), 16 deletions(-) diff --git a/src/main/java/ru/micord/ervu/av/kafka/config/output/OutputKafkaConsumerConfig.java b/src/main/java/ru/micord/ervu/av/kafka/config/output/OutputKafkaConsumerConfig.java index fd1c008..397512c 100644 --- a/src/main/java/ru/micord/ervu/av/kafka/config/output/OutputKafkaConsumerConfig.java +++ b/src/main/java/ru/micord/ervu/av/kafka/config/output/OutputKafkaConsumerConfig.java @@ -23,13 +23,13 @@ import org.springframework.kafka.listener.ContainerProperties; @Configuration @EnableKafka public class OutputKafkaConsumerConfig { - @Value("${spring.kafka.out.bootstrap-servers}") + @Value("${spring.kafka.out.consumer.bootstrap-servers}") private List bootstrapAddress; - @Value("${spring.kafka.properties.security.protocol}") + @Value("${spring.kafka.out.consumer.security.protocol}") private String securityProtocol; - @Value("${spring.kafka.properties.sasl.jaas.config}") + @Value("${spring.kafka.out.consumer.properties.sasl.jaas.config}") private String jaasConfig; - @Value("${spring.kafka.properties.sasl.mechanism}") + @Value("${spring.kafka.out.consumer.properties.sasl.mechanism}") private String saslMechanism; @Value("${spring.kafka.out.consumer.enable-auto-commit}") private String enableAutoCommit; diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index cc0dac4..e411671 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -1,3 +1,4 @@ +# spring kafka default beans properties begin -> # kafka out admin bean settings spring.kafka.admin.security.protocol=SASL_PLAINTEXT #login password to set @@ -7,7 +8,7 @@ spring.kafka.admin.properties.sasl.mechanism=SCRAM-SHA-256 #host1:port1, host2:port2 spring.kafka.bootstrap-servers=${OUT_KAFKA_SERVERS} # -# kafka in consumer +# kafka in consumer (with possibility for default bean) #host1:port1, host2:port2 spring.kafka.consumer.bootstrap-servers=${IN_KAFKA_SERVERS} spring.kafka.consumer.security.protocol=SASL_PLAINTEXT @@ -20,17 +21,7 @@ spring.kafka.consumer.group-id=file-to-upload-consumers # kafka in listeners spring.kafka.listener.ack-mode=MANUAL_IMMEDIATE # -# kafka out consumer -# kafka out servers for consumer -#host1:port1, host2:port2 -spring.kafka.out.bootstrap-servers=${OUT_KAFKA_SERVERS} -# -spring.kafka.out.consumer.enable-auto-commit=false -spring.kafka.out.consumer.group-id=response-consumers -# kafka out listeners -spring.kafka.out.listener.ack-mode=MANUAL_IMMEDIATE -# -# kafka out producer +# kafka out producer (with possibility for default bean) #host1:port1, host2:port2 spring.kafka.producer.bootstrap-servers=${OUT_KAFKA_SERVERS} spring.kafka.producer.security.protocol=SASL_PLAINTEXT @@ -43,6 +34,21 @@ spring.kafka.properties.security.protocol=SASL_PLAINTEXT #login password to set spring.kafka.properties.sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required username="${OUT_KAFKA_USERNAME}" password="${OUT_KAFKA_PASSWORD}"; spring.kafka.properties.sasl.mechanism=SCRAM-SHA-256 +# spring kafka default beans properties <- end +# +# kafka out consumer (not for default bean creation by spring) +#host1:port1, host2:port2 +spring.kafka.out.consumer.bootstrap-servers=${OUT_KAFKA_SERVERS} +spring.kafka.out.consumer.security.protocol=SASL_PLAINTEXT +#login password to set +spring.kafka.out.consumer.properties.sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required username="${OUT_KAFKA_USERNAME}" password="${OUT_KAFKA_PASSWORD}"; +spring.kafka.out.consumer.properties.sasl.mechanism=SCRAM-SHA-256 +# +spring.kafka.out.consumer.enable-auto-commit=false +spring.kafka.out.consumer.group-id=response-consumers +# kafka out listeners +spring.kafka.out.listener.ack-mode=MANUAL_IMMEDIATE +# # kafka-in.topic.name=${IN_KAFKA_TOPIC_NAME} kafka-out.error.topic.name=${OUT_KAFKA_ERROR_TOPIC_NAME} @@ -61,9 +67,11 @@ s3.out.access_key=${S3_ACCESS_KEY} s3.out.secret_key=${S3_SECRET_KEY} s3.out.bucket_name=${S3_OUT_BUCKET_NAME} # +# spring jooq dsl bean properties begin -> spring.jooq.sql-dialect=Postgres spring.datasource.driver-class-name=org.postgresql.Driver #host:port/database_name spring.datasource.url=jdbc:postgresql://${SPRING_DATASOURCE_URL} spring.datasource.username=${SPRING_DATASOURCE_USERNAME} spring.datasource.password=${SPRING_DATASOURCE_PASSWORD} +# spring jooq dsl bean properties <- end From 9d0bf38470bc728b5925e3d91d60745c3d4a1e45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A5=D0=B0=D0=BB=D1=82=D0=BE=D0=B1=D0=B8=D0=BD=20=D0=95?= =?UTF-8?q?=D0=B2=D0=B3=D0=B5=D0=BD=D0=B8=D0=B9?= Date: Wed, 18 Sep 2024 12:20:32 +0300 Subject: [PATCH 26/30] updated groupId, artifactId --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index e0d11d0..76d4298 100644 --- a/pom.xml +++ b/pom.xml @@ -11,9 +11,9 @@ - ru.cg.subproject + ru.micord.ervu.lkrp file-upload - 0.0.1-SNAPSHOT + 1.0.0-SNAPSHOT From 70c45b1dad6a1ac7d27005fc16fb7f3900b20452 Mon Sep 17 00:00:00 2001 From: Eduard Tihomirov Date: Thu, 19 Sep 2024 09:34:13 +0300 Subject: [PATCH 27/30] SUPPORT-8552: Fix --- src/main/java/ru/micord/ervu/av/kafka/dto/OrgInfo.java | 2 +- src/main/java/ru/micord/ervu/av/kafka/dto/SenderInfo.java | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 src/main/java/ru/micord/ervu/av/kafka/dto/SenderInfo.java diff --git a/src/main/java/ru/micord/ervu/av/kafka/dto/OrgInfo.java b/src/main/java/ru/micord/ervu/av/kafka/dto/OrgInfo.java index 12fcbbf..3b50b59 100644 --- a/src/main/java/ru/micord/ervu/av/kafka/dto/OrgInfo.java +++ b/src/main/java/ru/micord/ervu/av/kafka/dto/OrgInfo.java @@ -3,5 +3,5 @@ package ru.micord.ervu.av.kafka.dto; /** * @author r.latypov */ -public record OrgInfo(String orgId, String orgName, String prnOid) { +public record OrgInfo(String orgId, String orgName, SenderInfo senderInfo) { } diff --git a/src/main/java/ru/micord/ervu/av/kafka/dto/SenderInfo.java b/src/main/java/ru/micord/ervu/av/kafka/dto/SenderInfo.java new file mode 100644 index 0000000..af8f0b0 --- /dev/null +++ b/src/main/java/ru/micord/ervu/av/kafka/dto/SenderInfo.java @@ -0,0 +1,7 @@ +package ru.micord.ervu.av.kafka.dto; + +/** + * @author Eduard Tihomirov + */ +public record SenderInfo(String prnOid, String lastName, String firstName, String middleName) { +} From 6c41fa792c0c089b252fd72db8701c5a8cf6a0a4 Mon Sep 17 00:00:00 2001 From: gulnaz Date: Thu, 19 Sep 2024 11:24:22 +0300 Subject: [PATCH 28/30] SUPPORT-8553: fix properties --- .../input/InputKafkaConsumerConfig.java | 6 ++-- .../output/OutputKafkaConsumerConfig.java | 6 ++-- .../output/OutputKafkaProducerConfig.java | 2 +- .../config/output/OutputKafkaTopicConfig.java | 4 +-- .../ervu/av/service/FileUploadService.java | 8 ++--- .../service/ReceiveScanReportRetryable.java | 2 +- src/main/resources/application.properties | 30 +++++++++---------- 7 files changed, 29 insertions(+), 29 deletions(-) diff --git a/src/main/java/ru/micord/ervu/av/kafka/config/input/InputKafkaConsumerConfig.java b/src/main/java/ru/micord/ervu/av/kafka/config/input/InputKafkaConsumerConfig.java index 7d1f8cc..7c8f597 100644 --- a/src/main/java/ru/micord/ervu/av/kafka/config/input/InputKafkaConsumerConfig.java +++ b/src/main/java/ru/micord/ervu/av/kafka/config/input/InputKafkaConsumerConfig.java @@ -23,7 +23,7 @@ import org.springframework.kafka.listener.ContainerProperties; @Configuration @EnableKafka public class InputKafkaConsumerConfig { - @Value("${spring.kafka.consumer.bootstrap-servers}") + @Value("${spring.kafka.consumer.bootstrap.servers}") private List bootstrapAddress; @Value("${spring.kafka.consumer.security.protocol}") private String securityProtocol; @@ -31,9 +31,9 @@ public class InputKafkaConsumerConfig { private String jaasConfig; @Value("${spring.kafka.consumer.properties.sasl.mechanism}") private String saslMechanism; - @Value("${spring.kafka.consumer.enable-auto-commit}") + @Value("${spring.kafka.consumer.enable.auto.commit}") private String enableAutoCommit; - @Value("${spring.kafka.listener.ack-mode}") + @Value("${spring.kafka.listener.ack.mode}") private String ackMode; @Bean diff --git a/src/main/java/ru/micord/ervu/av/kafka/config/output/OutputKafkaConsumerConfig.java b/src/main/java/ru/micord/ervu/av/kafka/config/output/OutputKafkaConsumerConfig.java index 397512c..bc5ac81 100644 --- a/src/main/java/ru/micord/ervu/av/kafka/config/output/OutputKafkaConsumerConfig.java +++ b/src/main/java/ru/micord/ervu/av/kafka/config/output/OutputKafkaConsumerConfig.java @@ -23,7 +23,7 @@ import org.springframework.kafka.listener.ContainerProperties; @Configuration @EnableKafka public class OutputKafkaConsumerConfig { - @Value("${spring.kafka.out.consumer.bootstrap-servers}") + @Value("${spring.kafka.out.consumer.bootstrap.servers}") private List bootstrapAddress; @Value("${spring.kafka.out.consumer.security.protocol}") private String securityProtocol; @@ -31,9 +31,9 @@ public class OutputKafkaConsumerConfig { private String jaasConfig; @Value("${spring.kafka.out.consumer.properties.sasl.mechanism}") private String saslMechanism; - @Value("${spring.kafka.out.consumer.enable-auto-commit}") + @Value("${spring.kafka.out.consumer.enable.auto.commit}") private String enableAutoCommit; - @Value("${spring.kafka.out.listener.ack-mode}") + @Value("${spring.kafka.out.listener.ack.mode}") private String ackMode; @Bean diff --git a/src/main/java/ru/micord/ervu/av/kafka/config/output/OutputKafkaProducerConfig.java b/src/main/java/ru/micord/ervu/av/kafka/config/output/OutputKafkaProducerConfig.java index 340fa3d..2a14b10 100644 --- a/src/main/java/ru/micord/ervu/av/kafka/config/output/OutputKafkaProducerConfig.java +++ b/src/main/java/ru/micord/ervu/av/kafka/config/output/OutputKafkaProducerConfig.java @@ -20,7 +20,7 @@ import org.springframework.kafka.core.ProducerFactory; */ @Configuration public class OutputKafkaProducerConfig { - @Value("${spring.kafka.producer.bootstrap-servers}") + @Value("${spring.kafka.producer.bootstrap.servers}") private List bootstrapAddress; @Value("${spring.kafka.producer.security.protocol}") private String securityProtocol; diff --git a/src/main/java/ru/micord/ervu/av/kafka/config/output/OutputKafkaTopicConfig.java b/src/main/java/ru/micord/ervu/av/kafka/config/output/OutputKafkaTopicConfig.java index 8fb5ef9..79f1ef8 100644 --- a/src/main/java/ru/micord/ervu/av/kafka/config/output/OutputKafkaTopicConfig.java +++ b/src/main/java/ru/micord/ervu/av/kafka/config/output/OutputKafkaTopicConfig.java @@ -11,9 +11,9 @@ import org.springframework.kafka.config.TopicBuilder; */ @Configuration public class OutputKafkaTopicConfig { - @Value("${kafka-out.error.topic.name}") + @Value("${kafka.out.error.topic.name}") private String kafkaOutErrorTopicName; - @Value("${kafka-out.success.topic.name}") + @Value("${kafka.out.success.topic.name}") private String kafkaOutSuccessTopicName; @Bean diff --git a/src/main/java/ru/micord/ervu/av/service/FileUploadService.java b/src/main/java/ru/micord/ervu/av/service/FileUploadService.java index bd00a74..380cca8 100644 --- a/src/main/java/ru/micord/ervu/av/service/FileUploadService.java +++ b/src/main/java/ru/micord/ervu/av/service/FileUploadService.java @@ -73,7 +73,7 @@ public class FileUploadService { this.s3Service = s3Service; } - @KafkaListener(id = "${spring.kafka.consumer.group-id}", topics = "${kafka-in.topic.name}", + @KafkaListener(id = "${spring.kafka.consumer.group.id}", topics = "${kafka.in.topic.name}", containerFactory = "inputKafkaListenerContainerFactory") public void listenKafkaIn(String kafkaInMessage, Acknowledgment acknowledgment) { DownloadRequest downloadRequest = new Gson().fromJson(kafkaInMessage, DownloadRequest.class); @@ -128,8 +128,8 @@ public class FileUploadService { } } - @KafkaListener(id = "${spring.kafka.out.consumer.group-id}", - topics = "${kafka-out.response.topic.name}", + @KafkaListener(id = "${spring.kafka.out.consumer.group.id}", + topics = "${kafka.out.response.topic.name}", containerFactory = "outputKafkaListenerContainerFactory") public void listenKafkaOut(String kafkaOutResponseMessage, Acknowledgment acknowledgment) { DownloadResponse downloadResponse = new Gson().fromJson(kafkaOutResponseMessage, @@ -269,7 +269,7 @@ public class FileUploadService { } } catch (IOException e) { - // непредусмотренная ошибка доступа через http-клиент + // непредусмотренная ошибка доступа через http-клиент throw new FileUploadException(e); } } diff --git a/src/main/java/ru/micord/ervu/av/service/ReceiveScanReportRetryable.java b/src/main/java/ru/micord/ervu/av/service/ReceiveScanReportRetryable.java index 3c4f2cf..77d3838 100644 --- a/src/main/java/ru/micord/ervu/av/service/ReceiveScanReportRetryable.java +++ b/src/main/java/ru/micord/ervu/av/service/ReceiveScanReportRetryable.java @@ -22,7 +22,7 @@ import ru.micord.ervu.av.response.AvResponse; @Service public class ReceiveScanReportRetryable { @Retryable(retryFor = {RetryableException.class}, - maxAttemptsExpression = "${av.retry.max-attempts.count}", + maxAttemptsExpression = "${av.retry.max.attempts.count}", backoff = @Backoff(delayExpression = "${av.retry.delay.milliseconds}")) public AvResponse receiveScanReport(CloseableHttpClient client, HttpGet get) throws FileUploadException { diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index e411671..153cb49 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -6,24 +6,24 @@ spring.kafka.admin.properties.sasl.jaas.config=org.apache.kafka.common.security. spring.kafka.admin.properties.sasl.mechanism=SCRAM-SHA-256 # kafka out servers for admin bean #host1:port1, host2:port2 -spring.kafka.bootstrap-servers=${OUT_KAFKA_SERVERS} +spring.kafka.bootstrap.servers=${OUT_KAFKA_SERVERS} # # kafka in consumer (with possibility for default bean) #host1:port1, host2:port2 -spring.kafka.consumer.bootstrap-servers=${IN_KAFKA_SERVERS} +spring.kafka.consumer.bootstrap.servers=${IN_KAFKA_SERVERS} spring.kafka.consumer.security.protocol=SASL_PLAINTEXT #login password to set spring.kafka.consumer.properties.sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required username="${IN_KAFKA_USERNAME}" password="${IN_KAFKA_PASSWORD}"; spring.kafka.consumer.properties.sasl.mechanism=SCRAM-SHA-256 # -spring.kafka.consumer.enable-auto-commit=false -spring.kafka.consumer.group-id=file-to-upload-consumers +spring.kafka.consumer.enable.auto.commit=false +spring.kafka.consumer.group.id=file-to-upload-consumers # kafka in listeners -spring.kafka.listener.ack-mode=MANUAL_IMMEDIATE +spring.kafka.listener.ack.mode=MANUAL_IMMEDIATE # # kafka out producer (with possibility for default bean) #host1:port1, host2:port2 -spring.kafka.producer.bootstrap-servers=${OUT_KAFKA_SERVERS} +spring.kafka.producer.bootstrap.servers=${OUT_KAFKA_SERVERS} spring.kafka.producer.security.protocol=SASL_PLAINTEXT #login password to set spring.kafka.producer.properties.sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required username="${OUT_KAFKA_USERNAME}" password="${OUT_KAFKA_PASSWORD}"; @@ -38,26 +38,26 @@ spring.kafka.properties.sasl.mechanism=SCRAM-SHA-256 # # kafka out consumer (not for default bean creation by spring) #host1:port1, host2:port2 -spring.kafka.out.consumer.bootstrap-servers=${OUT_KAFKA_SERVERS} +spring.kafka.out.consumer.bootstrap.servers=${OUT_KAFKA_SERVERS} spring.kafka.out.consumer.security.protocol=SASL_PLAINTEXT #login password to set spring.kafka.out.consumer.properties.sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required username="${OUT_KAFKA_USERNAME}" password="${OUT_KAFKA_PASSWORD}"; spring.kafka.out.consumer.properties.sasl.mechanism=SCRAM-SHA-256 # -spring.kafka.out.consumer.enable-auto-commit=false -spring.kafka.out.consumer.group-id=response-consumers +spring.kafka.out.consumer.enable.auto.commit=false +spring.kafka.out.consumer.group.id=response-consumers # kafka out listeners -spring.kafka.out.listener.ack-mode=MANUAL_IMMEDIATE +spring.kafka.out.listener.ack.mode=MANUAL_IMMEDIATE # # -kafka-in.topic.name=${IN_KAFKA_TOPIC_NAME} -kafka-out.error.topic.name=${OUT_KAFKA_ERROR_TOPIC_NAME} -kafka-out.success.topic.name=${OUT_KAFKA_SUCCESS_TOPIC_NAME} -kafka-out.response.topic.name=${OUT_KAFKA_RESPONSE_TOPIC_NAME} +kafka.in.topic.name=${IN_KAFKA_TOPIC_NAME} +kafka.out.error.topic.name=${OUT_KAFKA_ERROR_TOPIC_NAME} +kafka.out.success.topic.name=${OUT_KAFKA_SUCCESS_TOPIC_NAME} +kafka.out.response.topic.name=${OUT_KAFKA_RESPONSE_TOPIC_NAME} # av.rest.address=${AV_REST_ADDRESS} av.first.timeout.milliseconds=${AV_FIRST_TIMEOUT_MILLISECONDS} -av.retry.max-attempts.count=${AV_RETRY_MAX_ATTEMPTS_COUNT} +av.retry.max.attempts.count=${AV_RETRY_MAX_ATTEMPTS_COUNT} av.retry.delay.milliseconds=${AV_RETRY_DELAY_MILLISECONDS} file.saving.path=${FILE_SAVING_PATH} # From 661d0687e8caa48dfd4699caf1cebc85ac5e9216 Mon Sep 17 00:00:00 2001 From: gulnaz Date: Tue, 24 Sep 2024 11:21:27 +0300 Subject: [PATCH 29/30] SUPPORT-8557: add actuator and kafka health check endpoint --- pom.xml | 13 +++++++++++++ src/main/resources/application.properties | 7 +++++++ 2 files changed, 20 insertions(+) diff --git a/pom.xml b/pom.xml index 76d4298..2050e66 100644 --- a/pom.xml +++ b/pom.xml @@ -100,15 +100,28 @@ org.springframework.boot spring-boot-starter + + org.springframework.boot + spring-boot-starter-web + org.springframework.boot spring-boot-starter-jooq + + org.springframework.boot + spring-boot-starter-actuator + org.springframework.kafka spring-kafka + + io.github.leofuso + actuator-kafka + v3.0.2.0.RELEASE + org.springframework.retry diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 153cb49..dd17600 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -1,3 +1,5 @@ +server.servlet.context-path=/av + # spring kafka default beans properties begin -> # kafka out admin bean settings spring.kafka.admin.security.protocol=SASL_PLAINTEXT @@ -75,3 +77,8 @@ spring.datasource.url=jdbc:postgresql://${SPRING_DATASOURCE_URL} spring.datasource.username=${SPRING_DATASOURCE_USERNAME} spring.datasource.password=${SPRING_DATASOURCE_PASSWORD} # spring jooq dsl bean properties <- end + +# endpoints management +management.endpoints.web.exposure.include = health, info, metrics +management.endpoint.health.show-details = always +management.endpoint.health.group.readiness.include = kafka From c844c6cea30b5bacf7a6860f597abad9c38b91a6 Mon Sep 17 00:00:00 2001 From: gulnaz Date: Tue, 24 Sep 2024 11:37:36 +0300 Subject: [PATCH 30/30] SUPPORT-8558: rename environment variables --- src/main/resources/application.properties | 22 +++++++++++----------- test.env | 22 ++++++++++------------ 2 files changed, 21 insertions(+), 23 deletions(-) diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 153cb49..df15b7e 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -10,10 +10,10 @@ spring.kafka.bootstrap.servers=${OUT_KAFKA_SERVERS} # # kafka in consumer (with possibility for default bean) #host1:port1, host2:port2 -spring.kafka.consumer.bootstrap.servers=${IN_KAFKA_SERVERS} +spring.kafka.consumer.bootstrap.servers=${AV_KAFKA_SERVERS} spring.kafka.consumer.security.protocol=SASL_PLAINTEXT #login password to set -spring.kafka.consumer.properties.sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required username="${IN_KAFKA_USERNAME}" password="${IN_KAFKA_PASSWORD}"; +spring.kafka.consumer.properties.sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required username="${AV_KAFKA_USERNAME}" password="${AV_KAFKA_PASSWORD}"; spring.kafka.consumer.properties.sasl.mechanism=SCRAM-SHA-256 # spring.kafka.consumer.enable.auto.commit=false @@ -23,25 +23,25 @@ spring.kafka.listener.ack.mode=MANUAL_IMMEDIATE # # kafka out producer (with possibility for default bean) #host1:port1, host2:port2 -spring.kafka.producer.bootstrap.servers=${OUT_KAFKA_SERVERS} +spring.kafka.producer.bootstrap.servers=${ERVU_KAFKA_SERVERS} spring.kafka.producer.security.protocol=SASL_PLAINTEXT #login password to set -spring.kafka.producer.properties.sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required username="${OUT_KAFKA_USERNAME}" password="${OUT_KAFKA_PASSWORD}"; +spring.kafka.producer.properties.sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required username="${ERVU_KAFKA_USERNAME}" password="${ERVU_KAFKA_PASSWORD}"; spring.kafka.producer.properties.sasl.mechanism=SCRAM-SHA-256 # # kafka out general spring.kafka.properties.security.protocol=SASL_PLAINTEXT #login password to set -spring.kafka.properties.sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required username="${OUT_KAFKA_USERNAME}" password="${OUT_KAFKA_PASSWORD}"; +spring.kafka.properties.sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required username="${ERVU_KAFKA_USERNAME}" password="${ERVU_KAFKA_PASSWORD}"; spring.kafka.properties.sasl.mechanism=SCRAM-SHA-256 # spring kafka default beans properties <- end # # kafka out consumer (not for default bean creation by spring) #host1:port1, host2:port2 -spring.kafka.out.consumer.bootstrap.servers=${OUT_KAFKA_SERVERS} +spring.kafka.out.consumer.bootstrap.servers=${ERVU_KAFKA_SERVERS} spring.kafka.out.consumer.security.protocol=SASL_PLAINTEXT #login password to set -spring.kafka.out.consumer.properties.sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required username="${OUT_KAFKA_USERNAME}" password="${OUT_KAFKA_PASSWORD}"; +spring.kafka.out.consumer.properties.sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule required username="${ERVU_KAFKA_USERNAME}" password="${ERVU_KAFKA_PASSWORD}"; spring.kafka.out.consumer.properties.sasl.mechanism=SCRAM-SHA-256 # spring.kafka.out.consumer.enable.auto.commit=false @@ -50,10 +50,10 @@ spring.kafka.out.consumer.group.id=response-consumers spring.kafka.out.listener.ack.mode=MANUAL_IMMEDIATE # # -kafka.in.topic.name=${IN_KAFKA_TOPIC_NAME} -kafka.out.error.topic.name=${OUT_KAFKA_ERROR_TOPIC_NAME} -kafka.out.success.topic.name=${OUT_KAFKA_SUCCESS_TOPIC_NAME} -kafka.out.response.topic.name=${OUT_KAFKA_RESPONSE_TOPIC_NAME} +kafka.in.topic.name=${AV_KAFKA_TOPIC_NAME} +kafka.out.error.topic.name=${ERVU_KAFKA_ERROR_TOPIC_NAME} +kafka.out.success.topic.name=${ERVU_KAFKA_SUCCESS_TOPIC_NAME} +kafka.out.response.topic.name=${ERVU_KAFKA_RESPONSE_TOPIC_NAME} # av.rest.address=${AV_REST_ADDRESS} av.first.timeout.milliseconds=${AV_FIRST_TIMEOUT_MILLISECONDS} diff --git a/test.env b/test.env index 01c5d43..8d3216d 100644 --- a/test.env +++ b/test.env @@ -1,15 +1,13 @@ -MINIO_ROOT_USER=changeIt123 -MINIO_ROOT_PASSWORD=changeIt123 -IN_KAFKA_SERVERS=10.10.31.11:32609 -IN_KAFKA_USERNAME=user1 -IN_KAFKA_PASSWORD=Blfi9d2OFG -IN_KAFKA_TOPIC_NAME=file-to-upload -OUT_KAFKA_SERVERS=10.10.31.11:32609 -OUT_KAFKA_USERNAME=user1 -OUT_KAFKA_PASSWORD=Blfi9d2OFG -OUT_KAFKA_ERROR_TOPIC_NAME=ervu.lkrp.download.request -OUT_KAFKA_SUCCESS_TOPIC_NAME=ervu.lkrp.download.request -OUT_KAFKA_RESPONSE_TOPIC_NAME=ervu.lkrp.download.response +AV_KAFKA_SERVERS=10.10.31.11:32609 +AV_KAFKA_USERNAME=user1 +AV_KAFKA_PASSWORD=Blfi9d2OFG +AV_KAFKA_TOPIC_NAME=file-to-upload +ERVU_KAFKA_SERVERS=10.10.31.11:32609 +ERVU_KAFKA_USERNAME=user1 +ERVU_KAFKA_PASSWORD=Blfi9d2OFG +ERVU_KAFKA_ERROR_TOPIC_NAME=ervu.lkrp.download.request +ERVU_KAFKA_SUCCESS_TOPIC_NAME=ervu.lkrp.download.request +ERVU_KAFKA_RESPONSE_TOPIC_NAME=ervu.lkrp.download.response AV_REST_ADDRESS=http://10.10.31.118:8085/scans AV_FIRST_TIMEOUT_MILLISECONDS=1000 AV_RETRY_MAX_ATTEMPTS_COUNT=10