From e6bc31f4c296bdb8f9b72956dfdd6c93ed3f9262 Mon Sep 17 00:00:00 2001 From: gulnaz Date: Tue, 5 Nov 2024 16:31:51 +0300 Subject: [PATCH] add job for WebDAV cleanup --- backend/pom.xml | 4 + .../ervu/client/fileupload/WebDavClient.java | 95 +++++++++++++++---- .../EmployeeInfoFileUploadService.java | 8 +- .../EsnsiOkopfSchedulerService.java | 2 +- .../EsnsiOkopfSchedulerServiceImpl.java | 2 +- .../scheduler/WebDavSchedulerService.java | 37 ++++++++ config/micord.env | 7 +- config/standalone/dev/standalone.xml | 4 + pom.xml | 5 + 9 files changed, 137 insertions(+), 27 deletions(-) rename backend/src/main/java/ervu/service/{scheduer => scheduler}/EsnsiOkopfSchedulerService.java (76%) rename backend/src/main/java/ervu/service/{scheduer => scheduler}/EsnsiOkopfSchedulerServiceImpl.java (98%) create mode 100644 backend/src/main/java/ervu/service/scheduler/WebDavSchedulerService.java diff --git a/backend/pom.xml b/backend/pom.xml index 47300155..c4bd6ea1 100644 --- a/backend/pom.xml +++ b/backend/pom.xml @@ -244,6 +244,10 @@ org.apache.logging.log4j log4j-web + + com.github.lookfirst + sardine + ${project.parent.artifactId} diff --git a/backend/src/main/java/ervu/client/fileupload/WebDavClient.java b/backend/src/main/java/ervu/client/fileupload/WebDavClient.java index acfa6458..42f7b251 100644 --- a/backend/src/main/java/ervu/client/fileupload/WebDavClient.java +++ b/backend/src/main/java/ervu/client/fileupload/WebDavClient.java @@ -10,7 +10,14 @@ import java.net.http.HttpClient; import java.net.http.HttpRequest; import java.net.http.HttpResponse; import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Predicate; +import com.github.sardine.DavResource; +import com.github.sardine.Sardine; +import com.github.sardine.SardineFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Value; @@ -36,32 +43,29 @@ public class WebDavClient { @Value("${file.webdav.upload.password}") private String password; - @Retryable(value = {IOException.class, InterruptedException.class}, backoff = @Backoff(delay = 500L)) - public boolean webDavUploadFile(String url, MultipartFile multipartFile) { + @Retryable(value = {IOException.class}, backoff = @Backoff(delayExpression = "${webdav.retry.delay:500}")) + public boolean uploadFile(String url, MultipartFile multipartFile) { + Sardine sardine = initClient(username, password); + try { - HttpClient httpClient = HttpClient.newBuilder() - .authenticator(new Authenticator() { - @Override - protected PasswordAuthentication getPasswordAuthentication() { - return new PasswordAuthentication(username, password.toCharArray()); - } - }) - .build(); - - HttpRequest httpRequest = HttpRequest.newBuilder().uri(URI.create(url)) - .PUT(HttpRequest.BodyPublishers.ofByteArray(multipartFile.getBytes())).build(); - - HttpResponse response = httpClient.send(httpRequest, HttpResponse.BodyHandlers.ofString()); - logger.debug("Response status code: {}", response.statusCode()); - return (response.statusCode() >= 200) && (response.statusCode() <= 202); + sardine.put(url, multipartFile.getBytes()); + return sardine.exists(url); } - catch (IOException | InterruptedException e) { - throw new RuntimeException(e); + catch (IOException e) { + throw new RuntimeException("Failed to put or check file in WebDAV", e); + } + finally { + try { + sardine.shutdown(); + } + catch (IOException e) { + logger.error("Failed to shutdown WebDAV client", e); + } } } @Retryable(value = {IOException.class, InterruptedException.class}, - backoff = @Backoff(delay = 500L)) + backoff = @Backoff(delayExpression = "${webdav.retry.delay:500}")) public ResponseEntity webDavDownloadFile(String url) { try { HttpClient httpClient = HttpClient.newBuilder() @@ -103,4 +107,55 @@ public class WebDavClient { String path = URI.create(url).getPath(); return path.substring(path.lastIndexOf('/') + 1); } + + @Retryable(value = {IOException.class}, backoff = @Backoff(delayExpression = "${webdav.retry.delay:500}")) + public void deleteFilesOlderThan(long seconds, String url, String username, String password, + String... extensions) { + Sardine sardine = initClient(username, password); + + try { + List resources = sardine.list(url); + resources.stream().filter(getPredicate(extensions)) + .forEach(davResource -> { + long age = System.currentTimeMillis() - davResource.getModified().getTime(); + + if (age > seconds * 1000) { + try { + sardine.delete(url + davResource.getPath()); + } + catch (IOException e) { + throw new RuntimeException("Failed to delete file " + davResource.getName(), e); + } + } + }); + } + catch (IOException e) { + throw new RuntimeException("Failed to delete old files from WebDAV", e); + } + finally { + try { + sardine.shutdown(); + } + catch (IOException e) { + logger.error("Failed to shutdown WebDAV client", e); + } + } + } + + private Sardine initClient(String username, String password) { + return SardineFactory.begin(username, password); + } + + private Predicate getPredicate(String... extensions) { + if (extensions.length == 0) { + return davResource -> true; + } + + return davResource -> { + AtomicBoolean condition = new AtomicBoolean(false); + Arrays.stream(extensions).forEach(extension -> condition.set( + condition.get() || davResource.getName().endsWith(extension))); + return condition.get(); + }; + } } diff --git a/backend/src/main/java/ervu/service/fileupload/EmployeeInfoFileUploadService.java b/backend/src/main/java/ervu/service/fileupload/EmployeeInfoFileUploadService.java index 5405b2a6..c0d9a538 100644 --- a/backend/src/main/java/ervu/service/fileupload/EmployeeInfoFileUploadService.java +++ b/backend/src/main/java/ervu/service/fileupload/EmployeeInfoFileUploadService.java @@ -43,7 +43,7 @@ public class EmployeeInfoFileUploadService { private static final Logger logger = LoggerFactory.getLogger(EmployeeInfoFileUploadService.class); private static final String FORMAT = "dd.MM.yyyy HH:mm:ss"; - private final WebDavClient fileWebDavUploadClient; + private final WebDavClient webDavClient; private final EmployeeInfoKafkaMessageService employeeInfoKafkaMessageService; private final KafkaTemplate kafkaTemplate; private final InteractionService interactionService; @@ -56,12 +56,12 @@ public class EmployeeInfoFileUploadService { private String url; public EmployeeInfoFileUploadService( - WebDavClient fileWebDavUploadClient, + WebDavClient webDavClient, EmployeeInfoKafkaMessageService employeeInfoKafkaMessageService, @Qualifier("avTemplate") KafkaTemplate kafkaTemplate, InteractionService interactionService, UlDataService ulDataService, JwtTokenService jwtTokenService) { - this.fileWebDavUploadClient = fileWebDavUploadClient; + this.webDavClient = webDavClient; this.employeeInfoKafkaMessageService = employeeInfoKafkaMessageService; this.kafkaTemplate = kafkaTemplate; this.interactionService = interactionService; @@ -73,7 +73,7 @@ public class EmployeeInfoFileUploadService { String fileUploadUrl = this.url + "/" + getNewFilename(multipartFile.getOriginalFilename()); LocalDateTime now = LocalDateTime.now(); - if (this.fileWebDavUploadClient.webDavUploadFile(fileUploadUrl, multipartFile)) { + if (this.webDavClient.uploadFile(fileUploadUrl, multipartFile)) { FileStatus fileStatus = new FileStatus(); fileStatus.setStatus("Загрузка."); fileStatus.setCode("01"); diff --git a/backend/src/main/java/ervu/service/scheduer/EsnsiOkopfSchedulerService.java b/backend/src/main/java/ervu/service/scheduler/EsnsiOkopfSchedulerService.java similarity index 76% rename from backend/src/main/java/ervu/service/scheduer/EsnsiOkopfSchedulerService.java rename to backend/src/main/java/ervu/service/scheduler/EsnsiOkopfSchedulerService.java index e2b884e0..1ec89c07 100644 --- a/backend/src/main/java/ervu/service/scheduer/EsnsiOkopfSchedulerService.java +++ b/backend/src/main/java/ervu/service/scheduler/EsnsiOkopfSchedulerService.java @@ -1,4 +1,4 @@ -package ervu.service.scheduer; +package ervu.service.scheduler; /** * @author Artyom Hackimullin diff --git a/backend/src/main/java/ervu/service/scheduer/EsnsiOkopfSchedulerServiceImpl.java b/backend/src/main/java/ervu/service/scheduler/EsnsiOkopfSchedulerServiceImpl.java similarity index 98% rename from backend/src/main/java/ervu/service/scheduer/EsnsiOkopfSchedulerServiceImpl.java rename to backend/src/main/java/ervu/service/scheduler/EsnsiOkopfSchedulerServiceImpl.java index 6f09fbb5..cdd4da77 100644 --- a/backend/src/main/java/ervu/service/scheduer/EsnsiOkopfSchedulerServiceImpl.java +++ b/backend/src/main/java/ervu/service/scheduler/EsnsiOkopfSchedulerServiceImpl.java @@ -1,4 +1,4 @@ -package ervu.service.scheduer; +package ervu.service.scheduler; import java.util.Arrays; import java.util.List; diff --git a/backend/src/main/java/ervu/service/scheduler/WebDavSchedulerService.java b/backend/src/main/java/ervu/service/scheduler/WebDavSchedulerService.java new file mode 100644 index 00000000..ff2b3770 --- /dev/null +++ b/backend/src/main/java/ervu/service/scheduler/WebDavSchedulerService.java @@ -0,0 +1,37 @@ +package ervu.service.scheduler; + +import ervu.client.fileupload.WebDavClient; +import net.javacrumbs.shedlock.core.SchedulerLock; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; + +/** + * @author gulnaz + */ +@Service +public class WebDavSchedulerService { + + private final WebDavClient webDavClient; + + @Value("${file.webdav.upload.url}") + private String url; + @Value("${file.webdav.upload.username}") + private String username; + @Value("${file.webdav.upload.password}") + private String password; + @Value("${file.webdav.lifetime.seconds:300}") + private long fileLifetime; + @Value("${file.webdav.extensions:}") + private String[] fileExtensions; + + public WebDavSchedulerService(WebDavClient webDavClient) { + this.webDavClient = webDavClient; + } + + @Scheduled(cron = "${webdav.cleanup.cron:0 0 0 * * *}") + @SchedulerLock + public void deleteOldFiles() { + webDavClient.deleteFilesOlderThan(fileLifetime, url, username, password, fileExtensions); + } +} diff --git a/config/micord.env b/config/micord.env index 9e25cbb5..40ce97a5 100644 --- a/config/micord.env +++ b/config/micord.env @@ -49,4 +49,9 @@ ERVU_FILE_UPLOAD_MAX_REQUEST_SIZE=6291456 ERVU_FILE_UPLOAD_FILE_SIZE_THRESHOLD=0 ESIA_TOKEN_CLEAR_CRON=0 0 */1 * * * -COOKIE_PATH=/ul \ No newline at end of file +COOKIE_PATH=/ul + +WEBDAV_CLEANUP_CRON=0 0 0 * * * +WEBDAV_RETRY_DELAY=500 +FILE_WEBDAV_LIFETIME_SECONDS=300 +FILE_WEBDAV_EXTENSIONS=csv,xlsx diff --git a/config/standalone/dev/standalone.xml b/config/standalone/dev/standalone.xml index 1e3a794c..8da2d622 100644 --- a/config/standalone/dev/standalone.xml +++ b/config/standalone/dev/standalone.xml @@ -93,6 +93,10 @@ + + + + diff --git a/pom.xml b/pom.xml index dd816620..03939610 100644 --- a/pom.xml +++ b/pom.xml @@ -387,6 +387,11 @@ log4j-web 2.23.1 + + com.github.lookfirst + sardine + 5.12 +