From 0e131fc247c22934f3b6508072dbbf83b6fa9551 Mon Sep 17 00:00:00 2001 From: Maksim Tereshin Date: Wed, 18 Dec 2024 13:46:13 +0100 Subject: [PATCH] Add new api method --- .../org/micord/controller/ApiController.java | 24 ++ .../java/org/micord/enums/ConfigType.java | 1 + .../models/requests/RequestParameters.java | 16 ++ .../java/org/micord/service/ApiService.java | 6 + .../org/micord/service/RequestService.java | 219 ++++++++++++++++++ 5 files changed, 266 insertions(+) create mode 100644 config-data-executor/src/main/java/org/micord/models/requests/RequestParameters.java diff --git a/config-data-executor/src/main/java/org/micord/controller/ApiController.java b/config-data-executor/src/main/java/org/micord/controller/ApiController.java index 53ff3f8..8fc94f5 100644 --- a/config-data-executor/src/main/java/org/micord/controller/ApiController.java +++ b/config-data-executor/src/main/java/org/micord/controller/ApiController.java @@ -3,6 +3,7 @@ package org.micord.controller; import org.micord.enums.ConfigType; import org.micord.exceptions.ValidationException; import org.micord.models.requests.DownloadCSVRequest; +import org.micord.models.requests.RequestParameters; import org.micord.service.ApiService; import org.micord.service.ValidationService; import org.slf4j.Logger; @@ -39,6 +40,21 @@ public class ApiController { @Autowired private ValidationService validationService; + @PostMapping("/removeMilitaryDraftNotices") + public ResponseEntity removeMilitaryDraftNotices(@RequestBody RequestParameters request) throws SQLException, FileNotFoundException { + + validateRequestDates(request); + + List ids = request.getIds(); + validationService.validateAll(ids); + + logger.debug("Starting removeMilitaryDraftNotices process for ids: {}", ids); + apiService.process(ConfigType.REMOVE_MILITARY_DRAFT_NOTICES, request); + logger.debug("Finished removeMilitaryDraftNotices process for ids: {}", ids); + + return ResponseEntity.ok("Операция \"Удаление повесток\" завершена успешно."); + } + @PostMapping("/block") public ResponseEntity block(@RequestBody List ids) throws SQLException, FileNotFoundException { validationService.validateAll(ids); @@ -113,6 +129,14 @@ public class ApiController { } } + private void validateRequestDates(RequestParameters request) { + if (request.getStartDate() != null && request.getEndDate() != null) { + if (request.getStartDate().isAfter(request.getEndDate())) { + throw new IllegalArgumentException("Start date must be before end date"); + } + } + } + @GetMapping("/listDownloadTypes") public ResponseEntity> listDownloadTypes() throws FileNotFoundException { diff --git a/config-data-executor/src/main/java/org/micord/enums/ConfigType.java b/config-data-executor/src/main/java/org/micord/enums/ConfigType.java index 3ff1925..5ebe34d 100644 --- a/config-data-executor/src/main/java/org/micord/enums/ConfigType.java +++ b/config-data-executor/src/main/java/org/micord/enums/ConfigType.java @@ -6,6 +6,7 @@ import lombok.Getter; public enum ConfigType { BLOCK("block"), + REMOVE_MILITARY_DRAFT_NOTICES("removeMilitaryDraftNotices"), UNBLOCK("unblock"), REMOVE_FROM_SYSTEM("removeFromSystem"), REMOVE_FROM_CALL_LIST("removeFromCallList"), diff --git a/config-data-executor/src/main/java/org/micord/models/requests/RequestParameters.java b/config-data-executor/src/main/java/org/micord/models/requests/RequestParameters.java new file mode 100644 index 0000000..0479158 --- /dev/null +++ b/config-data-executor/src/main/java/org/micord/models/requests/RequestParameters.java @@ -0,0 +1,16 @@ +package org.micord.models.requests; + + +import lombok.Data; + +import java.time.LocalDate; +import java.util.List; + +@Data +public class RequestParameters { + private String type; + private List ids; + private LocalDate startDate; + private LocalDate endDate; +} + diff --git a/config-data-executor/src/main/java/org/micord/service/ApiService.java b/config-data-executor/src/main/java/org/micord/service/ApiService.java index 97e5c6c..e5b29a4 100644 --- a/config-data-executor/src/main/java/org/micord/service/ApiService.java +++ b/config-data-executor/src/main/java/org/micord/service/ApiService.java @@ -3,6 +3,7 @@ package org.micord.service; import org.micord.enums.ConfigType; import org.micord.models.requests.DownloadCSVRequest; import org.micord.models.requests.DownloadRequest; +import org.micord.models.requests.RequestParameters; import org.micord.models.requests.Requests; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -35,6 +36,11 @@ public class ApiService { sqlAndAqlService.processSqlAndAqlRequests(config, ids); } + public void process(ConfigType methodName, RequestParameters parameters) throws FileNotFoundException { + Requests config = configService.getConfig(methodName, Requests.class); + sqlAndAqlService.processSqlAndAqlRequests(config, parameters); + } + public File download(ConfigType methodName, DownloadCSVRequest request) throws IOException { Requests config = configService.getConfig(methodName, Requests.class); diff --git a/config-data-executor/src/main/java/org/micord/service/RequestService.java b/config-data-executor/src/main/java/org/micord/service/RequestService.java index 3bc4afa..7285745 100644 --- a/config-data-executor/src/main/java/org/micord/service/RequestService.java +++ b/config-data-executor/src/main/java/org/micord/service/RequestService.java @@ -31,6 +31,7 @@ import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; +import java.time.LocalDate; import java.util.*; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -54,6 +55,77 @@ public class RequestService { } } + public void processS3Requests(List s3Requests, RequestParameters parameters) { + logger.info("A. Starting processing of S3 requests"); + + for (S3Request request : s3Requests) { + processS3Request(request, parameters); + } + } + + private void processS3Request(S3Request request, RequestParameters parameters) { + logger.info("B. Starting processing of single S3 request"); + try { + List files = new ArrayList<>(); + List ids = parameters.getIds(); + + if (request.getRequestArguments() != null && !request.getRequestArguments().isEmpty()) { + for (RequestArgument argument : request.getRequestArguments()) { + try (Connection connection = DatabaseConnection.getConnection( + argument.getRequestArgumentConnectionParams())) { + + Map query = buildSqlQueryForS3(argument.getRequestArgumentURL(), parameters); + logger.info("C. Calling query {} for ids {}: ", query.get("requestURL"), ids); + logger.debug("Starting fetching paths from database for S3 request"); + long startExecTime = System.currentTimeMillis(); + + List result = fetchFileListFromDatabaseSQL(connection, (String) query.get("requestURL")); + String formattedResult = IntStream.range(0, result.size()) + .mapToObj(i -> (i + 1) + ". " + result.get(i)) + .collect(Collectors.joining("\n")); + + logger.info("D. Found files for query {}:\n{}", query.get("requestURL"), formattedResult); + + if (result != null && !result.isEmpty()) { + files.addAll(result); + } + long endExecTime = System.currentTimeMillis(); + logger.debug("Paths fetched in {} ms", endExecTime - startExecTime); + } catch (SQLException e) { + logger.error("Failed to execute query for RequestArgument: {}", argument.getRequestArgumentURL(), e); + throw new RuntimeException("Database query error for argument: " + argument.getRequestArgumentURL(), e); + } + } + } + + String formattedFiles = IntStream.range(0, files.size()) + .mapToObj(i -> (i + 1) + ". " + files.get(i)) + .collect(Collectors.joining("\n")); + + logger.info("E. Found files for ids {}:\n{}", ids, formattedFiles); + + if (files.isEmpty()) { + logger.warn("No files found for S3 request {}", request); + } else { + + for (String file : files) { + logger.info("F. Starting query S3 for file: {}", file); + try { + processFileForS3Request(request, file); + } catch (RuntimeException e) { + logger.error("Error processing file: {}", file, e); + throw e; // Rethrow to propagate for exception handling + } + } + + } + + } catch (Exception e) { + logger.error("Failed to process S3 request: {}", request, e); + throw e; // Rethrow exception to propagate to the handler + } + } + private void processS3Request(S3Request request, List ids) { logger.info("B. Starting processing of single S3 request"); try { @@ -177,6 +249,25 @@ public class RequestService { } } + @Transactional + public void processSqlAndAqlRequests(Requests config, RequestParameters parameters) { + logger.debug("Starting transactional processing of requests"); + if (config.getSqlRequests() != null) { + for (SqlRequest request : config.getSqlRequests()) { + processSqlRequests(request, parameters); + } + } + + if (config.getAqlRequests() != null) { + for (AqlRequest request : config.getAqlRequests()) { + processAqlRequests(request, parameters.getIds()); + } + } + if (config.getS3Requests() != null && !config.getS3Requests().isEmpty()) { + processS3Requests(config.getS3Requests(), parameters); + } + } + private void processSqlRequests(SqlRequest request, List ids) { logger.debug("Starting transactional processing of SQL requests"); Map query = buildSqlQuery(request, ids); @@ -204,6 +295,34 @@ public class RequestService { } } + private void processSqlRequests(SqlRequest request, RequestParameters parameters) { + logger.debug("Starting transactional processing of SQL requests"); + Map query = buildSqlQuery(request, parameters); + List ids = parameters.getIds(); + logger.debug("Opening connection for SQL Request: {}", request.getRequestURL()); + long startExecTime = System.currentTimeMillis(); + try (Connection connection = DatabaseConnection.getConnection( + request.getSqlConnectionParams())) { + String requestURL = (String) query.get("requestURL"); + executeSqlQuery(connection, requestURL); + + List queryIds = (List) query.get("ids"); + if (queryIds != null && !queryIds.isEmpty()) { + ids.addAll(queryIds); + } else { + logger.warn("No IDs found for the query"); + } + long endExecTime = System.currentTimeMillis(); + logger.debug("SQL request executed in {} ms", endExecTime - startExecTime); + + logger.info("Successfully executed query {} for IDs: ({})", requestURL, String.join(", ", ids)); + } + catch (SQLException e) { + logger.error("SQL execution failed for query: {}", query, e); + throw new RuntimeException("Error executing SQL query", e); + } + } + private Map buildSqlQueryForS3(String requestURL, List ids) { logger.debug("Starting building SQL query for request: {}", requestURL); long startExecTime = System.currentTimeMillis(); @@ -229,6 +348,48 @@ public class RequestService { return resultMap; } + private Map buildSqlQueryForS3(String url, RequestParameters parameters) { + logger.debug("Starting building SQL query for request: {}", url); + long startExecTime = System.currentTimeMillis(); + Map resultMap = new HashMap<>(); + String endpointArguments; + + String requestURL = prepareDatesFilterInRequestURL(url, parameters.getStartDate(), parameters.getEndDate()); + + List ids = parameters.getIds(); + + if (requestURL.contains(":=")) { + endpointArguments = "'{" + ids.stream() + .map(String::trim) + .collect(Collectors.joining(", ")) + "}'"; + } else { + endpointArguments = "(" + ids.stream() + .map(s -> "'" + s.trim() + "'") + .collect(Collectors.joining(", ")) + ")"; + } + + resultMap.put("requestURL", requestURL + .replace("${endpointArguments}", endpointArguments)); + + long endExecTime = System.currentTimeMillis(); + logger.debug("SQL query for S3 built in {} ms", endExecTime - startExecTime); + + return resultMap; + } + + private String prepareDatesFilterInRequestURL(String requestURL, LocalDate startDate, LocalDate endDate) { + + if (startDate != null) { + requestURL = requestURL.replace("${startDate}", startDate.toString()); + } + + if (endDate != null) { + requestURL = requestURL.replace("${endDate}", endDate.toString()); + } + + return requestURL; + } + private Map buildSqlQuery(BaseRequest request, List ids) { logger.debug("Starting building SQL query for request: {}", request.getRequestURL()); long startExecTime = System.currentTimeMillis(); @@ -286,6 +447,64 @@ public class RequestService { return resultMap; } + private Map buildSqlQuery(BaseRequest request, RequestParameters parameters) { + logger.debug("Starting building SQL query for request: {}", request.getRequestURL()); + long startExecTime = System.currentTimeMillis(); + Map resultMap = new HashMap<>(); + String endpointArguments; + + String requestURL = prepareDatesFilterInRequestURL(request.getRequestURL(), parameters.getStartDate(), parameters.getEndDate()); + List ids = parameters.getIds(); + + if (requestURL.contains(":=")) { + endpointArguments = "'{" + ids.stream() + .map(String::trim) + .collect(Collectors.joining(", ")) + "}'"; + } else { + endpointArguments = "(" + ids.stream() + .map(s -> "'" + s.trim() + "'") + .collect(Collectors.joining(", ")) + ")"; + } + + if (request.getRequestArguments() != null && !request.getRequestArguments().isEmpty()) { + for (RequestArgument argument : request.getRequestArguments()) { + + if (argument.getRequestArgumentConnectionParams() != null) { + logger.debug("Opening connection for SQL RequestArgument: {}", argument.getRequestArgumentName()); + try (Connection connection = DatabaseConnection.getConnection( + argument.getRequestArgumentConnectionParams())) { + String query = argument.getRequestArgumentURL(); + List result = fetchFileListFromDatabaseSQL(connection, query); + + resultMap.put("ids", result); + + + if (result != null && !result.isEmpty()) { + String resultSet = "(" + result.stream() + .map(s -> "'" + s.trim() + "'") + .collect(Collectors.joining(", ")) + ")"; + + requestURL = requestURL.replace("${" + argument.getRequestArgumentName() + "}", resultSet); + + } + + } + catch (SQLException e) { + logger.error("Failed to execute query for RequestArgument", e); + } + } + } + } + + resultMap.put("requestURL", requestURL + .replace("${endpointArguments}", endpointArguments)); + + long endExecTime = System.currentTimeMillis(); + logger.debug("SQL query built in {} ms", endExecTime - startExecTime); + + return resultMap; + } + private boolean executeSqlQuery(Connection connection, String query) throws SQLException { try (PreparedStatement stmt = connection.prepareStatement(query)) {