micord/ervu-eks
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add dates and loggers

Browse source
This commit is contained in:
Maksim Tereshin 2024-11-21 08:58:41 +01:00
parent 725a26478c
commit d475f482d6
No known key found for this signature in database
6 changed files with 80 additions and 11 deletions
Download patch file Download diff file Expand all files Collapse all files

14
config-data-executor/src/main/java/org/micord/config/DatabaseConnection.java
View file

@ -2,6 +2,8 @@ package org.micord.config;
import com.atomikos.jdbc.AtomikosDataSourceBean;
import org.micord.models.SqlConnectionParams;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.sql.DataSource;
import java.sql.Connection;
@ -15,19 +17,28 @@ import java.util.Properties;
*/
public class DatabaseConnection {
private static final Logger logger = LoggerFactory.getLogger(DatabaseConnection.class);
private static final Map<String, DataSource> dataSources = new HashMap<>();
public static Connection getConnection(SqlConnectionParams params) throws SQLException {
logger.debug("Get connection for params: {}", params);
long startExecTime = System.currentTimeMillis();
try {
Class.forName(params.getJdbcDriverClassName());
} catch (ClassNotFoundException e) {
throw new SQLException("Unable to load the JDBC driver class", e);
}
long endExecTime = System.currentTimeMillis();
logger.debug("Time to load JDBC driver class: {} ms", endExecTime - startExecTime);
return getXaDataSource(params).getConnection();
}
public static DataSource getXaDataSource(SqlConnectionParams params) {
logger.debug("Getting XA data source for params: {}", params);
long startExecTime = System.currentTimeMillis();
String database = params.getJdbcDatabase();
if (!dataSources.containsKey(database)) {
@ -46,6 +57,9 @@ public class DatabaseConnection {
dataSources.put(database, xaDataSource);
}
long endExecTime = System.currentTimeMillis();
logger.debug("Time to get XA data source: {} ms", endExecTime - startExecTime);
return dataSources.get(database);
}

22
config-data-executor/src/main/java/org/micord/controller/ApiController.java
View file

@ -8,6 +8,8 @@ import java.util.List;
import org.micord.models.DownloadCSVRequest;
import org.micord.service.ApiService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.io.InputStreamResource;
import org.springframework.core.io.Resource;
@ -23,44 +25,60 @@ import org.springframework.web.bind.annotation.*;
@RequestMapping("/api")
public class ApiController {
private static final Logger logger = LoggerFactory.getLogger(ApiController.class);
@Autowired
private ApiService apiService;
@PostMapping("/block")
public ResponseEntity<?> block(@RequestBody List<String> ids) throws FileNotFoundException {
logger.debug("Starting block process for ids: {}", ids);
apiService.process("block", ids);
logger.debug("Finished block process for ids: {}", ids);
return ResponseEntity.ok("");
}
@PostMapping("/unblock")
public ResponseEntity<?> unblock(@RequestBody List<String> ids) throws FileNotFoundException {
logger.debug("Starting unblock process for ids: {}", ids);
apiService.process("unblock", ids);
logger.debug("Finished unblock process for ids: {}", ids);
return ResponseEntity.ok("");
}
@PostMapping("/removeFromSystem")
public ResponseEntity<?> removeFromSystem(@RequestBody List<String> ids)
throws FileNotFoundException {
logger.debug("Starting removeFromSystem process for ids: {}", ids);
apiService.process("removeFromSystem", ids);
logger.debug("Finished removeFromSystem process for ids: {}", ids);
return ResponseEntity.ok("");
}
@PostMapping("/removeFromCallList")
public ResponseEntity<?> removeFromCallList(@RequestBody List<String> ids)
throws FileNotFoundException {
logger.debug("Starting removeFromCallList process for ids: {}", ids);
apiService.process("removeFromCallList", ids);
logger.debug("Finished removeFromCallList process for ids: {}", ids);
return ResponseEntity.ok("");
}
@PostMapping("/downloadCSV")
public ResponseEntity<Resource> downloadCSV(@RequestBody DownloadCSVRequest request)
throws IOException {
logger.debug("Starting downloadCSV process for request: {}", request.getType());
if (request.getStartDate() != null && request.getEndDate() != null) {
if (request.getStartDate().isAfter(request.getEndDate())) {
throw new IllegalArgumentException("Start date must be before end date");
}
}
File csvFile = apiService.download("downloadCSV", request);
InputStreamResource resource = new InputStreamResource(new FileInputStream(csvFile));
logger.debug("Finished downloadCSV process for request: {}. Sending to user...", request.getType());
return ResponseEntity.ok()
.header(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=" + csvFile.getName())
.contentType(MediaType.parseMediaType("text/csv"))

3
config-data-executor/src/main/java/org/micord/models/DownloadCSVRequest.java
View file

@ -3,11 +3,14 @@ package org.micord.models;
import lombok.Data;
import java.time.LocalDate;
import java.util.List;
@Data
public class DownloadCSVRequest {
private String type;
private List<String> ids;
private LocalDate startDate;
private LocalDate endDate;
}

7
config-data-executor/src/main/java/org/micord/service/ApiService.java
View file

@ -2,6 +2,8 @@ package org.micord.service;
import org.micord.models.*;
import org.micord.utils.ConfigLoader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
@ -14,6 +16,8 @@ import java.util.stream.Collectors;
@Service
public class ApiService {
private static final Logger logger = LoggerFactory.getLogger(ApiService.class);
@Autowired
private ConfigLoader configLoader;
@ -41,7 +45,7 @@ public class ApiService {
.findFirst()
.orElseThrow(() -> new IllegalArgumentException("Invalid download type: " + type));
return downloadService.download(selectedRequest, ids);
return downloadService.download(selectedRequest, ids, request.getStartDate(), request.getEndDate());
}
public List<String> getDownloadTypes(String methodName) throws FileNotFoundException {
@ -53,6 +57,7 @@ public class ApiService {
}
private Requests getConfig(String methodName) throws FileNotFoundException {
logger.debug("Loading configuration for method: {}", methodName);
Optional<Requests> optionalConfig = configLoader.loadConfigIfModified(methodName);
if (optionalConfig.isEmpty()) {

23
config-data-executor/src/main/java/org/micord/service/DownloadService.java
View file

@ -14,6 +14,7 @@ import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.time.LocalDate;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@ -25,13 +26,13 @@ public class DownloadService {
private static final Logger logger = LoggerFactory.getLogger(DownloadService.class);
public File download(DownloadRequest selectedRequest, List<String> ids) {
public File download(DownloadRequest selectedRequest, List<String> ids, LocalDate startDate, LocalDate endDate) {
return processDownloadRequest(selectedRequest, ids);
return processDownloadRequest(selectedRequest, ids, startDate, endDate);
}
private File processDownloadRequest(DownloadRequest request, List<String> ids) {
Map<String, Object> query = buildSqlQuery(request, ids);
private File processDownloadRequest(DownloadRequest request, List<String> ids, LocalDate startDate, LocalDate endDate) {
Map<String, Object> query = buildSqlQuery(request, ids, startDate, endDate);
try (Connection connection = DatabaseConnection.getConnection(
request.getSqlConnectionParams())) {
String requestURL = (String) query.get("requestURL");
@ -57,15 +58,14 @@ public class DownloadService {
return null;
}
private Map<String, Object> buildSqlQuery(DownloadRequest request, List<String> ids) {
private Map<String, Object> buildSqlQuery(DownloadRequest request, List<String> ids, LocalDate startDate, LocalDate endDate) {
Map<String, Object> resultMap = new HashMap<>();
String endpointArguments;
String requestURL = request.getRequestURL();
String requestURL = prepareRequestURL(request, startDate, endDate);
if (ids == null || ids.isEmpty()) {
resultMap.put("requestURL", requestURL
.replace("${DB}", request.getSqlConnectionParams().getJdbcDatabase())
.replace("where id in ${endpointArguments}", ""));
return resultMap;
@ -105,18 +105,25 @@ public class DownloadService {
}
catch (SQLException e) {
logger.error("Failed to execute query for RequestArgument", e);
throw new RuntimeException("Error executing database query: " + argument.getRequestArgumentURL(), e);
}
}
}
}
resultMap.put("requestURL", requestURL
.replace("${DB}", request.getSqlConnectionParams().getJdbcDatabase())
.replace("${endpointArguments}", endpointArguments));
return resultMap;
}
private String prepareRequestURL(DownloadRequest request, LocalDate startDate, LocalDate endDate) {
return request.getRequestURL()
.replace("${startDate}", startDate.toString())
.replace("${endDate}", endDate.toString())
.replace("${DB}", request.getSqlConnectionParams().getJdbcDatabase());
}
private List<String[]> executeSqlQuery(Connection connection, String query) throws SQLException {
List<String[]> results = new ArrayList<>();
try (PreparedStatement stmt = connection.prepareStatement(query);

22
config-data-executor/src/main/java/org/micord/service/RequestService.java
View file

@ -41,6 +41,7 @@ public class RequestService {
private HttpClient httpClient;
public void processS3Requests(List<S3Request> s3Requests, List<String> ids) {
logger.debug("Starting processing of S3 requests");
if (s3Requests != null) {
s3Requests.forEach(request -> {
List<CompletableFuture<Void>> futures = ids.stream()
@ -58,6 +59,7 @@ public class RequestService {
}
private void processS3Request(S3Request request, String id) {
logger.debug("Starting processing of S3 request for id: {}", id);
try {
List<String> files = new ArrayList<>();
@ -66,10 +68,14 @@ public class RequestService {
try (Connection connection = DatabaseConnection.getConnection(
argument.getRequestArgumentConnectionParams())) {
String query = argument.getRequestArgumentURL();
logger.debug("Starting fetching paths from database for S3 request");
long startExecTime = System.currentTimeMillis();
List<String> result = fetchFileListFromDatabaseSQL(connection, query);
if (result != null && !result.isEmpty()) {
files.addAll(result);
}
long endExecTime = System.currentTimeMillis();
logger.debug("Paths fetched in {} ms", endExecTime - startExecTime);
}
catch (SQLException e) {
logger.error("Failed to execute query for RequestArgument", e);
@ -79,12 +85,16 @@ public class RequestService {
files.forEach(file -> {
HttpRequest httpRequest;
logger.debug("Starting building HTTP request for S3 request");
long startExecTime = System.currentTimeMillis();
try {
httpRequest = S3HttpConnection.buildHttpRequest(request, file);
}
catch (Exception e) {
throw new RuntimeException(e);
}
long endExecTime = System.currentTimeMillis();
logger.debug("HTTP request built in {} ms", endExecTime - startExecTime);
httpClient.sendAsync(httpRequest, HttpResponse.BodyHandlers.ofString())
.thenAccept(response -> {
@ -112,6 +122,7 @@ public class RequestService {
@Transactional
public void processSqlAndAqlRequests(Requests config, List<String> ids) {
logger.debug("Starting transactional processing of requests");
if (config.getSqlRequests() != null) {
for (SqlRequest request : config.getSqlRequests()) {
processSqlRequests(request, ids);
@ -127,7 +138,10 @@ public class RequestService {
}
private void processSqlRequests(SqlRequest request, List<String> ids) {
logger.debug("Starting transactional processing of SQL requests");
Map<String, Object> query = buildSqlQuery(request, ids);
logger.debug("Opening connection for SQL Request: {}", request.getRequestURL());
long startExecTime = System.currentTimeMillis();
try (Connection connection = DatabaseConnection.getConnection(
request.getSqlConnectionParams())) {
String requestURL = (String) query.get("requestURL");
@ -139,6 +153,8 @@ public class RequestService {
} else {
logger.warn("No IDs found for the query");
}
long endExecTime = System.currentTimeMillis();
logger.debug("SQL request executed in {} ms", endExecTime - startExecTime);
logger.info("Successfully executed query {} for IDs: ({})", requestURL, String.join(", ", ids));
}
@ -148,6 +164,8 @@ public class RequestService {
}
private Map<String, Object> buildSqlQuery(SqlRequest request, List<String> ids) {
logger.debug("Starting building SQL query for request: {}", request.getRequestURL());
long startExecTime = System.currentTimeMillis();
Map<String, Object> resultMap = new HashMap<>();
String endpointArguments;
@ -167,6 +185,7 @@ private Map<String, Object> buildSqlQuery(SqlRequest request, List<String> ids)
for (RequestArgument argument : request.getRequestArguments()) {
if (argument.getRequestArgumentConnectionParams() != null) {
logger.debug("Opening connection for SQL RequestArgument: {}", argument.getRequestArgumentName());
try (Connection connection = DatabaseConnection.getConnection(
argument.getRequestArgumentConnectionParams())) {
String query = argument.getRequestArgumentURL();
@ -196,6 +215,9 @@ private Map<String, Object> buildSqlQuery(SqlRequest request, List<String> ids)
.replace("${DB}", request.getSqlConnectionParams().getJdbcDatabase())
.replace("${endpointArguments}", endpointArguments));
long endExecTime = System.currentTimeMillis();
logger.debug("SQL query built in {} ms", endExecTime - startExecTime);
return resultMap;
}