micord/ervu-secret
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix

Browse source
This commit is contained in:
kochetkov 2024-09-19 15:36:45 +03:00
parent 702920da9f
commit 8fe886c626
3 changed files with 254 additions and 227 deletions
Download patch file Download diff file Expand all files Collapse all files

2
сonfig-data-executor/src/main/java/org/micord/service/ApiService.java
View file

@ -1,6 +1,5 @@
package org.micord.service; package org.micord.service;
import org.micord.exceptions.FileNotModifiedException;
import org.micord.models.*; import org.micord.models.*;
import org.micord.utils.ConfigLoader; import org.micord.utils.ConfigLoader;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
@ -8,7 +7,6 @@ import org.springframework.stereotype.Service;
import java.io.FileNotFoundException; import java.io.FileNotFoundException;
import java.util.*; import java.util.*;
import java.util.concurrent.CompletableFuture;
@Service @Service
public class ApiService { public class ApiService {

63
сonfig-data-executor/src/main/java/org/micord/service/S3Service.java
View file

@ -30,69 +30,6 @@ public class S3Service {
@Autowired @Autowired
private SqlAqlService sqlAndAqlService; private SqlAqlService sqlAndAqlService;
@Autowired
private HttpClient httpClient;
public void processS3Requests(List<S3Request> s3Requests, List<String> ids) {
if (s3Requests != null) {
s3Requests.forEach(request -> {
List<CompletableFuture<Void>> futures = ids.stream()
.map(id -> CompletableFuture.runAsync(() -> processS3Request(request, id)))
.toList();
CompletableFuture.allOf(futures.toArray(new CompletableFuture[0]))
.thenRun(() -> logger.info("Successfully processed all S3 requests."))
.exceptionally(ex -> {
logger.error("Failed to process S3 requests", ex);
return null;
});
});
}
}
private void processS3Request(S3Request request, String id) {
try {
List<String> files = new ArrayList<>();
if (request.getRequestArguments() != null && !request.getRequestArguments().isEmpty()) {
for (RequestArgument argument : request.getRequestArguments()) {
try (Connection connection = DatabaseConnection.getConnection(argument.getSqlConnectionParams())) {
String query = argument.getRequestURL();
List<String> result = sqlAndAqlService.fetchFileListFromDatabaseSQL(connection, query);
if (result != null && !result.isEmpty()) {
files.addAll(result);
}
} catch (SQLException e) {
logger.error("Failed to execute query for RequestArgument", e);
}
}
}
files.forEach(file -> {
HttpRequest httpRequest;
try {
httpRequest = S3HttpConnection.buildHttpRequest(request, file);
} catch (Exception e) {
throw new RuntimeException(e);
}
httpClient.sendAsync(httpRequest, HttpResponse.BodyHandlers.ofString())
.thenAccept(response -> {
if (response.statusCode() == HttpURLConnection.HTTP_NO_CONTENT || response.statusCode() == HttpURLConnection.HTTP_OK) {
logger.info("Successfully deleted object for ID {}", id);
} else {
logger.error("Failed to delete object for ID {}. Response code: {}", id, response.statusCode());
}
})
.exceptionally(ex -> {
logger.error("Failed to delete object for ID {}", id, ex);
return null;
});
});
} catch (Exception e) {
logger.error("Failed to process S3 request for id: {}", id, e);
}
}
} }

156
сonfig-data-executor/src/main/java/org/micord/service/SqlAqlService.java
View file

@ -1,24 +1,9 @@
package org.micord.service; package org.micord.service;
import com.arangodb.ArangoCursor; import java.net.HttpURLConnection;
import com.arangodb.ArangoDBException; import java.net.http.HttpClient;
import com.arangodb.ArangoDatabase; import java.net.http.HttpRequest;
import com.arangodb.entity.StreamTransactionEntity; import java.net.http.HttpResponse;
import com.arangodb.model.AqlQueryOptions;
import com.arangodb.model.StreamTransactionOptions;
import com.arangodb.model.TransactionOptions;
import org.micord.config.ArangoDBConnection;
import org.micord.config.DatabaseConnection;
import org.micord.models.AqlRequest;
import org.micord.models.RequestArgument;
import org.micord.models.Requests;
import org.micord.models.SqlRequest;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import java.sql.Connection; import java.sql.Connection;
import java.sql.PreparedStatement; import java.sql.PreparedStatement;
import java.sql.ResultSet; import java.sql.ResultSet;
@ -27,6 +12,27 @@ import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.concurrent.CompletableFuture;
import com.arangodb.ArangoCursor;
import com.arangodb.ArangoDBException;
import com.arangodb.ArangoDatabase;
import com.arangodb.entity.StreamTransactionEntity;
import com.arangodb.model.AqlQueryOptions;
import com.arangodb.model.StreamTransactionOptions;
import org.micord.config.ArangoDBConnection;
import org.micord.config.DatabaseConnection;
import org.micord.config.S3HttpConnection;
import org.micord.models.AqlRequest;
import org.micord.models.RequestArgument;
import org.micord.models.Requests;
import org.micord.models.S3Request;
import org.micord.models.SqlRequest;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
/** /**
* @author Maksim Tereshin * @author Maksim Tereshin
@ -37,7 +43,77 @@ public class SqlAqlService {
private static final Logger logger = LoggerFactory.getLogger(SqlAqlService.class); private static final Logger logger = LoggerFactory.getLogger(SqlAqlService.class);
@Autowired @Autowired
private S3Service s3Service; private HttpClient httpClient;
public void processS3Requests(List<S3Request> s3Requests, List<String> ids) {
if (s3Requests != null) {
s3Requests.forEach(request -> {
List<CompletableFuture<Void>> futures = ids.stream()
.map(id -> CompletableFuture.runAsync(() -> processS3Request(request, id)))
.toList();
CompletableFuture.allOf(futures.toArray(new CompletableFuture[0]))
.thenRun(() -> logger.info("Successfully processed all S3 requests."))
.exceptionally(ex -> {
logger.error("Failed to process S3 requests", ex);
return null;
});
});
}
}
private void processS3Request(S3Request request, String id) {
try {
List<String> files = new ArrayList<>();
if (request.getRequestArguments() != null && !request.getRequestArguments().isEmpty()) {
for (RequestArgument argument : request.getRequestArguments()) {
try (Connection connection = DatabaseConnection.getConnection(
argument.getSqlConnectionParams())) {
String query = argument.getRequestURL();
List<String> result = fetchFileListFromDatabaseSQL(connection, query);
if (result != null && !result.isEmpty()) {
files.addAll(result);
}
}
catch (SQLException e) {
logger.error("Failed to execute query for RequestArgument", e);
}
}
}
files.forEach(file -> {
HttpRequest httpRequest;
try {
httpRequest = S3HttpConnection.buildHttpRequest(request, file);
}
catch (Exception e) {
throw new RuntimeException(e);
}
httpClient.sendAsync(httpRequest, HttpResponse.BodyHandlers.ofString())
.thenAccept(response -> {
if (response.statusCode() == HttpURLConnection.HTTP_NO_CONTENT
|| response.statusCode() == HttpURLConnection.HTTP_OK) {
logger.info("Successfully deleted object for ID {}", id);
}
else {
logger.error("Failed to delete object for ID {}. Response code: {}", id,
response.statusCode()
);
}
})
.exceptionally(ex -> {
logger.error("Failed to delete object for ID {}", id, ex);
return null;
});
});
}
catch (Exception e) {
logger.error("Failed to process S3 request for id: {}", id, e);
}
}
@Transactional @Transactional
public void processSqlAndAqlRequests(Requests config, List<String> ids) { public void processSqlAndAqlRequests(Requests config, List<String> ids) {
@ -52,14 +128,16 @@ public class SqlAqlService {
processAqlRequests(request, ids); processAqlRequests(request, ids);
} }
} }
s3Service.processS3Requests(config.getS3Requests(), ids); processS3Requests(config.getS3Requests(), ids);
} }
private void processSqlRequests(SqlRequest request, List<String> ids) { private void processSqlRequests(SqlRequest request, List<String> ids) {
try (Connection connection = DatabaseConnection.getConnection(request.getSqlConnectionParams())) { try (Connection connection = DatabaseConnection.getConnection(
request.getSqlConnectionParams())) {
String query = buildSqlQuery(request, String.join(",", ids)); String query = buildSqlQuery(request, String.join(",", ids));
executeSqlQuery(connection, query); executeSqlQuery(connection, query);
} catch (SQLException e) { }
catch (SQLException e) {
logger.error("SQL query execution failed", e); logger.error("SQL query execution failed", e);
} }
} }
@ -72,7 +150,8 @@ public class SqlAqlService {
clauseBuilder.append(argument.getId()).append(" IN (").append(ids).append(")"); clauseBuilder.append(argument.getId()).append(" IN (").append(ids).append(")");
if (argument.getSqlConnectionParams() != null) { if (argument.getSqlConnectionParams() != null) {
try (Connection connection = DatabaseConnection.getConnection(argument.getSqlConnectionParams())) { try (Connection connection = DatabaseConnection.getConnection(
argument.getSqlConnectionParams())) {
String query = argument.getRequestURL(); String query = argument.getRequestURL();
List<String> result = fetchFileListFromDatabaseSQL(connection, query); List<String> result = fetchFileListFromDatabaseSQL(connection, query);
@ -80,9 +159,14 @@ public class SqlAqlService {
String resultSet = String.join(", ", result.stream() String resultSet = String.join(", ", result.stream()
.map(s -> "'" + s + "'") .map(s -> "'" + s + "'")
.toArray(String[]::new)); .toArray(String[]::new));
clauseBuilder.append(" OR ").append(argument.getId()).append(" IN (").append(resultSet).append(")"); clauseBuilder.append(" OR ")
.append(argument.getId())
.append(" IN (")
.append(resultSet)
.append(")");
} }
} catch (SQLException e) { }
catch (SQLException e) {
logger.error("Failed to execute query for RequestArgument", e); logger.error("Failed to execute query for RequestArgument", e);
} }
} }
@ -102,7 +186,8 @@ public class SqlAqlService {
} }
} }
public List<String> fetchFileListFromDatabaseSQL(Connection connection, String query) throws SQLException { public List<String> fetchFileListFromDatabaseSQL(Connection connection, String query)
throws SQLException {
List<String> results = new ArrayList<>(); List<String> results = new ArrayList<>();
try (PreparedStatement stmt = connection.prepareStatement(query); try (PreparedStatement stmt = connection.prepareStatement(query);
ResultSet rs = stmt.executeQuery()) { ResultSet rs = stmt.executeQuery()) {
@ -131,13 +216,16 @@ public class SqlAqlService {
logger.info("Stream transaction started with ID: {}", transactionId); logger.info("Stream transaction started with ID: {}", transactionId);
List<String> entities = executeSelectAqlRequest(arangoDb, request.getRequestArguments(), ids, transactionId); List<String> entities = executeSelectAqlRequest(arangoDb, request.getRequestArguments(), ids,
transactionId
);
executeMainAqlRequest(arangoDb, request, entities, transactionId); executeMainAqlRequest(arangoDb, request, entities, transactionId);
arangoDb.commitStreamTransaction(transactionId); arangoDb.commitStreamTransaction(transactionId);
logger.info("Stream transaction with ID {} committed successfully", transactionId); logger.info("Stream transaction with ID {} committed successfully", transactionId);
} catch (ArangoDBException e) { }
catch (ArangoDBException e) {
if (tx != null) { if (tx != null) {
arangoDb.abortStreamTransaction(tx.getId()); arangoDb.abortStreamTransaction(tx.getId());
logger.error("Stream transaction with ID {} aborted due to an error", tx.getId(), e); logger.error("Stream transaction with ID {} aborted due to an error", tx.getId(), e);
@ -148,7 +236,9 @@ public class SqlAqlService {
logger.info("Successfully executed AQL request"); logger.info("Successfully executed AQL request");
} }
private List<String> executeSelectAqlRequest(ArangoDatabase arangoDb, List<RequestArgument> requestArguments, List<String> ids, String transactionId) { private List<String> executeSelectAqlRequest(ArangoDatabase arangoDb,
List<RequestArgument> requestArguments,
List<String> ids, String transactionId) {
List<String> entityIdList = new ArrayList<>(); List<String> entityIdList = new ArrayList<>();
RequestArgument argument = requestArguments.get(0); RequestArgument argument = requestArguments.get(0);
@ -184,14 +274,16 @@ public class SqlAqlService {
throw new IllegalArgumentException("Invalid requestArgumentId: " + entityType); throw new IllegalArgumentException("Invalid requestArgumentId: " + entityType);
} }
} }
} catch (Exception e) { }
catch (Exception e) {
logger.error("Failed to execute AQL query", e); logger.error("Failed to execute AQL query", e);
} }
return entityIdList; return entityIdList;
} }
private void executeMainAqlRequest(ArangoDatabase arangoDb, AqlRequest request, List<String> entityIdList, String transactionId) { private void executeMainAqlRequest(ArangoDatabase arangoDb, AqlRequest request,
List<String> entityIdList, String transactionId) {
if (entityIdList == null || entityIdList.isEmpty()) { if (entityIdList == null || entityIdList.isEmpty()) {
logger.warn("No entities found for main AQL request."); logger.warn("No entities found for main AQL request.");
return; return;