diff --git a/config-data-executor/pom.xml b/config-data-executor/pom.xml index 6695a17..a622486 100644 --- a/config-data-executor/pom.xml +++ b/config-data-executor/pom.xml @@ -2,113 +2,109 @@ - 4.0.0 - - ervu_secret - ervu_secret - 1.0.0-SNAPSHOT - - org.micord - config-data-executor - war + 4.0.0 - - 17 - 17 - UTF-8 - + org.micord + ConfigDataExecutor + 1.0-SNAPSHOT + + + 17 + 17 + UTF-8 + + + + + + com.amazonaws + aws-java-sdk-bom + 1.12.770 + pom + import + + + - - - + + org.jooq + jooq + 3.19.11 + - - - org.springframework.boot - spring-boot-starter-web - - - org.springframework - spring-tx - - - com.atomikos - transactions-jta - - - com.atomikos - transactions-jdbc - - - javax.transaction - javax.transaction-api - - - org.postgresql - postgresql - runtime - - - org.projectlombok - lombok - provided - - - jakarta.xml.bind - jakarta.xml.bind-api - - - com.sun.xml.bind - jaxb-impl - - - com.arangodb - arangodb-java-driver - - - com.amazonaws - aws-java-sdk-s3 - - - javax.servlet - javax.servlet-api - provided - - - - - maven_central - Maven Central - https://repo.maven.apache.org/maven2/ - - - - ${artifactId} - - - org.apache.maven.plugins - maven-war-plugin - 3.1.0 - - false - - - - org.springframework.boot - spring-boot-maven-plugin - 3.3.3 - - org.micord.Main - - - - - repackage - - - - - - - + + org.springframework.boot + spring-boot-starter-web + 3.3.2 + + + org.springframework.boot + spring-boot-starter-data-jpa + 3.3.2 + + + org.projectlombok + lombok + provided + 1.18.34 + + + javax.xml.bind + jaxb-api + 2.3.1 + + + org.glassfish.jaxb + jaxb-runtime + 2.3.1 + + + mysql + mysql-connector-java + 8.0.33 + runtime + + + org.postgresql + postgresql + 42.7.3 + runtime + + + com.arangodb + arangodb-java-driver + 7.7.1 + + + com.amazonaws + aws-java-sdk-s3 + + + com.atomikos + transactions-spring-boot3-starter + 6.0.0 + + + + javax.transaction + jta + 1.1 + + + + + + + + + + + + maven_central + Maven Central + https://repo.maven.apache.org/maven2/ + + + + \ No newline at end of file diff --git a/config-data-executor/src/main/java/org/micord/Main.java b/config-data-executor/src/main/java/org/micord/Main.java index e0307b5..b37af38 100644 --- a/config-data-executor/src/main/java/org/micord/Main.java +++ b/config-data-executor/src/main/java/org/micord/Main.java @@ -1,15 +1,15 @@ -//package org.micord; -// -//import org.springframework.boot.SpringApplication; -//import org.springframework.boot.autoconfigure.SpringBootApplication; -//import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration; -// -///** -// * @author Maksim Tereshin -// */ -//@SpringBootApplication(exclude = {DataSourceAutoConfiguration.class}) -//public class Main { -// public static void main(String[] args) { -// SpringApplication.run(Main.class, args); -// } -//} +package org.micord; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration; + +/** + * @author Maksim Tereshin + */ +@SpringBootApplication(exclude = {DataSourceAutoConfiguration.class}) +public class Main { + public static void main(String[] args) { + SpringApplication.run(Main.class, args); + } +} \ No newline at end of file diff --git a/config-data-executor/src/main/java/org/micord/config/AtomikosConfig.java b/config-data-executor/src/main/java/org/micord/config/AtomikosConfig.java index 02d06f3..1cc4563 100644 --- a/config-data-executor/src/main/java/org/micord/config/AtomikosConfig.java +++ b/config-data-executor/src/main/java/org/micord/config/AtomikosConfig.java @@ -24,7 +24,7 @@ public class AtomikosConfig { } @Bean - public TransactionManager atomikosTransactionManager() throws Throwable { + public TransactionManager atomikosTransactionManager() { UserTransactionManager userTransactionManager = new UserTransactionManager(); userTransactionManager.setForceShutdown(true); return userTransactionManager; @@ -36,3 +36,4 @@ public class AtomikosConfig { } } + diff --git a/config-data-executor/src/main/java/org/micord/enums/RequestArgumentType.java b/config-data-executor/src/main/java/org/micord/enums/RequestArgumentType.java new file mode 100644 index 0000000..66e9048 --- /dev/null +++ b/config-data-executor/src/main/java/org/micord/enums/RequestArgumentType.java @@ -0,0 +1,19 @@ +package org.micord.enums; + +import lombok.Getter; + +/** + * @author Maksim Tereshin + */ +@Getter +public enum RequestArgumentType { + SQL("SQL"), + AQL("AQL"), + S3("S3"); + + private final String type; + + RequestArgumentType(String type) { + this.type = type; + } +} diff --git a/config-data-executor/src/main/java/org/micord/models/AqlRequest.java b/config-data-executor/src/main/java/org/micord/models/AqlRequest.java index 8152605..ba26178 100644 --- a/config-data-executor/src/main/java/org/micord/models/AqlRequest.java +++ b/config-data-executor/src/main/java/org/micord/models/AqlRequest.java @@ -3,18 +3,40 @@ package org.micord.models; import lombok.Setter; import jakarta.xml.bind.annotation.XmlElement; +import jakarta.xml.bind.annotation.XmlElementWrapper; +import java.util.List; +import java.util.stream.Collectors; /** * @author Maksim Tereshin */ @Setter -public class AqlRequest extends Request { +public class AqlRequest extends BaseRequest { private AqlConnectionParams aqlConnectionParams; + private List aqlRequestCollections; @XmlElement(name = "AqlConnectionParams") public AqlConnectionParams getAqlConnectionParams() { return aqlConnectionParams; } + @XmlElementWrapper(name = "AqlRequestCollections") + @XmlElement(name = "AqlRequestCollection") + public List getAqlRequestCollections() { + return aqlRequestCollections; + } + + public List getReadCollections() { + return aqlRequestCollections.stream() + .filter(collection -> collection.getType() != null && collection.getType().contains("read")) + .collect(Collectors.toList()); + } + + public List getWriteCollections() { + return aqlRequestCollections.stream() + .filter(collection -> collection.getType() != null && collection.getType().contains("write")) + .collect(Collectors.toList()); + } + } diff --git a/config-data-executor/src/main/java/org/micord/models/AqlRequestCollection.java b/config-data-executor/src/main/java/org/micord/models/AqlRequestCollection.java new file mode 100644 index 0000000..0a56ccd --- /dev/null +++ b/config-data-executor/src/main/java/org/micord/models/AqlRequestCollection.java @@ -0,0 +1,27 @@ +package org.micord.models; + +import lombok.Setter; + +import javax.xml.bind.annotation.XmlAttribute; +import javax.xml.bind.annotation.XmlValue; + +/** + * @author Maksim Tereshin + */ +@Setter +public class AqlRequestCollection { + + private String type; + private String collectionName; + + @XmlAttribute(name = "type") + public String getType() { + return type; + } + + @XmlValue + public String getCollectionName() { + return collectionName; + } + +} diff --git a/config-data-executor/src/main/java/org/micord/models/BaseRequest.java b/config-data-executor/src/main/java/org/micord/models/BaseRequest.java new file mode 100644 index 0000000..68ae085 --- /dev/null +++ b/config-data-executor/src/main/java/org/micord/models/BaseRequest.java @@ -0,0 +1,29 @@ +package org.micord.models; + +import lombok.Setter; + +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlSeeAlso; +import java.util.List; + +/** + * @author Maksim Tereshin + */ +@Setter +@XmlSeeAlso({SqlRequest.class, S3Request.class, AqlRequest.class}) +public abstract class BaseRequest { + + private List requestArguments; + private String requestURL; + + @XmlElement(name = "RequestArgument") + public List getRequestArguments() { + return requestArguments; + } + + @XmlElement(name = "RequestURL") + public String getRequestURL() { + return requestURL; + } + +} diff --git a/config-data-executor/src/main/java/org/micord/models/RequestArgument.java b/config-data-executor/src/main/java/org/micord/models/RequestArgument.java index 87d5ebb..d05d7a6 100644 --- a/config-data-executor/src/main/java/org/micord/models/RequestArgument.java +++ b/config-data-executor/src/main/java/org/micord/models/RequestArgument.java @@ -1,7 +1,9 @@ package org.micord.models; import lombok.Setter; +import org.micord.enums.RequestArgumentType; +import jakarta.xml.bind.annotation.XmlAttribute; import jakarta.xml.bind.annotation.XmlElement; import jakarta.xml.bind.annotation.XmlElementWrapper; import jakarta.xml.bind.annotation.XmlRootElement; @@ -14,30 +16,29 @@ import java.util.List; @XmlRootElement(name = "RequestArgument") public class RequestArgument { - private String id; - private List aqlCollectionRead; - private String requestURL; - private SqlConnectionParams sqlConnectionParams; + private RequestArgumentType type; + private String requestArgumentName;; + private String requestArgumentURL; + private SqlConnectionParams requestArgumentConnectionParams; - @XmlElement(name = "SqlConnectionParams") - public SqlConnectionParams getSqlConnectionParams() { - return sqlConnectionParams; + @XmlAttribute(name = "type") + public RequestArgumentType getType() { + return type; } - @XmlElement(name = "Id") - public String getId() { - return id; + @XmlElement(name = "RequestArgumentName") + public String getRequestArgumentName() { + return requestArgumentName; } - @XmlElementWrapper(name = "AqlCollectionReads") - @XmlElement(name = "AqlCollectionRead") - public List getAqlCollectionRead() { - return aqlCollectionRead; + @XmlElement(name = "RequestArgumentURL") + public String getRequestArgumentURL() { + return requestArgumentURL; } - @XmlElement(name = "RequestURL") - public String getRequestURL() { - return requestURL; + @XmlElement(name = "RequestArgumentConnectionParams") + public SqlConnectionParams getRequestArgumentConnectionParams() { + return requestArgumentConnectionParams; } } diff --git a/config-data-executor/src/main/java/org/micord/models/S3Request.java b/config-data-executor/src/main/java/org/micord/models/S3Request.java index c282e6f..c690271 100644 --- a/config-data-executor/src/main/java/org/micord/models/S3Request.java +++ b/config-data-executor/src/main/java/org/micord/models/S3Request.java @@ -8,7 +8,7 @@ import jakarta.xml.bind.annotation.XmlElement; * @author Maksim Tereshin */ @Setter -public class S3Request extends Request { +public class S3Request extends BaseRequest { private S3ConnectionParams s3ConnectionParams; diff --git a/config-data-executor/src/main/java/org/micord/models/SqlConnectionParams.java b/config-data-executor/src/main/java/org/micord/models/SqlConnectionParams.java index df9ede9..7f38c05 100644 --- a/config-data-executor/src/main/java/org/micord/models/SqlConnectionParams.java +++ b/config-data-executor/src/main/java/org/micord/models/SqlConnectionParams.java @@ -56,4 +56,4 @@ public class SqlConnectionParams { return jdbcDatabase; } -} +} \ No newline at end of file diff --git a/config-data-executor/src/main/java/org/micord/models/SqlRequest.java b/config-data-executor/src/main/java/org/micord/models/SqlRequest.java index 2b3c3d0..68f8bc4 100644 --- a/config-data-executor/src/main/java/org/micord/models/SqlRequest.java +++ b/config-data-executor/src/main/java/org/micord/models/SqlRequest.java @@ -8,7 +8,7 @@ import jakarta.xml.bind.annotation.XmlElement; * @author Maksim Tereshin */ @Setter -public class SqlRequest extends Request { +public class SqlRequest extends BaseRequest { private SqlConnectionParams sqlConnectionParams; diff --git a/config-data-executor/src/main/java/org/micord/service/RequestService.java b/config-data-executor/src/main/java/org/micord/service/RequestService.java index be19cdc..a434ba3 100644 --- a/config-data-executor/src/main/java/org/micord/service/RequestService.java +++ b/config-data-executor/src/main/java/org/micord/service/RequestService.java @@ -21,11 +21,8 @@ import com.arangodb.model.StreamTransactionOptions; import org.micord.config.ArangoDBConnection; import org.micord.config.DatabaseConnection; import org.micord.config.S3HttpConnection; -import org.micord.models.AqlRequest; -import org.micord.models.RequestArgument; -import org.micord.models.Requests; -import org.micord.models.S3Request; -import org.micord.models.SqlRequest; +import org.micord.enums.RequestArgumentType; +import org.micord.models.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -67,8 +64,8 @@ public class RequestService { if (request.getRequestArguments() != null && !request.getRequestArguments().isEmpty()) { for (RequestArgument argument : request.getRequestArguments()) { try (Connection connection = DatabaseConnection.getConnection( - argument.getSqlConnectionParams())) { - String query = argument.getRequestURL(); + argument.getRequestArgumentConnectionParams())) { + String query = argument.getRequestArgumentURL(); List result = fetchFileListFromDatabaseSQL(connection, query); if (result != null && !result.isEmpty()) { files.addAll(result); @@ -130,20 +127,29 @@ public class RequestService { } private void processSqlRequests(SqlRequest request, List ids) { - String query = null; + Map query = buildSqlQuery(request, ids); try (Connection connection = DatabaseConnection.getConnection( request.getSqlConnectionParams())) { - query = buildSqlQuery(request, String.join(",", ids)); - int rowsAffected = executeSqlQuery(connection, query); - logger.info("Successfully deleted {} rows for IDs: {} in query: {}", rowsAffected, String.join(", ", ids), query); + String requestURL = (String) query.get("requestURL"); + executeSqlQuery(connection, requestURL); + + List queryIds = (List) query.get("ids"); + if (queryIds != null && !queryIds.isEmpty()) { + ids.addAll(queryIds); + } else { + logger.warn("No IDs found for the query"); + } + + logger.info("Successfully executed query {} for IDs: ({})", requestURL, String.join(", ", ids)); } catch (SQLException e) { logger.error("SQL execution failed for query: {}", query, e); } } - private String buildSqlQuery(SqlRequest request, String ids) { - String endpointArguments = " (" + Arrays.stream(ids.split(",")) + private Map buildSqlQuery(SqlRequest request, List ids) { + Map resultMap = new HashMap<>(); + String endpointArguments = "(" + ids.stream() .map(id -> "'" + id.trim() + "'") .collect(Collectors.joining(", ")) + ")"; @@ -152,19 +158,24 @@ public class RequestService { if (request.getRequestArguments() != null && !request.getRequestArguments().isEmpty()) { for (RequestArgument argument : request.getRequestArguments()) { - if (argument.getSqlConnectionParams() != null) { + if (argument.getRequestArgumentConnectionParams() != null) { try (Connection connection = DatabaseConnection.getConnection( - argument.getSqlConnectionParams())) { - String query = argument.getRequestURL(); + argument.getRequestArgumentConnectionParams())) { + String query = argument.getRequestArgumentURL(); List result = fetchFileListFromDatabaseSQL(connection, query); + resultMap.put("ids", result); + + if (result != null && !result.isEmpty()) { - String resultSet = " (" + result.stream() + String resultSet = "(" + result.stream() .map(s -> "'" + s.trim() + "'") .collect(Collectors.joining(", ")) + ")"; - requestURL = requestURL.replace("${" + argument.getId() + "}", resultSet); + requestURL = requestURL.replace("${" + argument.getRequestArgumentName() + "}", resultSet); + } + } catch (SQLException e) { logger.error("Failed to execute query for RequestArgument", e); @@ -173,14 +184,16 @@ public class RequestService { } } - return requestURL - .replace("${DB}", request.getSqlConnectionParams().getJdbcDatabase()) - .replace("${endpointArguments}", endpointArguments); + resultMap.put("requestURL", requestURL + .replace("${DB}", request.getSqlConnectionParams().getJdbcDatabase()) + .replace("${endpointArguments}", endpointArguments)); + + return resultMap; } - private int executeSqlQuery(Connection connection, String query) throws SQLException { + private boolean executeSqlQuery(Connection connection, String query) throws SQLException { try (PreparedStatement stmt = connection.prepareStatement(query)) { - return stmt.executeUpdate(); + return stmt.execute(); } } @@ -199,9 +212,14 @@ public class RequestService { private void processAqlRequests(AqlRequest request, List ids) { ArangoDatabase arangoDb = ArangoDBConnection.getConnection(request.getAqlConnectionParams()); + // TODO: implement for multiple request arguments RequestArgument requestArgument = request.getRequestArguments().get(0); - List aqlCollectionRead = requestArgument.getAqlCollectionRead(); - String aqlCollectionWrite = requestArgument.getId(); + List aqlCollectionRead = request.getReadCollections().stream() + .map(AqlRequestCollection::getCollectionName) + .toList(); + String aqlCollectionWrite = String.valueOf(request.getWriteCollections().stream() + .map(AqlRequestCollection::getCollectionName) + .findFirst()); StreamTransactionEntity tx = null; try { @@ -214,10 +232,8 @@ public class RequestService { logger.info("Stream transaction started with ID: {}", transactionId); - List entities = executeSelectAqlRequest(arangoDb, request.getRequestArguments(), ids, - transactionId - ); - executeMainAqlRequest(arangoDb, request, entities, transactionId); + Map entities = executeSelectAqlRequest(arangoDb, aqlCollectionWrite, requestArgument, ids, transactionId); + executeMainAqlRequest(arangoDb, aqlCollectionWrite, request.getRequestURL(), entities, transactionId); arangoDb.commitStreamTransaction(transactionId); logger.info("Stream transaction with ID {} committed successfully", transactionId); @@ -234,73 +250,71 @@ public class RequestService { logger.info("Successfully executed AQL request"); } - private List executeSelectAqlRequest(ArangoDatabase arangoDb, - List requestArguments, + private Map executeSelectAqlRequest(ArangoDatabase arangoDb, + String aqlCollectionWrite, + RequestArgument requestArgument, List ids, String transactionId) { - List entityIdList = new ArrayList<>(); + Map entities = new HashMap<>(); - RequestArgument argument = requestArguments.get(0); + String url = requestArgument.getRequestArgumentURL(); + RequestArgumentType type = requestArgument.getType(); - String query = argument.getRequestURL(); - String entityType = argument.getId(); + if (type == RequestArgumentType.AQL) { + Map bindVars = new HashMap<>(); + bindVars.put("ids", ids); - Map bindVars = new HashMap<>(); - bindVars.put("ids", ids); + AqlQueryOptions aqlQueryOptions = new AqlQueryOptions().streamTransactionId(transactionId); - AqlQueryOptions aqlQueryOptions = new AqlQueryOptions().streamTransactionId(transactionId); + try (ArangoCursor cursor = arangoDb.query(url, Map.class, bindVars, aqlQueryOptions)) { + while (cursor.hasNext()) { + Map result = cursor.next(); - try (ArangoCursor cursor = arangoDb.query(query, Map.class, bindVars, aqlQueryOptions)) { - while (cursor.hasNext()) { - Map result = cursor.next(); - switch (entityType) { - case "applicationId": - entityIdList.add((String) result.get("applicationId")); - break; - case "edgesId": - entityIdList.addAll((List) result.get("edgesId")); - break; - case "subjectId": - entityIdList.addAll((List) result.get("subjectId")); - break; - case "historyId": - entityIdList.addAll((List) result.get("historyId")); - break; - case "interdepreqId": - entityIdList.addAll((List) result.get("interdepreqId")); - break; - default: - throw new IllegalArgumentException("Invalid requestArgumentId: " + entityType); + for (Map.Entry entry : result.entrySet()) { + String key = entry.getKey(); + Object entityValue = entry.getValue(); + + entities.put(key, entityValue); + } + } + } + catch (Exception e) { + logger.error("Failed to execute AQL url", e); + } + } else if (type == RequestArgumentType.SQL) { + if (requestArgument.getRequestArgumentConnectionParams() != null) { + try (Connection connection = DatabaseConnection.getConnection( + requestArgument.getRequestArgumentConnectionParams())) { + String query = requestArgument.getRequestArgumentURL(); + List result = fetchFileListFromDatabaseSQL(connection, query); + + entities.put(aqlCollectionWrite, result); + } + catch (SQLException e) { + logger.error("Failed to execute query for RequestArgument", e); } } } - catch (Exception e) { - logger.error("Failed to execute AQL query", e); - } - return entityIdList; + return entities; } - private void executeMainAqlRequest(ArangoDatabase arangoDb, AqlRequest request, - List entityIdList, String transactionId) { - if (entityIdList == null || entityIdList.isEmpty()) { + private void executeMainAqlRequest(ArangoDatabase arangoDb, String aqlCollectionWrite, String requestURL, + Map entities, String transactionId) { + if (entities == null || entities.isEmpty()) { logger.warn("No entities found for main AQL request."); return; } - String entity = request.getRequestArguments().get(0).getId(); - Map bindVars = new HashMap<>(); - bindVars.put("ids", entityIdList); - - - String finalQuery = request.getRequestURL() - .replace("${entity}", entity); + // TODO: verify correctness of received entities and compare keys + Object writeEntity = entities.get(aqlCollectionWrite); + bindVars.put("ids", entities); AqlQueryOptions aqlQueryOptions = new AqlQueryOptions().streamTransactionId(transactionId); - arangoDb.query(finalQuery, null, bindVars, aqlQueryOptions); + arangoDb.query(requestURL, null, bindVars, aqlQueryOptions); - logger.info("Successfully removed {}: {}", entity, entityIdList); + logger.info("Successfully removed {}: {}", aqlCollectionWrite, writeEntity); } } diff --git a/config/standalone.xml b/config/standalone.xml index cf01d06..709c050 100644 --- a/config/standalone.xml +++ b/config/standalone.xml @@ -38,6 +38,7 @@ +