micord/ervu-secret
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

executor_fix

Browse source
This commit is contained in:
ilyin 2024-09-27 14:07:04 +03:00
parent 245225bc7d
commit cc8b6f7823
13 changed files with 328 additions and 218 deletions
Download patch file Download diff file Expand all files Collapse all files

208
config-data-executor/pom.xml
View file

@ -2,113 +2,109 @@
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>ervu_secret</groupId>
<artifactId>ervu_secret</artifactId>
<version>1.0.0-SNAPSHOT</version>
</parent>
<groupId>org.micord</groupId>
<artifactId>config-data-executor</artifactId>
<packaging>war</packaging>
<modelVersion>4.0.0</modelVersion>
<properties>
<maven.compiler.source>17</maven.compiler.source>
<maven.compiler.target>17</maven.compiler.target>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
</properties>
<groupId>org.micord</groupId>
<artifactId>ConfigDataExecutor</artifactId>
<version>1.0-SNAPSHOT</version>
<properties>
<maven.compiler.source>17</maven.compiler.source>
<maven.compiler.target>17</maven.compiler.target>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
</properties>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-bom</artifactId>
<version>1.12.770</version>
<type>pom</type>
<scope>import</scope>
</dependency>
</dependencies>
</dependencyManagement>
<dependencyManagement>
<dependencies>
</dependencies>
</dependencyManagement>
<dependency>
<groupId>org.jooq</groupId>
<artifactId>jooq</artifactId>
<version>3.19.11</version>
</dependency>
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-tx</artifactId>
</dependency>
<dependency>
<groupId>com.atomikos</groupId>
<artifactId>transactions-jta</artifactId>
</dependency>
<dependency>
<groupId>com.atomikos</groupId>
<artifactId>transactions-jdbc</artifactId>
</dependency>
<dependency>
<groupId>javax.transaction</groupId>
<artifactId>javax.transaction-api</artifactId>
</dependency>
<dependency>
<groupId>org.postgresql</groupId>
<artifactId>postgresql</artifactId>
<scope>runtime</scope>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>jakarta.xml.bind</groupId>
<artifactId>jakarta.xml.bind-api</artifactId>
</dependency>
<dependency>
<groupId>com.sun.xml.bind</groupId>
<artifactId>jaxb-impl</artifactId>
</dependency>
<dependency>
<groupId>com.arangodb</groupId>
<artifactId>arangodb-java-driver</artifactId>
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-s3</artifactId>
</dependency>
<dependency>
<groupId>javax.servlet</groupId>
<artifactId>javax.servlet-api</artifactId>
<scope>provided</scope>
</dependency>
</dependencies>
<repositories>
<repository>
<id>maven_central</id>
<name>Maven Central</name>
<url>https://repo.maven.apache.org/maven2/</url>
</repository>
</repositories>
<build>
<finalName>${artifactId}</finalName>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-war-plugin</artifactId>
<version>3.1.0</version>
<configuration>
<failOnMissingWebXml>false</failOnMissingWebXml>
</configuration>
</plugin>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<version>3.3.3</version>
<configuration>
<mainClass>org.micord.Main</mainClass>
</configuration>
<executions>
<execution>
<goals>
<goal>repackage</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
<version>3.3.2</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-jpa</artifactId>
<version>3.3.2</version>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<scope>provided</scope>
<version>1.18.34</version>
</dependency>
<dependency>
<groupId>javax.xml.bind</groupId>
<artifactId>jaxb-api</artifactId>
<version>2.3.1</version>
</dependency>
<dependency>
<groupId>org.glassfish.jaxb</groupId>
<artifactId>jaxb-runtime</artifactId>
<version>2.3.1</version>
</dependency>
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<version>8.0.33</version>
<scope>runtime</scope>
</dependency>
<dependency>
<groupId>org.postgresql</groupId>
<artifactId>postgresql</artifactId>
<version>42.7.3</version>
<scope>runtime</scope>
</dependency>
<dependency>
<groupId>com.arangodb</groupId>
<artifactId>arangodb-java-driver</artifactId>
<version>7.7.1</version>
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-s3</artifactId>
</dependency>
<dependency>
<groupId>com.atomikos</groupId>
<artifactId>transactions-spring-boot3-starter</artifactId>
<version>6.0.0</version>
</dependency>
<dependency>
<groupId>javax.transaction</groupId>
<artifactId>jta</artifactId>
<version>1.1</version>
</dependency>
</dependencies>
<repositories>
<repository>
<id>maven_central</id>
<name>Maven Central</name>
<url>https://repo.maven.apache.org/maven2/</url>
</repository>
</repositories>
</project>

30
config-data-executor/src/main/java/org/micord/Main.java
View file

@ -1,15 +1,15 @@
//package org.micord;
//
//import org.springframework.boot.SpringApplication;
//import org.springframework.boot.autoconfigure.SpringBootApplication;
//import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration;
//
///**
// * @author Maksim Tereshin
// */
//@SpringBootApplication(exclude = {DataSourceAutoConfiguration.class})
//public class Main {
// public static void main(String[] args) {
// SpringApplication.run(Main.class, args);
// }
//}
package org.micord;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration;
/**
* @author Maksim Tereshin
*/
@SpringBootApplication(exclude = {DataSourceAutoConfiguration.class})
public class Main {
public static void main(String[] args) {
SpringApplication.run(Main.class, args);
}
}

3
config-data-executor/src/main/java/org/micord/config/AtomikosConfig.java
View file

@ -24,7 +24,7 @@ public class AtomikosConfig {
}
@Bean
public TransactionManager atomikosTransactionManager() throws Throwable {
public TransactionManager atomikosTransactionManager() {
UserTransactionManager userTransactionManager = new UserTransactionManager();
userTransactionManager.setForceShutdown(true);
return userTransactionManager;
@ -36,3 +36,4 @@ public class AtomikosConfig {
}
}

19
config-data-executor/src/main/java/org/micord/enums/RequestArgumentType.java Normal file
View file

@ -0,0 +1,19 @@
package org.micord.enums;
import lombok.Getter;
/**
* @author Maksim Tereshin
*/
@Getter
public enum RequestArgumentType {
SQL("SQL"),
AQL("AQL"),
S3("S3");
private final String type;
RequestArgumentType(String type) {
this.type = type;
}
}

24
config-data-executor/src/main/java/org/micord/models/AqlRequest.java
View file

@ -3,18 +3,40 @@ package org.micord.models;
import lombok.Setter;
import jakarta.xml.bind.annotation.XmlElement;
import jakarta.xml.bind.annotation.XmlElementWrapper;
import java.util.List;
import java.util.stream.Collectors;
/**
* @author Maksim Tereshin
*/
@Setter
public class AqlRequest extends Request {
public class AqlRequest extends BaseRequest {
private AqlConnectionParams aqlConnectionParams;
private List<AqlRequestCollection> aqlRequestCollections;
@XmlElement(name = "AqlConnectionParams")
public AqlConnectionParams getAqlConnectionParams() {
return aqlConnectionParams;
}
@XmlElementWrapper(name = "AqlRequestCollections")
@XmlElement(name = "AqlRequestCollection")
public List<AqlRequestCollection> getAqlRequestCollections() {
return aqlRequestCollections;
}
public List<AqlRequestCollection> getReadCollections() {
return aqlRequestCollections.stream()
.filter(collection -> collection.getType() != null && collection.getType().contains("read"))
.collect(Collectors.toList());
}
public List<AqlRequestCollection> getWriteCollections() {
return aqlRequestCollections.stream()
.filter(collection -> collection.getType() != null && collection.getType().contains("write"))
.collect(Collectors.toList());
}
}

27
config-data-executor/src/main/java/org/micord/models/AqlRequestCollection.java Normal file
View file

@ -0,0 +1,27 @@
package org.micord.models;
import lombok.Setter;
import javax.xml.bind.annotation.XmlAttribute;
import javax.xml.bind.annotation.XmlValue;
/**
* @author Maksim Tereshin
*/
@Setter
public class AqlRequestCollection {
private String type;
private String collectionName;
@XmlAttribute(name = "type")
public String getType() {
return type;
}
@XmlValue
public String getCollectionName() {
return collectionName;
}
}

29
config-data-executor/src/main/java/org/micord/models/BaseRequest.java Normal file
View file

@ -0,0 +1,29 @@
package org.micord.models;
import lombok.Setter;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlSeeAlso;
import java.util.List;
/**
* @author Maksim Tereshin
*/
@Setter
@XmlSeeAlso({SqlRequest.class, S3Request.class, AqlRequest.class})
public abstract class BaseRequest {
private List<RequestArgument> requestArguments;
private String requestURL;
@XmlElement(name = "RequestArgument")
public List<RequestArgument> getRequestArguments() {
return requestArguments;
}
@XmlElement(name = "RequestURL")
public String getRequestURL() {
return requestURL;
}
}

35
config-data-executor/src/main/java/org/micord/models/RequestArgument.java
View file

@ -1,7 +1,9 @@
package org.micord.models;
import lombok.Setter;
import org.micord.enums.RequestArgumentType;
import jakarta.xml.bind.annotation.XmlAttribute;
import jakarta.xml.bind.annotation.XmlElement;
import jakarta.xml.bind.annotation.XmlElementWrapper;
import jakarta.xml.bind.annotation.XmlRootElement;
@ -14,30 +16,29 @@ import java.util.List;
@XmlRootElement(name = "RequestArgument")
public class RequestArgument {
private String id;
private List<String> aqlCollectionRead;
private String requestURL;
private SqlConnectionParams sqlConnectionParams;
private RequestArgumentType type;
private String requestArgumentName;;
private String requestArgumentURL;
private SqlConnectionParams requestArgumentConnectionParams;
@XmlElement(name = "SqlConnectionParams")
public SqlConnectionParams getSqlConnectionParams() {
return sqlConnectionParams;
@XmlAttribute(name = "type")
public RequestArgumentType getType() {
return type;
}
@XmlElement(name = "Id")
public String getId() {
return id;
@XmlElement(name = "RequestArgumentName")
public String getRequestArgumentName() {
return requestArgumentName;
}
@XmlElementWrapper(name = "AqlCollectionReads")
@XmlElement(name = "AqlCollectionRead")
public List<String> getAqlCollectionRead() {
return aqlCollectionRead;
@XmlElement(name = "RequestArgumentURL")
public String getRequestArgumentURL() {
return requestArgumentURL;
}
@XmlElement(name = "RequestURL")
public String getRequestURL() {
return requestURL;
@XmlElement(name = "RequestArgumentConnectionParams")
public SqlConnectionParams getRequestArgumentConnectionParams() {
return requestArgumentConnectionParams;
}
}

2
config-data-executor/src/main/java/org/micord/models/S3Request.java
View file

@ -8,7 +8,7 @@ import jakarta.xml.bind.annotation.XmlElement;
* @author Maksim Tereshin
*/
@Setter
public class S3Request extends Request {
public class S3Request extends BaseRequest {
private S3ConnectionParams s3ConnectionParams;

2
config-data-executor/src/main/java/org/micord/models/SqlConnectionParams.java
View file

@ -56,4 +56,4 @@ public class SqlConnectionParams {
return jdbcDatabase;
}
}
}

2
config-data-executor/src/main/java/org/micord/models/SqlRequest.java
View file

@ -8,7 +8,7 @@ import jakarta.xml.bind.annotation.XmlElement;
* @author Maksim Tereshin
*/
@Setter
public class SqlRequest extends Request {
public class SqlRequest extends BaseRequest {
private SqlConnectionParams sqlConnectionParams;

164
config-data-executor/src/main/java/org/micord/service/RequestService.java
View file

@ -21,11 +21,8 @@ import com.arangodb.model.StreamTransactionOptions;
import org.micord.config.ArangoDBConnection;
import org.micord.config.DatabaseConnection;
import org.micord.config.S3HttpConnection;
import org.micord.models.AqlRequest;
import org.micord.models.RequestArgument;
import org.micord.models.Requests;
import org.micord.models.S3Request;
import org.micord.models.SqlRequest;
import org.micord.enums.RequestArgumentType;
import org.micord.models.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
@ -67,8 +64,8 @@ public class RequestService {
if (request.getRequestArguments() != null && !request.getRequestArguments().isEmpty()) {
for (RequestArgument argument : request.getRequestArguments()) {
try (Connection connection = DatabaseConnection.getConnection(
argument.getSqlConnectionParams())) {
String query = argument.getRequestURL();
argument.getRequestArgumentConnectionParams())) {
String query = argument.getRequestArgumentURL();
List<String> result = fetchFileListFromDatabaseSQL(connection, query);
if (result != null && !result.isEmpty()) {
files.addAll(result);
@ -130,20 +127,29 @@ public class RequestService {
}
private void processSqlRequests(SqlRequest request, List<String> ids) {
String query = null;
Map<String, Object> query = buildSqlQuery(request, ids);
try (Connection connection = DatabaseConnection.getConnection(
request.getSqlConnectionParams())) {
query = buildSqlQuery(request, String.join(",", ids));
int rowsAffected = executeSqlQuery(connection, query);
logger.info("Successfully deleted {} rows for IDs: {} in query: {}", rowsAffected, String.join(", ", ids), query);
String requestURL = (String) query.get("requestURL");
executeSqlQuery(connection, requestURL);
List<String> queryIds = (List<String>) query.get("ids");
if (queryIds != null && !queryIds.isEmpty()) {
ids.addAll(queryIds);
} else {
logger.warn("No IDs found for the query");
}
logger.info("Successfully executed query {} for IDs: ({})", requestURL, String.join(", ", ids));
}
catch (SQLException e) {
logger.error("SQL execution failed for query: {}", query, e);
}
}
private String buildSqlQuery(SqlRequest request, String ids) {
String endpointArguments = " (" + Arrays.stream(ids.split(","))
private Map<String, Object> buildSqlQuery(SqlRequest request, List<String> ids) {
Map<String, Object> resultMap = new HashMap<>();
String endpointArguments = "(" + ids.stream()
.map(id -> "'" + id.trim() + "'")
.collect(Collectors.joining(", ")) + ")";
@ -152,19 +158,24 @@ public class RequestService {
if (request.getRequestArguments() != null && !request.getRequestArguments().isEmpty()) {
for (RequestArgument argument : request.getRequestArguments()) {
if (argument.getSqlConnectionParams() != null) {
if (argument.getRequestArgumentConnectionParams() != null) {
try (Connection connection = DatabaseConnection.getConnection(
argument.getSqlConnectionParams())) {
String query = argument.getRequestURL();
argument.getRequestArgumentConnectionParams())) {
String query = argument.getRequestArgumentURL();
List<String> result = fetchFileListFromDatabaseSQL(connection, query);
resultMap.put("ids", result);
if (result != null && !result.isEmpty()) {
String resultSet = " (" + result.stream()
String resultSet = "(" + result.stream()
.map(s -> "'" + s.trim() + "'")
.collect(Collectors.joining(", ")) + ")";
requestURL = requestURL.replace("${" + argument.getId() + "}", resultSet);
requestURL = requestURL.replace("${" + argument.getRequestArgumentName() + "}", resultSet);
}
}
catch (SQLException e) {
logger.error("Failed to execute query for RequestArgument", e);
@ -173,14 +184,16 @@ public class RequestService {
}
}
return requestURL
.replace("${DB}", request.getSqlConnectionParams().getJdbcDatabase())
.replace("${endpointArguments}", endpointArguments);
resultMap.put("requestURL", requestURL
.replace("${DB}", request.getSqlConnectionParams().getJdbcDatabase())
.replace("${endpointArguments}", endpointArguments));
return resultMap;
}
private int executeSqlQuery(Connection connection, String query) throws SQLException {
private boolean executeSqlQuery(Connection connection, String query) throws SQLException {
try (PreparedStatement stmt = connection.prepareStatement(query)) {
return stmt.executeUpdate();
return stmt.execute();
}
}
@ -199,9 +212,14 @@ public class RequestService {
private void processAqlRequests(AqlRequest request, List<String> ids) {
ArangoDatabase arangoDb = ArangoDBConnection.getConnection(request.getAqlConnectionParams());
// TODO: implement for multiple request arguments
RequestArgument requestArgument = request.getRequestArguments().get(0);
List<String> aqlCollectionRead = requestArgument.getAqlCollectionRead();
String aqlCollectionWrite = requestArgument.getId();
List<String> aqlCollectionRead = request.getReadCollections().stream()
.map(AqlRequestCollection::getCollectionName)
.toList();
String aqlCollectionWrite = String.valueOf(request.getWriteCollections().stream()
.map(AqlRequestCollection::getCollectionName)
.findFirst());
StreamTransactionEntity tx = null;
try {
@ -214,10 +232,8 @@ public class RequestService {
logger.info("Stream transaction started with ID: {}", transactionId);
List<String> entities = executeSelectAqlRequest(arangoDb, request.getRequestArguments(), ids,
transactionId
);
executeMainAqlRequest(arangoDb, request, entities, transactionId);
Map<String, Object> entities = executeSelectAqlRequest(arangoDb, aqlCollectionWrite, requestArgument, ids, transactionId);
executeMainAqlRequest(arangoDb, aqlCollectionWrite, request.getRequestURL(), entities, transactionId);
arangoDb.commitStreamTransaction(transactionId);
logger.info("Stream transaction with ID {} committed successfully", transactionId);
@ -234,73 +250,71 @@ public class RequestService {
logger.info("Successfully executed AQL request");
}
private List<String> executeSelectAqlRequest(ArangoDatabase arangoDb,
List<RequestArgument> requestArguments,
private Map<String, Object> executeSelectAqlRequest(ArangoDatabase arangoDb,
String aqlCollectionWrite,
RequestArgument requestArgument,
List<String> ids, String transactionId) {
List<String> entityIdList = new ArrayList<>();
Map<String, Object> entities = new HashMap<>();
RequestArgument argument = requestArguments.get(0);
String url = requestArgument.getRequestArgumentURL();
RequestArgumentType type = requestArgument.getType();
String query = argument.getRequestURL();
String entityType = argument.getId();
if (type == RequestArgumentType.AQL) {
Map<String, Object> bindVars = new HashMap<>();
bindVars.put("ids", ids);
Map<String, Object> bindVars = new HashMap<>();
bindVars.put("ids", ids);
AqlQueryOptions aqlQueryOptions = new AqlQueryOptions().streamTransactionId(transactionId);
AqlQueryOptions aqlQueryOptions = new AqlQueryOptions().streamTransactionId(transactionId);
try (ArangoCursor<Map> cursor = arangoDb.query(url, Map.class, bindVars, aqlQueryOptions)) {
while (cursor.hasNext()) {
Map<String, Object> result = cursor.next();
try (ArangoCursor<Map> cursor = arangoDb.query(query, Map.class, bindVars, aqlQueryOptions)) {
while (cursor.hasNext()) {
Map result = cursor.next();
switch (entityType) {
case "applicationId":
entityIdList.add((String) result.get("applicationId"));
break;
case "edgesId":
entityIdList.addAll((List<String>) result.get("edgesId"));
break;
case "subjectId":
entityIdList.addAll((List<String>) result.get("subjectId"));
break;
case "historyId":
entityIdList.addAll((List<String>) result.get("historyId"));
break;
case "interdepreqId":
entityIdList.addAll((List<String>) result.get("interdepreqId"));
break;
default:
throw new IllegalArgumentException("Invalid requestArgumentId: " + entityType);
for (Map.Entry<String, Object> entry : result.entrySet()) {
String key = entry.getKey();
Object entityValue = entry.getValue();
entities.put(key, entityValue);
}
}
}
catch (Exception e) {
logger.error("Failed to execute AQL url", e);
}
} else if (type == RequestArgumentType.SQL) {
if (requestArgument.getRequestArgumentConnectionParams() != null) {
try (Connection connection = DatabaseConnection.getConnection(
requestArgument.getRequestArgumentConnectionParams())) {
String query = requestArgument.getRequestArgumentURL();
List<String> result = fetchFileListFromDatabaseSQL(connection, query);
entities.put(aqlCollectionWrite, result);
}
catch (SQLException e) {
logger.error("Failed to execute query for RequestArgument", e);
}
}
}
catch (Exception e) {
logger.error("Failed to execute AQL query", e);
}
return entityIdList;
return entities;
}
private void executeMainAqlRequest(ArangoDatabase arangoDb, AqlRequest request,
List<String> entityIdList, String transactionId) {
if (entityIdList == null || entityIdList.isEmpty()) {
private void executeMainAqlRequest(ArangoDatabase arangoDb, String aqlCollectionWrite, String requestURL,
Map<String, Object> entities, String transactionId) {
if (entities == null || entities.isEmpty()) {
logger.warn("No entities found for main AQL request.");
return;
}
String entity = request.getRequestArguments().get(0).getId();
Map<String, Object> bindVars = new HashMap<>();
bindVars.put("ids", entityIdList);
String finalQuery = request.getRequestURL()
.replace("${entity}", entity);
// TODO: verify correctness of received entities and compare keys
Object writeEntity = entities.get(aqlCollectionWrite);
bindVars.put("ids", entities);
AqlQueryOptions aqlQueryOptions = new AqlQueryOptions().streamTransactionId(transactionId);
arangoDb.query(finalQuery, null, bindVars, aqlQueryOptions);
arangoDb.query(requestURL, null, bindVars, aqlQueryOptions);
logger.info("Successfully removed {}: {}", entity, entityIdList);
logger.info("Successfully removed {}: {}", aqlCollectionWrite, writeEntity);
}
}

1
config/standalone.xml
View file

@ -38,6 +38,7 @@
<extension module="org.wildfly.extension.request-controller"/>
<extension module="org.wildfly.extension.security.manager"/>
<extension module="org.wildfly.extension.undertow"/>
<property name="configDirectory" value="C:\work\ervu-secret\config-data-executor\config-examples"/>
</extensions>
<system-properties>
<property name="webbpm.mode" value="production"/>