Add download report for Arango

This commit is contained in:
Maksim Tereshin 2025-03-03 10:54:02 +01:00
parent 80442b082b
commit 5fe775267e
No known key found for this signature in database
21 changed files with 661 additions and 337 deletions

View file

@ -1,18 +1,14 @@
package org.micord.controller; package org.micord.controller;
import org.micord.enums.ConfigType; import org.micord.enums.ConfigType;
import org.micord.exceptions.ValidationException;
import org.micord.models.requests.DownloadCSVRequest;
import org.micord.models.requests.RequestParameters; import org.micord.models.requests.RequestParameters;
import org.micord.service.ApiService; import org.micord.service.ApiService;
import org.micord.service.ValidationService;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.io.InputStreamResource; import org.springframework.core.io.InputStreamResource;
import org.springframework.core.io.Resource; import org.springframework.core.io.Resource;
import org.springframework.http.HttpHeaders; import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType; import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity; import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.*; import org.springframework.web.bind.annotation.*;
@ -23,7 +19,6 @@ import java.io.FileNotFoundException;
import java.io.IOException; import java.io.IOException;
import java.sql.SQLException; import java.sql.SQLException;
import java.util.List; import java.util.List;
import java.util.Map;
/** /**
* REST Controller for API operations. * REST Controller for API operations.
@ -37,16 +32,9 @@ public class ApiController {
@Autowired @Autowired
private ApiService apiService; private ApiService apiService;
@Autowired
private ValidationService validationService;
@PostMapping("/removeMilitaryDraftNotices") @PostMapping("/removeMilitaryDraftNotices")
public ResponseEntity<?> removeMilitaryDraftNotices(@RequestBody RequestParameters request) throws SQLException, FileNotFoundException { public ResponseEntity<?> removeMilitaryDraftNotices(@RequestBody RequestParameters request) throws SQLException, FileNotFoundException {
validateRequestDates(request);
List<String> ids = request.getIds(); List<String> ids = request.getIds();
validationService.validateAll(ids);
logger.debug("Starting removeMilitaryDraftNotices process for ids: {}", ids); logger.debug("Starting removeMilitaryDraftNotices process for ids: {}", ids);
apiService.process(ConfigType.REMOVE_MILITARY_DRAFT_NOTICES, request); apiService.process(ConfigType.REMOVE_MILITARY_DRAFT_NOTICES, request);
@ -56,7 +44,7 @@ public class ApiController {
} }
@PostMapping("/deleteFiles") @PostMapping("/deleteFiles")
public ResponseEntity<?> deleteFiles(@RequestBody List<String> ids) throws FileNotFoundException { public ResponseEntity<?> deleteFiles(@RequestBody List<String> ids) throws FileNotFoundException, SQLException {
apiService.process(ConfigType.DELETE_FILES, ids); apiService.process(ConfigType.DELETE_FILES, ids);
@ -65,7 +53,6 @@ public class ApiController {
@PostMapping("/block") @PostMapping("/block")
public ResponseEntity<?> block(@RequestBody List<String> ids) throws SQLException, FileNotFoundException { public ResponseEntity<?> block(@RequestBody List<String> ids) throws SQLException, FileNotFoundException {
validationService.validateAll(ids);
logger.debug("Starting block process for ids: {}", ids); logger.debug("Starting block process for ids: {}", ids);
apiService.process(ConfigType.BLOCK, ids); apiService.process(ConfigType.BLOCK, ids);
@ -77,7 +64,6 @@ public class ApiController {
@PostMapping("/unblock") @PostMapping("/unblock")
public ResponseEntity<?> unblock(@RequestBody List<String> ids) throws SQLException, FileNotFoundException { public ResponseEntity<?> unblock(@RequestBody List<String> ids) throws SQLException, FileNotFoundException {
validationService.validateAll(ids);
logger.debug("Starting unblock process for ids: {}", ids); logger.debug("Starting unblock process for ids: {}", ids);
apiService.process(ConfigType.UNBLOCK, ids); apiService.process(ConfigType.UNBLOCK, ids);
@ -89,7 +75,6 @@ public class ApiController {
@PostMapping("/removeFromSystem") @PostMapping("/removeFromSystem")
public ResponseEntity<?> removeFromSystem(@RequestBody List<String> ids) throws SQLException, FileNotFoundException { public ResponseEntity<?> removeFromSystem(@RequestBody List<String> ids) throws SQLException, FileNotFoundException {
validationService.validateAll(ids);
logger.debug("Starting removeFromSystem process for ids: {}", ids); logger.debug("Starting removeFromSystem process for ids: {}", ids);
apiService.process(ConfigType.REMOVE_FROM_SYSTEM, ids); apiService.process(ConfigType.REMOVE_FROM_SYSTEM, ids);
@ -101,7 +86,6 @@ public class ApiController {
@PostMapping("/removeFromCallList") @PostMapping("/removeFromCallList")
public ResponseEntity<?> removeFromCallList(@RequestBody List<String> ids) throws SQLException, FileNotFoundException { public ResponseEntity<?> removeFromCallList(@RequestBody List<String> ids) throws SQLException, FileNotFoundException {
validationService.validateAll(ids);
logger.debug("Starting removeFromCallList process for ids: {}", ids); logger.debug("Starting removeFromCallList process for ids: {}", ids);
apiService.process(ConfigType.REMOVE_FROM_CALL_LIST, ids); apiService.process(ConfigType.REMOVE_FROM_CALL_LIST, ids);
@ -112,11 +96,9 @@ public class ApiController {
@PostMapping("/downloadCSV") @PostMapping("/downloadCSV")
public ResponseEntity<Resource> downloadCSV(@RequestBody DownloadCSVRequest request) throws IOException { public ResponseEntity<Resource> downloadCSV(@RequestBody RequestParameters request) throws IOException {
logger.debug("Starting downloadCSV process for request: {}", request.getType()); logger.debug("Starting downloadCSV process for request: {}", request.getType());
validateRequestDates(request);
File csvFile = apiService.download(ConfigType.DOWNLOAD_CSV, request); File csvFile = apiService.download(ConfigType.DOWNLOAD_CSV, request);
InputStreamResource resource = new InputStreamResource(new FileInputStream(csvFile)); InputStreamResource resource = new InputStreamResource(new FileInputStream(csvFile));
@ -124,27 +106,12 @@ public class ApiController {
return ResponseEntity.ok() return ResponseEntity.ok()
.header(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=" + csvFile.getName()) .header(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=" + csvFile.getName())
.contentType(MediaType.parseMediaType("text/csv")) .contentType(MediaType.parseMediaType("text/csv; charset=UTF-8"))
.header(HttpHeaders.CONTENT_ENCODING, "UTF-8")
.contentLength(csvFile.length()) .contentLength(csvFile.length())
.body(resource); .body(resource);
} }
private void validateRequestDates(DownloadCSVRequest request) {
if (request.getStartDate() != null && request.getEndDate() != null) {
if (request.getStartDate().isAfter(request.getEndDate())) {
throw new IllegalArgumentException("Start date must be before end date");
}
}
}
private void validateRequestDates(RequestParameters request) {
if (request.getStartDate() != null && request.getEndDate() != null) {
if (request.getStartDate().isAfter(request.getEndDate())) {
throw new IllegalArgumentException("Start date must be before end date");
}
}
}
@GetMapping("/listDownloadTypes") @GetMapping("/listDownloadTypes")
public ResponseEntity<List<String>> listDownloadTypes() throws FileNotFoundException { public ResponseEntity<List<String>> listDownloadTypes() throws FileNotFoundException {

View file

@ -83,6 +83,14 @@ public class GlobalExceptionHandler {
)); ));
} }
@ExceptionHandler(IllegalRequestParametersException.class)
public ResponseEntity<?> handleIllegalRequestParametersException(IllegalRequestParametersException e) {
return ResponseEntity.badRequest().body(Map.of(
"message", "Произошла ошибка ввода данных. Проверьте правильность заполнения полей",
"details", e.getMessage()
));
}
@ExceptionHandler(NoSuchElementException.class) @ExceptionHandler(NoSuchElementException.class)
public ResponseEntity<?> handleNoSuchElementException(NoSuchElementException e) { public ResponseEntity<?> handleNoSuchElementException(NoSuchElementException e) {
logger.error("Resource not found: {}", e.getMessage()); logger.error("Resource not found: {}", e.getMessage());

View file

@ -0,0 +1,13 @@
package org.micord.exceptions;
import lombok.Getter;
@Getter
public class IllegalRequestParametersException extends IllegalArgumentException {
public IllegalRequestParametersException(String message) {
super(message);
}
}

View file

@ -1,41 +0,0 @@
package org.micord.models;
import lombok.Setter;
import jakarta.xml.bind.annotation.XmlElement;
import jakarta.xml.bind.annotation.XmlRootElement;
import java.util.List;
/**
* @author Maksim Tereshin
*/
@Setter
@XmlRootElement(name = "Requests")
public class Requests {
private List<SqlRequest> sqlRequests;
private List<AqlRequest> aqlRequests;
private List<S3Request> s3Requests;
private List<DownloadRequest> downloadRequests;
@XmlElement(name = "DownloadRequest")
public List<DownloadRequest> getDownloadRequests() {
return downloadRequests;
}
@XmlElement(name = "SqlRequest")
public List<SqlRequest> getSqlRequests() {
return sqlRequests;
}
@XmlElement(name = "AqlRequest")
public List<AqlRequest> getAqlRequests() {
return aqlRequests;
}
@XmlElement(name = "S3Request")
public List<S3Request> getS3Requests() {
return s3Requests;
}
}

View file

@ -1,26 +1,24 @@
package org.micord.models.requests; package org.micord.models.requests;
import jakarta.xml.bind.annotation.XmlAccessType;
import jakarta.xml.bind.annotation.XmlAccessorType;
import jakarta.xml.bind.annotation.XmlElement; import jakarta.xml.bind.annotation.XmlElement;
import jakarta.xml.bind.annotation.XmlElementWrapper; import jakarta.xml.bind.annotation.XmlElementWrapper;
import lombok.Getter;
import lombok.Setter; import lombok.Setter;
import java.util.List; import java.util.List;
@Getter
@Setter @Setter
@XmlAccessorType(XmlAccessType.FIELD)
public class AqlRequest extends BaseRequest { public class AqlRequest extends BaseRequest {
private AqlConnectionParams aqlConnectionParams;
private List<AqlRequestCollection> aqlRequestCollections;
@XmlElement(name = "AqlConnectionParams") @XmlElement(name = "AqlConnectionParams")
public AqlConnectionParams getAqlConnectionParams() { private AqlConnectionParams aqlConnectionParams;
return aqlConnectionParams;
}
@XmlElementWrapper(name = "AqlRequestCollections") @XmlElementWrapper(name = "AqlRequestCollections")
@XmlElement(name = "AqlRequestCollection") @XmlElement(name = "AqlRequestCollection")
public List<AqlRequestCollection> getAqlRequestCollections() { private List<AqlRequestCollection> aqlRequestCollections;
return aqlRequestCollections;
}
} }

View file

@ -8,6 +8,7 @@ import lombok.Setter;
public class AqlRequestCollection { public class AqlRequestCollection {
private String type; private String type;
private String dateAttribute;
private String collectionUrl; private String collectionUrl;
@XmlAttribute(name = "type") @XmlAttribute(name = "type")
@ -15,6 +16,11 @@ public class AqlRequestCollection {
return type; return type;
} }
@XmlAttribute(name = "dateAttribute")
public String getDateAttribute() {
return dateAttribute;
}
@XmlValue @XmlValue
public String getCollectionUrl() { public String getCollectionUrl() {
return collectionUrl; return collectionUrl;

View file

@ -1,29 +1,27 @@
package org.micord.models.requests; package org.micord.models.requests;
import jakarta.xml.bind.annotation.XmlAccessType;
import jakarta.xml.bind.annotation.XmlAccessorType;
import jakarta.xml.bind.annotation.XmlElement; import jakarta.xml.bind.annotation.XmlElement;
import jakarta.xml.bind.annotation.XmlSeeAlso; import jakarta.xml.bind.annotation.XmlSeeAlso;
import lombok.Getter;
import lombok.Setter; import lombok.Setter;
import org.micord.models.requests.downloads.AQLDownloadRequest;
import org.micord.models.requests.downloads.SQLDownloadRequest;
import java.util.List; import java.util.List;
/** @Getter
* @author Maksim Tereshin
*/
@Setter @Setter
@XmlSeeAlso({SqlRequest.class, S3Request.class, AqlRequest.class}) @XmlAccessorType(XmlAccessType.FIELD)
public abstract class BaseRequest { public abstract class BaseRequest {
private List<RequestArgument> requestArguments;
private String requestURL;
@XmlElement(name = "RequestArgument") @XmlElement(name = "RequestArgument")
public List<RequestArgument> getRequestArguments() { private List<RequestArgument> requestArguments;
return requestArguments;
}
@XmlElement(name = "RequestURL") @XmlElement(name = "RequestURL")
public String getRequestURL() { private String requestURL;
return requestURL;
}
@XmlElement(name = "RequestValidationRules")
private RequestValidationRules requestValidationRules;
} }

View file

@ -1,16 +0,0 @@
package org.micord.models.requests;
import lombok.Data;
import java.time.LocalDate;
import java.util.List;
@Data
public class DownloadCSVRequest {
private String type;
private List<String> ids;
private LocalDate startDate;
private LocalDate endDate;
}

View file

@ -1,23 +0,0 @@
package org.micord.models.requests;
import jakarta.xml.bind.annotation.XmlElement;
import lombok.Setter;
@Setter
public class DownloadRequest extends BaseRequest {
private SqlConnectionParams sqlConnectionParams;
private String downloadRequestType;
@XmlElement(name = "SqlConnectionParams")
public SqlConnectionParams getSqlConnectionParams() {
return sqlConnectionParams;
}
@XmlElement(name = "DownloadRequestType")
public String getDownloadRequestType() {
return downloadRequestType;
}
}

View file

@ -0,0 +1,24 @@
package org.micord.models.requests;
import jakarta.xml.bind.annotation.XmlAccessType;
import jakarta.xml.bind.annotation.XmlAccessorType;
import jakarta.xml.bind.annotation.XmlAttribute;
import jakarta.xml.bind.annotation.XmlRootElement;
import lombok.Getter;
import lombok.Setter;
@Getter
@Setter
@XmlRootElement(name = "RequestValidationRules")
@XmlAccessorType(XmlAccessType.FIELD)
public class RequestValidationRules {
@XmlAttribute(name = "isEmptyIdsAllowed")
private Boolean isEmptyIdsAllowed;
@XmlAttribute(name = "isEmptyDatesAllowed")
private Boolean isEmptyDatesAllowed;
@XmlAttribute(name = "isIdsFormatted")
private Boolean isIdsFormatted;
}

View file

@ -1,41 +1,35 @@
package org.micord.models.requests; package org.micord.models.requests;
import jakarta.xml.bind.annotation.XmlAccessType;
import jakarta.xml.bind.annotation.XmlAccessorType;
import jakarta.xml.bind.annotation.XmlElement; import jakarta.xml.bind.annotation.XmlElement;
import jakarta.xml.bind.annotation.XmlRootElement; import jakarta.xml.bind.annotation.XmlRootElement;
import lombok.Getter;
import lombok.Setter; import lombok.Setter;
import org.micord.models.requests.downloads.AQLDownloadRequest;
import org.micord.models.requests.downloads.SQLDownloadRequest;
import java.util.List; import java.util.List;
/** @Getter
* @author Maksim Tereshin
*/
@Setter @Setter
@XmlRootElement(name = "Requests") @XmlRootElement(name = "Requests")
@XmlAccessorType(XmlAccessType.FIELD)
public class Requests { public class Requests {
private List<SqlRequest> sqlRequests;
private List<AqlRequest> aqlRequests;
private List<S3Request> s3Requests;
private List<DownloadRequest> downloadRequests;
@XmlElement(name = "DownloadRequest")
public List<DownloadRequest> getDownloadRequests() {
return downloadRequests;
}
@XmlElement(name = "SqlRequest") @XmlElement(name = "SqlRequest")
public List<SqlRequest> getSqlRequests() { private List<SqlRequest> sqlRequests;
return sqlRequests;
}
@XmlElement(name = "AqlRequest") @XmlElement(name = "AqlRequest")
public List<AqlRequest> getAqlRequests() { private List<AqlRequest> aqlRequests;
return aqlRequests;
}
@XmlElement(name = "S3Request") @XmlElement(name = "S3Request")
public List<S3Request> getS3Requests() { private List<S3Request> s3Requests;
return s3Requests;
} @XmlElement(name = "AQLDownloadRequest")
private List<AQLDownloadRequest> aqlDownloadRequests;
@XmlElement(name = "SQLDownloadRequest")
private List<SQLDownloadRequest> sqlDownloadRequests;
} }

View file

@ -1,19 +1,17 @@
package org.micord.models.requests; package org.micord.models.requests;
import jakarta.xml.bind.annotation.XmlAccessType;
import jakarta.xml.bind.annotation.XmlAccessorType;
import jakarta.xml.bind.annotation.XmlElement; import jakarta.xml.bind.annotation.XmlElement;
import lombok.Getter;
import lombok.Setter; import lombok.Setter;
/** @Getter
* @author Maksim Tereshin
*/
@Setter @Setter
@XmlAccessorType(XmlAccessType.FIELD)
public class S3Request extends BaseRequest { public class S3Request extends BaseRequest {
@XmlElement(name = "S3ConnectionParams")
private S3ConnectionParams s3ConnectionParams; private S3ConnectionParams s3ConnectionParams;
@XmlElement(name = "S3ConnectionParams")
public S3ConnectionParams getS3ConnectionParams() {
return s3ConnectionParams;
}
} }

View file

@ -1,19 +1,17 @@
package org.micord.models.requests; package org.micord.models.requests;
import jakarta.xml.bind.annotation.XmlAccessType;
import jakarta.xml.bind.annotation.XmlAccessorType;
import jakarta.xml.bind.annotation.XmlElement; import jakarta.xml.bind.annotation.XmlElement;
import lombok.Getter;
import lombok.Setter; import lombok.Setter;
/** @Getter
* @author Maksim Tereshin
*/
@Setter @Setter
@XmlAccessorType(XmlAccessType.FIELD)
public class SqlRequest extends BaseRequest { public class SqlRequest extends BaseRequest {
@XmlElement(name = "SqlConnectionParams")
private SqlConnectionParams sqlConnectionParams; private SqlConnectionParams sqlConnectionParams;
@XmlElement(name = "SqlConnectionParams")
public SqlConnectionParams getSqlConnectionParams() {
return sqlConnectionParams;
}
} }

View file

@ -0,0 +1,27 @@
package org.micord.models.requests.downloads;
import jakarta.xml.bind.annotation.*;
import lombok.Getter;
import lombok.Setter;
import org.micord.models.requests.AqlConnectionParams;
import org.micord.models.requests.AqlRequestCollection;
import org.micord.models.requests.SqlConnectionParams;
import java.util.List;
@Getter
@Setter
@XmlAccessorType(XmlAccessType.FIELD)
public class AQLDownloadRequest extends BaseDownloadRequest {
@XmlElement(name = "AqlConnectionParams")
private AqlConnectionParams aqlConnectionParams;
@XmlElement(name = "DownloadRequestEntitySelectorQuery")
private String downloadRequestEntitySelectorQuery;
@XmlElementWrapper(name = "AqlRequestCollections")
@XmlElement(name = "AqlRequestCollection")
private List<AqlRequestCollection> aqlRequestCollections;
}

View file

@ -0,0 +1,18 @@
package org.micord.models.requests.downloads;
import jakarta.xml.bind.annotation.XmlAccessType;
import jakarta.xml.bind.annotation.XmlAccessorType;
import jakarta.xml.bind.annotation.XmlElement;
import jakarta.xml.bind.annotation.XmlSeeAlso;
import lombok.Getter;
import lombok.Setter;
import org.micord.models.requests.BaseRequest;
@Getter
@Setter
@XmlAccessorType(XmlAccessType.FIELD)
public abstract class BaseDownloadRequest extends BaseRequest {
@XmlElement(name = "DownloadRequestType")
private String downloadRequestType;
}

View file

@ -0,0 +1,19 @@
package org.micord.models.requests.downloads;
import jakarta.xml.bind.annotation.XmlAccessType;
import jakarta.xml.bind.annotation.XmlAccessorType;
import jakarta.xml.bind.annotation.XmlElement;
import lombok.Getter;
import lombok.Setter;
import org.micord.models.requests.SqlConnectionParams;
@Setter
@Getter
@XmlAccessorType(XmlAccessType.FIELD)
public class SQLDownloadRequest extends BaseDownloadRequest {
@XmlElement(name = "SqlConnectionParams")
private SqlConnectionParams sqlConnectionParams;
}

View file

@ -1,8 +1,7 @@
package org.micord.service; package org.micord.service;
import org.micord.enums.ConfigType; import org.micord.enums.ConfigType;
import org.micord.models.requests.DownloadCSVRequest; import org.micord.models.requests.downloads.*;
import org.micord.models.requests.DownloadRequest;
import org.micord.models.requests.RequestParameters; import org.micord.models.requests.RequestParameters;
import org.micord.models.requests.Requests; import org.micord.models.requests.Requests;
import org.slf4j.Logger; import org.slf4j.Logger;
@ -13,9 +12,12 @@ import org.springframework.stereotype.Service;
import java.io.File; import java.io.File;
import java.io.FileNotFoundException; import java.io.FileNotFoundException;
import java.io.IOException; import java.io.IOException;
import java.sql.SQLException;
import java.util.List; import java.util.List;
import java.util.Map;
import java.util.Optional; import java.util.Optional;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import java.util.stream.Stream;
@Service @Service
public class ApiService { public class ApiService {
@ -31,37 +33,51 @@ public class ApiService {
@Autowired @Autowired
private ConfigService configService; private ConfigService configService;
public void process(ConfigType methodName, List<String> ids) throws FileNotFoundException { @Autowired
private ValidationService validationService;
public void process(ConfigType methodName, List<String> ids) throws FileNotFoundException, SQLException {
Requests config = configService.getConfig(methodName, Requests.class); Requests config = configService.getConfig(methodName, Requests.class);
sqlAndAqlService.processSqlAndAqlRequests(config, ids); sqlAndAqlService.processSqlAndAqlRequests(config, ids);
} }
public void process(ConfigType methodName, RequestParameters parameters) throws FileNotFoundException { public void process(ConfigType methodName, RequestParameters parameters) throws FileNotFoundException, SQLException {
Requests config = configService.getConfig(methodName, Requests.class); Requests config = configService.getConfig(methodName, Requests.class);
sqlAndAqlService.processSqlAndAqlRequests(config, parameters); sqlAndAqlService.processSqlAndAqlRequests(config, parameters);
} }
public File download(ConfigType methodName, DownloadCSVRequest request) throws IOException { public File download(ConfigType methodName, RequestParameters downloadRequest) throws IOException {
Requests config = configService.getConfig(methodName, Requests.class); Requests config = configService.getConfig(methodName, Requests.class);
String type = request.getType(); String type = downloadRequest.getType();
List<String> ids = Optional.ofNullable(request.getIds()) List<String> ids = Optional.ofNullable(downloadRequest.getIds())
.filter(list -> !list.isEmpty()) .filter(list -> !list.isEmpty())
.orElse(null); .orElse(null);
DownloadRequest selectedRequest = config.getDownloadRequests().stream() BaseDownloadRequest selectedRequest = config.getAqlDownloadRequests().stream()
.filter(r -> r.getDownloadRequestType().equals(type)) .filter(r -> r.getDownloadRequestType().equals(type))
.findFirst() .findFirst()
.map(BaseDownloadRequest.class::cast)
.or(() -> config.getSqlDownloadRequests().stream()
.filter(r -> r.getDownloadRequestType().equals(type))
.findFirst()
.map(BaseDownloadRequest.class::cast))
.orElseThrow(() -> new IllegalArgumentException("Invalid download type: " + type)); .orElseThrow(() -> new IllegalArgumentException("Invalid download type: " + type));
return downloadService.download(selectedRequest, ids, request.getStartDate(), request.getEndDate()); Map<String, Boolean> validationResults = validationService.validateDownloadRequest(selectedRequest, downloadRequest, ids);
return downloadService.download(selectedRequest, ids, downloadRequest, validationResults);
} }
public List<String> getDownloadTypes(ConfigType methodName) throws FileNotFoundException { public List<String> getDownloadTypes(ConfigType methodName) throws FileNotFoundException {
Requests config = configService.getConfig(methodName, Requests.class); Requests config = configService.getConfig(methodName, Requests.class);
return config.getDownloadRequests().stream() return Stream.concat(
.map(DownloadRequest::getDownloadRequestType) config.getSqlDownloadRequests().stream(),
config.getAqlDownloadRequests().stream()
)
.map(BaseDownloadRequest::getDownloadRequestType)
.distinct()
.collect(Collectors.toList()); .collect(Collectors.toList());
} }

View file

@ -1,7 +1,15 @@
package org.micord.service; package org.micord.service;
import com.arangodb.ArangoCursor;
import com.arangodb.ArangoDBException;
import com.arangodb.ArangoDatabase;
import com.arangodb.model.AqlQueryOptions;
import org.micord.config.ArangoDBConnection;
import org.micord.config.DatabaseConnection; import org.micord.config.DatabaseConnection;
import org.micord.models.requests.DownloadRequest; import org.micord.models.requests.RequestParameters;
import org.micord.models.requests.downloads.AQLDownloadRequest;
import org.micord.models.requests.downloads.BaseDownloadRequest;
import org.micord.models.requests.downloads.SQLDownloadRequest;
import org.micord.models.requests.RequestArgument; import org.micord.models.requests.RequestArgument;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@ -14,10 +22,7 @@ import java.sql.PreparedStatement;
import java.sql.ResultSet; import java.sql.ResultSet;
import java.sql.SQLException; import java.sql.SQLException;
import java.time.LocalDate; import java.time.LocalDate;
import java.util.ArrayList; import java.util.*;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors; import java.util.stream.Collectors;
@Service @Service
@ -25,13 +30,59 @@ public class DownloadService {
private static final Logger logger = LoggerFactory.getLogger(DownloadService.class); private static final Logger logger = LoggerFactory.getLogger(DownloadService.class);
public File download(DownloadRequest selectedRequest, List<String> ids, LocalDate startDate, LocalDate endDate) { public File download(BaseDownloadRequest selectedRequest, List<String> ids, RequestParameters parameters, Map<String, Boolean> validationResults) {
LocalDate startDate = parameters.getStartDate();
return processDownloadRequest(selectedRequest, ids, startDate, endDate); LocalDate endDate = parameters.getEndDate();
if (selectedRequest instanceof SQLDownloadRequest) {
return processSqlDownloadRequest((SQLDownloadRequest) selectedRequest, ids, startDate, endDate, validationResults);
} else if (selectedRequest instanceof AQLDownloadRequest) {
return processAqlDownloadRequest((AQLDownloadRequest) selectedRequest, ids, startDate, endDate, validationResults);
}
throw new IllegalArgumentException("Unsupported request type: " + selectedRequest.getClass().getSimpleName());
} }
private File processDownloadRequest(DownloadRequest request, List<String> ids, LocalDate startDate, LocalDate endDate) { private File processAqlDownloadRequest(AQLDownloadRequest request, List<String> ids, LocalDate startDate, LocalDate endDate, Map<String, Boolean> validationResults) {
Map<String, Object> query = buildSqlQuery(request, ids, startDate, endDate); ArangoDatabase arangoDb = ArangoDBConnection.getConnection(request.getAqlConnectionParams());
Boolean emptyIdsAllowed = validationResults.getOrDefault(ValidationService.IS_EMPTY_IDS_ALLOWED, false);
Boolean emptyDatesAllowed = validationResults.getOrDefault(ValidationService.IS_EMPTY_DATES_ALLOWED, false);
Map<String, Object> entities = executeSelectAqlRequest(arangoDb, request.getDownloadRequestEntitySelectorQuery(), ids, emptyIdsAllowed);
if (entities.isEmpty()) {
logger.warn("No entities found for main AQL request.");
return null;
}
List<Map<String, Object>> results = new ArrayList<>();
request.getAqlRequestCollections().forEach(collection -> {
String type = collection.getCollectionUrl();
String entityType;
if (Objects.equals(type, "applications")) {
entityType = "applicationId";
} else {
entityType = type + "Id";
}
Object entityIds = entities.get(entityType);
if (entityIds instanceof String) {
entityIds = Collections.singletonList((String) entityIds);
}
String aqlQuery = buildAqlQuery(type, ids, collection.getDateAttribute(), startDate, endDate, emptyIdsAllowed, emptyDatesAllowed);
results.addAll(executeAqlQuery(arangoDb, aqlQuery, (List<String>) entityIds, startDate, endDate, emptyIdsAllowed, emptyDatesAllowed));
});
return writeResultsToCsv(results);
}
private File processSqlDownloadRequest(SQLDownloadRequest request, List<String> ids, LocalDate startDate, LocalDate endDate, Map<String, Boolean> validationResults) {
Map<String, Object> query = buildSqlQuery(request, ids, startDate, endDate, validationResults);
try (Connection connection = DatabaseConnection.getConnection( try (Connection connection = DatabaseConnection.getConnection(
request.getSqlConnectionParams())) { request.getSqlConnectionParams())) {
String requestURL = (String) query.get("requestURL"); String requestURL = (String) query.get("requestURL");
@ -62,6 +113,134 @@ public class DownloadService {
return null; return null;
} }
private Map<String, Object> executeSelectAqlRequest(ArangoDatabase arangoDb,
String downloadRequestEntitySelectorQuery,
List<String> ids, Boolean emptyIdsAllowed) {
Map<String, Object> entities = new HashMap<>();
Map<String, Object> bindVars = new HashMap<>();
if (!emptyIdsAllowed) bindVars.put("ids", ids);
AqlQueryOptions aqlQueryOptions = new AqlQueryOptions();
try (ArangoCursor<Map> cursor = arangoDb.query(downloadRequestEntitySelectorQuery, Map.class, bindVars, aqlQueryOptions)) {
while (cursor.hasNext()) {
Map<String, Object> result = cursor.next();
for (Map.Entry<String, Object> entry : result.entrySet()) {
String key = entry.getKey();
Object entityValue = entry.getValue();
entities.put(key, entityValue);
}
}
}
catch (Exception e) {
logger.error("Failed to execute AQL url", e);
}
return entities;
}
private String buildAqlQuery(String collectionName, List<String> ids, String dateAttribute, LocalDate startDate, LocalDate endDate, Boolean emptyIdsAllowed, Boolean emptyDatesAllowed) {
StringBuilder queryBuilder = new StringBuilder();
queryBuilder.append("FOR doc IN ").append(collectionName).append(" ");
List<String> conditions = new ArrayList<>();
if (!emptyIdsAllowed && ids != null && !ids.isEmpty()) {
conditions.add("doc._key IN @ids");
}
if (!emptyDatesAllowed && dateAttribute != null) {
if (startDate != null) {
conditions.add("doc." + dateAttribute + " >= @startDate");
}
if (endDate != null) {
conditions.add("doc." + dateAttribute + " <= @endDate");
}
}
if (!conditions.isEmpty()) {
queryBuilder.append("FILTER ").append(String.join(" AND ", conditions)).append(" ");
}
queryBuilder.append("RETURN doc");
return queryBuilder.toString();
}
private List<Map<String, Object>> executeAqlQuery(ArangoDatabase arangoDb, String aqlQuery, List<String> ids, LocalDate startDate, LocalDate endDate, Boolean emptyIdsAllowed, Boolean emptyDatesAllowed) {
List<Map<String, Object>> results = new ArrayList<>();
try {
Map<String, Object> bindVars = new HashMap<>();
if (!emptyIdsAllowed && ids != null && !ids.isEmpty()) {
bindVars.put("ids", ids);
}
if (!emptyDatesAllowed) {
if (startDate != null) {
bindVars.put("startDate", startDate.toString());
}
if (endDate != null) {
bindVars.put("endDate", endDate.toString());
}
}
logger.info("Executing AQL query: {}\nWith bindVars: {}", aqlQuery, bindVars);
AqlQueryOptions aqlQueryOptions = new AqlQueryOptions();
try (ArangoCursor<Map> cursor = arangoDb.query(aqlQuery, Map.class, bindVars, aqlQueryOptions)) {
while (cursor.hasNext()) {
results.add(cursor.next());
}
} catch (IOException e) {
throw new RuntimeException(e);
}
} catch (ArangoDBException e) {
logger.error("AQL query execution failed: {}\nError: {}", aqlQuery, e.getMessage(), e);
}
return results;
}
private File writeResultsToCsv(List<Map<String, Object>> results) {
File csvFile;
try {
csvFile = File.createTempFile("arango-download-", ".csv");
try (BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(csvFile), StandardCharsets.UTF_8))) {
if (!results.isEmpty()) {
List<String> headers = new ArrayList<>(results.get(0).keySet());
writer.write(String.join(",", headers));
writer.newLine();
for (Map<String, Object> row : results) {
List<String> rowValues = headers.stream()
.map(header -> formatCsvField(row.get(header)))
.collect(Collectors.toList());
writer.write(String.join(",", rowValues));
writer.newLine();
}
}
}
} catch (IOException e) {
logger.error("Failed to write results to CSV", e);
return null;
}
return csvFile;
}
private String formatCsvField(Object value) {
if (value == null) {
return "\"\"";
}
String strValue = value.toString().replace("\"", "\"\"");
return "\"" + strValue + "\"";
}
private String formatCsvRow(String[] row) { private String formatCsvRow(String[] row) {
StringBuilder formattedRow = new StringBuilder(); StringBuilder formattedRow = new StringBuilder();
@ -85,16 +264,27 @@ public class DownloadService {
return field.replace("\"", "\"\""); return field.replace("\"", "\"\"");
} }
private Map<String, Object> buildSqlQuery(DownloadRequest request, List<String> ids, LocalDate startDate, LocalDate endDate) { private Map<String, Object> buildSqlQuery(SQLDownloadRequest request, List<String> ids, LocalDate startDate, LocalDate endDate, Map<String, Boolean> validationResults) {
Boolean emptyIdsAllowed = validationResults.getOrDefault(ValidationService.IS_EMPTY_IDS_ALLOWED, false);
Boolean emptyDatesAllowed = validationResults.getOrDefault(ValidationService.IS_EMPTY_DATES_ALLOWED, false);
Map<String, Object> resultMap = new HashMap<>(); Map<String, Object> resultMap = new HashMap<>();
String endpointArguments; String endpointArguments;
String requestURL = prepareRequestURL(request, startDate, endDate); String requestURL = request.getRequestURL();
if (!emptyDatesAllowed) {
requestURL = prepareRequestURL(request, startDate, endDate);
}
if (emptyIdsAllowed != null && emptyIdsAllowed) {
resultMap.put("requestURL", requestURL.replace("where id in ${endpointArguments}", ""));
return resultMap;
}
if (ids == null || ids.isEmpty()) { if (ids == null || ids.isEmpty()) {
resultMap.put("requestURL", requestURL resultMap.put("requestURL", requestURL.replace("where id in ${endpointArguments}", ""));
.replace("where id in ${endpointArguments}", ""));
return resultMap; return resultMap;
} }
@ -143,7 +333,7 @@ public class DownloadService {
return resultMap; return resultMap;
} }
private String prepareRequestURL(DownloadRequest request, LocalDate startDate, private String prepareRequestURL(SQLDownloadRequest request, LocalDate startDate,
LocalDate endDate) { LocalDate endDate) {
String requestURL = request.getRequestURL(); String requestURL = request.getRequestURL();

View file

@ -10,6 +10,7 @@ import org.micord.config.ArangoDBConnection;
import org.micord.config.DatabaseConnection; import org.micord.config.DatabaseConnection;
import org.micord.config.S3HttpConnection; import org.micord.config.S3HttpConnection;
import org.micord.enums.RequestArgumentType; import org.micord.enums.RequestArgumentType;
import org.micord.exceptions.IllegalRequestParametersException;
import org.micord.models.requests.*; import org.micord.models.requests.*;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@ -22,6 +23,7 @@ import org.xml.sax.InputSource;
import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.DocumentBuilderFactory;
import java.io.FileNotFoundException;
import java.io.StringReader; import java.io.StringReader;
import java.net.HttpURLConnection; import java.net.HttpURLConnection;
import java.net.http.HttpClient; import java.net.http.HttpClient;
@ -36,9 +38,6 @@ import java.util.*;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import java.util.stream.IntStream; import java.util.stream.IntStream;
/**
* @author Maksim Tereshin
*/
@Service @Service
public class RequestService { public class RequestService {
@ -47,23 +46,10 @@ public class RequestService {
@Autowired @Autowired
private HttpClient httpClient; private HttpClient httpClient;
public void processS3Requests(List<S3Request> s3Requests, List<String> ids) { @Autowired
logger.info("A. Starting processing of S3 requests"); private ValidationService validationService;
for (S3Request request : s3Requests) { private void processS3Request(S3Request request, RequestParameters parameters, Map<String, Boolean> validationResults) {
processS3Request(request, ids);
}
}
public void processS3Requests(List<S3Request> s3Requests, RequestParameters parameters) {
logger.info("A. Starting processing of S3 requests");
for (S3Request request : s3Requests) {
processS3Request(request, parameters);
}
}
private void processS3Request(S3Request request, RequestParameters parameters) {
logger.info("B. Starting processing of single S3 request"); logger.info("B. Starting processing of single S3 request");
try { try {
List<String> files = new ArrayList<>(); List<String> files = new ArrayList<>();
@ -74,7 +60,7 @@ public class RequestService {
try (Connection connection = DatabaseConnection.getConnection( try (Connection connection = DatabaseConnection.getConnection(
argument.getRequestArgumentConnectionParams())) { argument.getRequestArgumentConnectionParams())) {
Map<String, Object> query = buildSqlQueryForS3(argument.getRequestArgumentURL(), parameters); Map<String, Object> query = buildSqlQueryForS3(argument.getRequestArgumentURL(), parameters, validationResults);
logger.info("C. Calling query {} for ids {}: ", query.get("requestURL"), ids); logger.info("C. Calling query {} for ids {}: ", query.get("requestURL"), ids);
logger.debug("Starting fetching paths from database for S3 request"); logger.debug("Starting fetching paths from database for S3 request");
long startExecTime = System.currentTimeMillis(); long startExecTime = System.currentTimeMillis();
@ -126,8 +112,9 @@ public class RequestService {
} }
} }
private void processS3Request(S3Request request, List<String> ids) { private void processS3Request(S3Request request, List<String> ids, Map<String, Boolean> validationResults) {
logger.info("B. Starting processing of single S3 request"); logger.info("B. Starting processing of single S3 request");
Boolean emptyIdsAllowed = validationResults.getOrDefault(ValidationService.IS_EMPTY_IDS_ALLOWED, false);
try { try {
List<String> files = new ArrayList<>(); List<String> files = new ArrayList<>();
@ -136,17 +123,21 @@ public class RequestService {
try (Connection connection = DatabaseConnection.getConnection( try (Connection connection = DatabaseConnection.getConnection(
argument.getRequestArgumentConnectionParams())) { argument.getRequestArgumentConnectionParams())) {
Map<String, Object> query = buildSqlQueryForS3(argument.getRequestArgumentURL(), ids); String requestURL = argument.getRequestArgumentURL();
logger.info("C. Calling query {} for ids {}: ", query.get("requestURL"), ids); if (!emptyIdsAllowed) {
Map<String, Object> sqlQueryForS3 = buildSqlQueryForS3(argument.getRequestArgumentURL(), ids);
requestURL = (String) sqlQueryForS3.get("requestURL");
}
logger.info("C. Calling query {} for ids {}: ", requestURL, ids);
logger.debug("Starting fetching paths from database for S3 request"); logger.debug("Starting fetching paths from database for S3 request");
long startExecTime = System.currentTimeMillis(); long startExecTime = System.currentTimeMillis();
List<String> result = fetchFileListFromDatabaseSQL(connection, (String) query.get("requestURL")); List<String> result = fetchFileListFromDatabaseSQL(connection, requestURL);
String formattedResult = IntStream.range(0, result.size()) String formattedResult = IntStream.range(0, result.size())
.mapToObj(i -> (i + 1) + ". " + result.get(i)) .mapToObj(i -> (i + 1) + ". " + result.get(i))
.collect(Collectors.joining("\n")); .collect(Collectors.joining("\n"));
logger.info("D. Found files for query {}:\n{}", query.get("requestURL"), formattedResult); logger.info("D. Found files for query {}:\n{}", requestURL, formattedResult);
if (result != null && !result.isEmpty()) { if (result != null && !result.isEmpty()) {
files.addAll(result); files.addAll(result);
@ -231,75 +222,99 @@ public class RequestService {
@Transactional @Transactional
public void processSqlAndAqlRequests(Requests config, List<String> ids) { public void processSqlAndAqlRequests(Requests config, List<String> ids) throws SQLException, FileNotFoundException {
logger.debug("Starting transactional processing of requests"); logger.debug("Starting transactional processing of requests");
if (config.getS3Requests() != null && !config.getS3Requests().isEmpty()) { if (config.getS3Requests() != null && !config.getS3Requests().isEmpty()) {
processS3Requests(config.getS3Requests(), ids); logger.info("A. Starting processing of S3 requests");
for (S3Request request : config.getS3Requests()) {
Map<String, Boolean> validationResults = validationService.validateRequest(request, ids);
processS3Request(request, ids, validationResults);
}
} }
if (config.getSqlRequests() != null) { if (config.getSqlRequests() != null) {
for (SqlRequest request : config.getSqlRequests()) { for (SqlRequest request : config.getSqlRequests()) {
processSqlRequests(request, ids); Map<String, Boolean> validationResults = validationService.validateRequest(request, ids);
processSqlRequests(request, ids, validationResults);
} }
} }
if (config.getAqlRequests() != null) { if (config.getAqlRequests() != null) {
for (AqlRequest request : config.getAqlRequests()) { for (AqlRequest request : config.getAqlRequests()) {
processAqlRequests(request, ids); Map<String, Boolean> validationResults = validationService.validateRequest(request, ids);
processAqlRequests(request, ids, validationResults);
} }
} }
} }
@Transactional @Transactional
public void processSqlAndAqlRequests(Requests config, RequestParameters parameters) { public void processSqlAndAqlRequests(Requests config, RequestParameters parameters) throws SQLException, FileNotFoundException {
logger.debug("Starting transactional processing of requests"); logger.debug("Starting transactional processing of requests");
if (config.getS3Requests() != null && !config.getS3Requests().isEmpty()) { if (config.getS3Requests() != null && !config.getS3Requests().isEmpty()) {
processS3Requests(config.getS3Requests(), parameters);
logger.info("A. Starting processing of S3 requests");
for (S3Request request : config.getS3Requests()) {
Map<String, Boolean> validationResults = validationService.validateMilitaryNoticeRequest(request, parameters);
processS3Request(request, parameters, validationResults);
}
} }
if (config.getSqlRequests() != null) { if (config.getSqlRequests() != null) {
for (SqlRequest request : config.getSqlRequests()) { for (SqlRequest request : config.getSqlRequests()) {
processSqlRequests(request, parameters); Map<String, Boolean> validationResults = validationService.validateMilitaryNoticeRequest(request, parameters);
processSqlRequests(request, parameters, validationResults);
} }
} }
if (config.getAqlRequests() != null) { if (config.getAqlRequests() != null) {
for (AqlRequest request : config.getAqlRequests()) { for (AqlRequest request : config.getAqlRequests()) {
processAqlRequests(request, parameters.getIds()); Map<String, Boolean> validationResults = validationService.validateMilitaryNoticeRequest(request, parameters);
processAqlRequests(request, parameters.getIds(), validationResults);
} }
} }
} }
private void processSqlRequests(SqlRequest request, List<String> ids) { private void processSqlRequests(SqlRequest request, List<String> ids, Map<String, Boolean> validationResults) {
logger.debug("Starting transactional processing of SQL requests"); logger.debug("Starting transactional processing of SQL requests");
Map<String, Object> query = buildSqlQuery(request, ids); Boolean emptyIdsAllowed = validationResults.getOrDefault(ValidationService.IS_EMPTY_IDS_ALLOWED, false);
String requestURL;
Map<String, Object> query = null;
if (emptyIdsAllowed != null && emptyIdsAllowed) {
requestURL = request.getRequestURL();
logger.info("Empty IDs allowed. Using original request URL: {}", requestURL);
} else {
query = buildSqlQuery(request, ids);
requestURL = (String) query.get("requestURL");
}
logger.debug("Opening connection for SQL Request: {}", request.getRequestURL()); logger.debug("Opening connection for SQL Request: {}", request.getRequestURL());
long startExecTime = System.currentTimeMillis(); long startExecTime = System.currentTimeMillis();
try (Connection connection = DatabaseConnection.getConnection( try (Connection connection = DatabaseConnection.getConnection(request.getSqlConnectionParams())) {
request.getSqlConnectionParams())) {
String requestURL = (String) query.get("requestURL");
executeSqlQuery(connection, requestURL); executeSqlQuery(connection, requestURL);
if (query != null) {
List<String> queryIds = (List<String>) query.get("ids"); List<String> queryIds = (List<String>) query.get("ids");
if (queryIds != null && !queryIds.isEmpty()) { if (queryIds != null && !queryIds.isEmpty()) {
ids.addAll(queryIds); ids.addAll(queryIds);
} else { } else {
logger.warn("No IDs found for the query"); logger.warn("No IDs found for the query");
} }
}
long endExecTime = System.currentTimeMillis(); long endExecTime = System.currentTimeMillis();
logger.debug("SQL request executed in {} ms", endExecTime - startExecTime); logger.debug("SQL request executed in {} ms", endExecTime - startExecTime);
logger.info("Successfully executed query {} for IDs: ({})", requestURL, String.join(", ", ids)); logger.info("Successfully executed query {} for IDs: ({})", requestURL, String.join(", ", ids));
} } catch (SQLException e) {
catch (SQLException e) { logger.error("SQL execution failed for query: {}", requestURL, e);
logger.error("SQL execution failed for query: {}", query, e);
throw new RuntimeException("Error executing SQL query", e); throw new RuntimeException("Error executing SQL query", e);
} }
} }
private void processSqlRequests(SqlRequest request, RequestParameters parameters) { private void processSqlRequests(SqlRequest request, RequestParameters parameters, Map<String, Boolean> validationResults) {
logger.debug("Starting transactional processing of SQL requests"); logger.debug("Starting transactional processing of SQL requests");
Map<String, Object> query = buildSqlQuery(request, parameters); Map<String, Object> query = buildSqlQuery(request, parameters, validationResults);
List<String> ids = parameters.getIds(); List<String> ids = parameters.getIds();
logger.debug("Opening connection for SQL Request: {}", request.getRequestURL()); logger.debug("Opening connection for SQL Request: {}", request.getRequestURL());
long startExecTime = System.currentTimeMillis(); long startExecTime = System.currentTimeMillis();
@ -350,33 +365,49 @@ public class RequestService {
return resultMap; return resultMap;
} }
private Map<String, Object> buildSqlQueryForS3(String url, RequestParameters parameters) { private Map<String, Object> buildSqlQueryForS3(String url, RequestParameters parameters, Map<String, Boolean> validationResults) {
logger.debug("Starting building SQL query for request: {}", url); logger.debug("Starting building SQL query for request: {}", url);
long startExecTime = System.currentTimeMillis(); long startExecTime = System.currentTimeMillis();
Map<String, Object> resultMap = new HashMap<>(); Map<String, Object> resultMap = new HashMap<>();
String endpointArguments;
String requestURL = prepareDatesFilterInRequestURL(url, parameters.getStartDate(), parameters.getEndDate()); Boolean isEmptyDatesAllowed = validationResults.getOrDefault(ValidationService.IS_EMPTY_DATES_ALLOWED, false);
List<String> ids = parameters.getIds(); String requestURL = applyDateFilter(url, parameters.getStartDate(), parameters.getEndDate(), isEmptyDatesAllowed);
if (isEmptyDatesAllowed) {
if (requestURL.contains(":=")) { logger.info("Skipping date filtering as empty dates are allowed.");
endpointArguments = "'{" + ids.stream()
.map(String::trim)
.collect(Collectors.joining(", ")) + "}'";
} else {
endpointArguments = "(" + ids.stream()
.map(s -> "'" + s.trim() + "'")
.collect(Collectors.joining(", ")) + ")";
} }
resultMap.put("requestURL", requestURL String finalUrl = applyEndpointArguments(requestURL, parameters.getIds());
.replace("${endpointArguments}", endpointArguments)); resultMap.put("requestURL", finalUrl);
long endExecTime = System.currentTimeMillis(); long endExecTime = System.currentTimeMillis();
logger.debug("SQL query for S3 built in {} ms", endExecTime - startExecTime); logger.debug("SQL query for S3 built in {} ms", endExecTime - startExecTime);
return resultMap; return Collections.unmodifiableMap(resultMap);
}
private String applyDateFilter(String url, LocalDate startDate, LocalDate endDate, boolean skipFiltering) {
if (!skipFiltering) {
return prepareDatesFilterInRequestURL(url, startDate, endDate);
}
return url;
}
private String applyEndpointArguments(String requestURL, List<String> ids) {
String endpointArguments = formatEndpointArguments(requestURL, ids);
return requestURL.replace("${endpointArguments}", endpointArguments);
}
private String formatEndpointArguments(String requestURL, List<String> ids) {
if (requestURL.contains(":=")) {
return "'{" + ids.stream()
.map(String::trim)
.collect(Collectors.joining(", ")) + "}'";
} else {
return "(" + ids.stream()
.map(s -> "'" + s.trim() + "'")
.collect(Collectors.joining(", ")) + ")";
}
} }
private String prepareDatesFilterInRequestURL(String requestURL, LocalDate startDate, LocalDate endDate) { private String prepareDatesFilterInRequestURL(String requestURL, LocalDate startDate, LocalDate endDate) {
@ -449,57 +480,43 @@ public class RequestService {
return resultMap; return resultMap;
} }
private Map<String, Object> buildSqlQuery(BaseRequest request, RequestParameters parameters) { private Map<String, Object> buildSqlQuery(BaseRequest request, RequestParameters parameters, Map<String, Boolean> validationResults) {
logger.debug("Starting building SQL query for request: {}", request.getRequestURL()); logger.debug("Starting building SQL query for request: {}", request.getRequestURL());
long startExecTime = System.currentTimeMillis(); long startExecTime = System.currentTimeMillis();
Map<String, Object> resultMap = new HashMap<>(); Map<String, Object> resultMap = new HashMap<>();
String endpointArguments;
String requestURL = prepareDatesFilterInRequestURL(request.getRequestURL(), parameters.getStartDate(), parameters.getEndDate()); Boolean isEmptyDatesAllowed = validationResults.getOrDefault(ValidationService.IS_EMPTY_DATES_ALLOWED, false);
List<String> ids = parameters.getIds();
if (requestURL.contains(":=")) { String requestURL = applyDateFilter(request.getRequestURL(), parameters.getStartDate(), parameters.getEndDate(), isEmptyDatesAllowed);
endpointArguments = "'{" + ids.stream() if (isEmptyDatesAllowed) {
.map(String::trim) logger.info("Skipping date filtering as empty dates are allowed.");
.collect(Collectors.joining(", ")) + "}'";
} else {
endpointArguments = "(" + ids.stream()
.map(s -> "'" + s.trim() + "'")
.collect(Collectors.joining(", ")) + ")";
} }
List<String> ids = parameters.getIds();
if (request.getRequestArguments() != null && !request.getRequestArguments().isEmpty()) { if (request.getRequestArguments() != null && !request.getRequestArguments().isEmpty()) {
for (RequestArgument argument : request.getRequestArguments()) { for (RequestArgument argument : request.getRequestArguments()) {
if (argument.getRequestArgumentConnectionParams() != null) { if (argument.getRequestArgumentConnectionParams() != null) {
logger.debug("Opening connection for SQL RequestArgument: {}", argument.getRequestArgumentName()); logger.debug("Opening connection for SQL RequestArgument: {}", argument.getRequestArgumentName());
try (Connection connection = DatabaseConnection.getConnection( try (Connection connection = DatabaseConnection.getConnection(argument.getRequestArgumentConnectionParams())) {
argument.getRequestArgumentConnectionParams())) {
String query = argument.getRequestArgumentURL(); String query = argument.getRequestArgumentURL();
List<String> result = fetchFileListFromDatabaseSQL(connection, query); List<String> result = fetchFileListFromDatabaseSQL(connection, query);
resultMap.put("ids", result); resultMap.put("ids", result);
if (result != null && !result.isEmpty()) { if (result != null && !result.isEmpty()) {
String resultSet = "(" + result.stream() String resultSet = "(" + result.stream()
.map(s -> "'" + s.trim() + "'") .map(s -> "'" + s.trim() + "'")
.collect(Collectors.joining(", ")) + ")"; .collect(Collectors.joining(", ")) + ")";
requestURL = requestURL.replace("${" + argument.getRequestArgumentName() + "}", resultSet); requestURL = requestURL.replace("${" + argument.getRequestArgumentName() + "}", resultSet);
} }
} catch (SQLException e) {
}
catch (SQLException e) {
logger.error("Failed to execute query for RequestArgument", e); logger.error("Failed to execute query for RequestArgument", e);
} }
} }
} }
} }
resultMap.put("requestURL", requestURL String finalUrl = applyEndpointArguments(requestURL, ids);
.replace("${endpointArguments}", endpointArguments)); resultMap.put("requestURL", finalUrl);
long endExecTime = System.currentTimeMillis(); long endExecTime = System.currentTimeMillis();
logger.debug("SQL query built in {} ms", endExecTime - startExecTime); logger.debug("SQL query built in {} ms", endExecTime - startExecTime);
@ -526,7 +543,7 @@ public class RequestService {
return results; return results;
} }
private void processAqlRequests(AqlRequest request, List<String> ids) { private void processAqlRequests(AqlRequest request, List<String> ids, Map<String, Boolean> validationResults) {
ArangoDatabase arangoDb = ArangoDBConnection.getConnection(request.getAqlConnectionParams()); ArangoDatabase arangoDb = ArangoDBConnection.getConnection(request.getAqlConnectionParams());
// TODO: implement for multiple request arguments // TODO: implement for multiple request arguments
@ -548,7 +565,7 @@ public class RequestService {
logger.info("Stream transaction started with ID: {}", transactionId); logger.info("Stream transaction started with ID: {}", transactionId);
Map<String, Object> entities = executeSelectAqlRequest(arangoDb, requestArgument, ids, transactionId); Map<String, Object> entities = executeSelectAqlRequest(arangoDb, requestArgument, ids, transactionId, validationResults);
if (entities.isEmpty()) { if (entities.isEmpty()) {
logger.warn("No entities found for main AQL request."); logger.warn("No entities found for main AQL request.");
@ -591,15 +608,18 @@ public class RequestService {
private Map<String, Object> executeSelectAqlRequest(ArangoDatabase arangoDb, private Map<String, Object> executeSelectAqlRequest(ArangoDatabase arangoDb,
RequestArgument requestArgument, RequestArgument requestArgument,
List<String> ids, String transactionId) { List<String> ids, String transactionId, Map<String, Boolean> validationResults) {
Map<String, Object> entities = new HashMap<>(); Map<String, Object> entities = new HashMap<>();
String url = requestArgument.getRequestArgumentURL(); String url = requestArgument.getRequestArgumentURL();
RequestArgumentType type = requestArgument.getType(); RequestArgumentType type = requestArgument.getType();
if (type == RequestArgumentType.AQL) { if (type == RequestArgumentType.AQL) {
Boolean emptyIdsAllowed = validationResults.getOrDefault(ValidationService.IS_EMPTY_IDS_ALLOWED, false);
Map<String, Object> bindVars = new HashMap<>(); Map<String, Object> bindVars = new HashMap<>();
bindVars.put("ids", ids);
if (!emptyIdsAllowed) bindVars.put("ids", ids);
AqlQueryOptions aqlQueryOptions = new AqlQueryOptions().streamTransactionId(transactionId); AqlQueryOptions aqlQueryOptions = new AqlQueryOptions().streamTransactionId(transactionId);
@ -619,20 +639,6 @@ public class RequestService {
logger.error("Failed to execute AQL url", e); logger.error("Failed to execute AQL url", e);
} }
} }
// else if (type == RequestArgumentType.SQL) {
// if (requestArgument.getRequestArgumentConnectionParams() != null) {
// try (Connection connection = DatabaseConnection.getConnection(
// requestArgument.getRequestArgumentConnectionParams())) {
// String query = requestArgument.getRequestArgumentURL();
// List<String> result = fetchFileListFromDatabaseSQL(connection, query);
//
// entities.put(aqlCollectionWrite, result);
// }
// catch (SQLException e) {
// logger.error("Failed to execute query for RequestArgument", e);
// }
// }
// }
return entities; return entities;
} }

View file

@ -2,7 +2,11 @@ package org.micord.service;
import org.micord.config.DatabaseConnection; import org.micord.config.DatabaseConnection;
import org.micord.enums.ConfigType; import org.micord.enums.ConfigType;
import org.micord.exceptions.IllegalRequestParametersException;
import org.micord.exceptions.ValidationException; import org.micord.exceptions.ValidationException;
import org.micord.models.requests.BaseRequest;
import org.micord.models.requests.RequestParameters;
import org.micord.models.requests.downloads.BaseDownloadRequest;
import org.micord.models.validations.ValidationRule; import org.micord.models.validations.ValidationRule;
import org.micord.models.validations.ValidationRules; import org.micord.models.validations.ValidationRules;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
@ -13,18 +17,26 @@ import java.sql.Connection;
import java.sql.PreparedStatement; import java.sql.PreparedStatement;
import java.sql.ResultSet; import java.sql.ResultSet;
import java.sql.SQLException; import java.sql.SQLException;
import java.time.LocalDate;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.UUID; import java.util.UUID;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import java.util.stream.Stream;
@Service @Service
public class ValidationService { public class ValidationService {
public static final String IS_EMPTY_DATES_ALLOWED = "isEmptyDatesAllowed";
public static final String IS_EMPTY_IDS_ALLOWED = "isEmptyIdsAllowed";
@Autowired @Autowired
private ConfigService configService; private ConfigService configService;
private boolean isRequestValidatedByValidationConfig = false;
private boolean isMilitaryNoticeRequestValidatedByValidationConfig = false;
public Map<String, Map<String, Object>> validate(List<String> ids, ValidationRule rule) throws SQLException { public Map<String, Map<String, Object>> validate(List<String> ids, ValidationRule rule) throws SQLException {
String query = rule.getRequestURL(); String query = rule.getRequestURL();
@ -63,7 +75,95 @@ public class ValidationService {
} }
} }
public Map<String, String> validateAll(List<String> ids) throws ValidationException, FileNotFoundException, SQLException { public Map<String, Boolean> validateRequest(BaseRequest request, List<String> ids) throws ValidationException, SQLException, FileNotFoundException {
Map<String, Boolean> validationResults = validateIds(request, ids);
if (isRequestValidatedByValidationConfig) validateByValidationConfig(ids, BaseRequest.class);
return validationResults;
}
public Map<String, Boolean> validateMilitaryNoticeRequest(BaseRequest request, RequestParameters parameters) throws ValidationException, SQLException, FileNotFoundException {
List<String> ids = parameters.getIds();
validateIds(request, ids);
Map<String, Boolean> validation = validateDates(request, parameters);
if (isMilitaryNoticeRequestValidatedByValidationConfig) validateByValidationConfig(ids, RequestParameters.class);
return validation;
}
public Map<String, Boolean> validateDownloadRequest(BaseDownloadRequest request, RequestParameters downloadRequest, List<String> ids) throws ValidationException {
Map<String, Boolean> validateIds = validateIds(request, ids);
Map<String, Boolean> validateDates = validateDates(request, downloadRequest);
Map<String, Boolean> merged = new HashMap<>();
merged.putAll(validateIds);
merged.putAll(validateDates);
return merged;
}
private static Map<String, Boolean> validateIds(BaseRequest request, List<String> ids) {
if (request.getRequestValidationRules() == null) {
return Map.of();
}
Boolean emptyIdsAllowed = request.getRequestValidationRules().getIsEmptyIdsAllowed();
if (emptyIdsAllowed) {
return Map.of(ValidationService.IS_EMPTY_IDS_ALLOWED, emptyIdsAllowed);
}
if (ids == null || ids.isEmpty()) {
throw new IllegalRequestParametersException("пустые идентификаторы не допускаются");
}
Boolean isIdsFormatted = request.getRequestValidationRules().getIsIdsFormatted();
if (isIdsFormatted) {
String uuidRegex = "^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$";
boolean invalidIdFound = ids.stream().anyMatch(id -> !id.matches(uuidRegex));
if (invalidIdFound) {
throw new IllegalRequestParametersException("Некоторые идентификаторы не соответствуют формату GUID");
}
}
return Map.of(ValidationService.IS_EMPTY_IDS_ALLOWED, emptyIdsAllowed);
}
private static <T extends RequestParameters> Boolean validateEmptyDates(BaseRequest request, T parameters) {
if (request.getRequestValidationRules() == null) {
return Boolean.FALSE;
}
Boolean emptyDatesAllowed = request.getRequestValidationRules().getIsEmptyDatesAllowed();
LocalDate startDate = parameters.getStartDate();
LocalDate endDate = parameters.getEndDate();
if (Boolean.FALSE.equals(emptyDatesAllowed) && (startDate == null || endDate == null)) {
throw new IllegalRequestParametersException("пустые даты не допускаются");
}
return emptyDatesAllowed;
}
private <R extends BaseRequest, T extends RequestParameters> Map<String, Boolean> validateDates(R request, T parameters) {
Boolean emptyDatesAllowed = validateEmptyDates(request, parameters);
if (!emptyDatesAllowed && parameters.getStartDate() != null && parameters.getEndDate() != null) {
if (parameters.getStartDate().isAfter(parameters.getEndDate())) {
throw new IllegalArgumentException("Start date must be before end date");
}
}
return Map.of(ValidationService.IS_EMPTY_DATES_ALLOWED, emptyDatesAllowed);
}
public <T> Map<String, String> validateByValidationConfig(List<String> ids, T c) throws ValidationException, FileNotFoundException, SQLException {
if (c instanceof BaseRequest) {
isRequestValidatedByValidationConfig = true;
}
if (c instanceof RequestParameters) {
isMilitaryNoticeRequestValidatedByValidationConfig = true;
}
ValidationRules config = configService.getConfig(ConfigType.VALIDATE_BLOCK, ValidationRules.class); ValidationRules config = configService.getConfig(ConfigType.VALIDATE_BLOCK, ValidationRules.class);
if (config.getValidationRules() == null || config.getValidationRules().isEmpty()) { if (config.getValidationRules() == null || config.getValidationRules().isEmpty()) {
@ -85,7 +185,7 @@ public class ValidationService {
.toList(); .toList();
if (!invalidColumns.isEmpty()) { if (!invalidColumns.isEmpty()) {
String message = "Запись UUID = " + id + " не удовлетворяет критериям валидации."; String message = "Запись UUID = " + id + " имеет не отменённые временные меры или подписанные повестки.";
invalidRecords.put(id, message); invalidRecords.put(id, message);
} }
}); });
@ -97,5 +197,4 @@ public class ValidationService {
return invalidRecords; return invalidRecords;
} }
} }

View file

@ -1,5 +1,47 @@
<Requests> <Requests>
<DownloadRequest> <AQLDownloadRequest>
<RequestValidationRules
isEmptyIdsAllowed="false"
isEmptyDatesAllowed="false"
isIdsFormatted="true"
/>
<DownloadRequestType>Arango</DownloadRequestType>
<DownloadRequestEntitySelectorQuery>
<![CDATA[
WITH applications, subject, history, edges
FOR app IN applications
FILTER app.statement.recruitsData.mainInfo[0].id IN @ids
LET parentEdges = (
FOR vertex, edge, path
IN 1..1
OUTBOUND app._id edges
OPTIONS { uniqueVertices: "path" }
FILTER edge.field IN ["applicant", "history", "interdepreq"]
RETURN { edgesId: edge._key, parent: DOCUMENT(vertex._id) }
)
RETURN {
applicationId: app._key,
edgesId: (FOR e IN parentEdges RETURN e.edgesId),
subjectId: (FOR e IN parentEdges FILTER e.parent.schema == "Subject" RETURN e.parent._key),
historyId: (FOR e IN parentEdges FILTER e.parent.schema == "History" RETURN e.parent._key),
interdepreqId: (FOR e IN parentEdges FILTER e.parent.schema == "Interdepreq" RETURN e.parent._key)
}
]]>
</DownloadRequestEntitySelectorQuery>
<AqlRequestCollections>
<AqlRequestCollection type="read" dateAttribute="statement.recruitsData.mainInfo[0].systemCreateDate">applications</AqlRequestCollection>
<AqlRequestCollection type="read" dateAttribute="date">history</AqlRequestCollection>
</AqlRequestCollections>
<AqlConnectionParams>
<Host>localhost</Host>
<Port>8529</Port>
<Username>root</Username>
<Password>test</Password>
<Database>_system</Database>
</AqlConnectionParams>
</AQLDownloadRequest>
<SQLDownloadRequest>
<DownloadRequestType>Type_A</DownloadRequestType> <DownloadRequestType>Type_A</DownloadRequestType>
<RequestURL> <RequestURL>
Select system_id_ern from public.recruits where id in ${endpointArguments}; Select system_id_ern from public.recruits where id in ${endpointArguments};
@ -15,8 +57,8 @@
<JdbcDatabase>person_registry</JdbcDatabase> <JdbcDatabase>person_registry</JdbcDatabase>
<JdbcXaDataSourceBorrowConnectionTimeout>4000</JdbcXaDataSourceBorrowConnectionTimeout> <JdbcXaDataSourceBorrowConnectionTimeout>4000</JdbcXaDataSourceBorrowConnectionTimeout>
</SqlConnectionParams> </SqlConnectionParams>
</DownloadRequest> </SQLDownloadRequest>
<DownloadRequest> <SQLDownloadRequest>
<DownloadRequestType>Type_B</DownloadRequestType> <DownloadRequestType>Type_B</DownloadRequestType>
<RequestURL> <RequestURL>
Select system_id_ern from public.recruits where id in ${endpointArguments}; Select system_id_ern from public.recruits where id in ${endpointArguments};
@ -32,23 +74,6 @@
<JdbcDatabase>person_registry</JdbcDatabase> <JdbcDatabase>person_registry</JdbcDatabase>
<JdbcXaDataSourceBorrowConnectionTimeout>4000</JdbcXaDataSourceBorrowConnectionTimeout> <JdbcXaDataSourceBorrowConnectionTimeout>4000</JdbcXaDataSourceBorrowConnectionTimeout>
</SqlConnectionParams> </SqlConnectionParams>
</DownloadRequest> </SQLDownloadRequest>
<DownloadRequest>
<DownloadRequestType>Type_C</DownloadRequestType>
<RequestURL>
Select system_id_ern from public.recruits where id in ${endpointArguments};
</RequestURL>
<SqlConnectionParams>
<JdbcHost>10.10.31.118</JdbcHost>
<JdbcPort>5432</JdbcPort>
<JdbcUsername>ervu</JdbcUsername>
<JdbcPassword>ervu</JdbcPassword>
<JdbcDriverClassName>org.postgresql.Driver</JdbcDriverClassName>
<JdbcXaDataSourceClassName>org.postgresql.xa.PGXADataSource</JdbcXaDataSourceClassName>
<JdbcXaDataSourcePoolSize>50</JdbcXaDataSourcePoolSize>
<JdbcDatabase>person_registry</JdbcDatabase>
<JdbcXaDataSourceBorrowConnectionTimeout>4000</JdbcXaDataSourceBorrowConnectionTimeout>
</SqlConnectionParams>
</DownloadRequest>
</Requests> </Requests>