Zip multiple csvs
This commit is contained in:
Maksim Tereshin
parent
4b709e1a3d
commit
2a1d34ac10
9 changed files with 7158 additions and 2888 deletions
2
.gitignore
vendored
2
.gitignore
vendored
|
|
@ -65,3 +65,5 @@ npm-debug.log
|
|||
#Sublime project files
|
||||
*.sublime-project
|
||||
*.sublime-workspace
|
||||
|
||||
CLAUDE.md
|
||||
|
|
@ -20,6 +20,36 @@
|
|||
|
||||
<dependencyManagement>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-test</artifactId>
|
||||
<version>${spring-boot.version}</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.junit.jupiter</groupId>
|
||||
<artifactId>junit-jupiter</artifactId>
|
||||
<version>5.8.2</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.mockito</groupId>
|
||||
<artifactId>mockito-core</artifactId>
|
||||
<version>4.6.1</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.mockito</groupId>
|
||||
<artifactId>mockito-junit-jupiter</artifactId>
|
||||
<version>4.6.1</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.assertj</groupId>
|
||||
<artifactId>assertj-core</artifactId>
|
||||
<version>3.23.1</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</dependencyManagement>
|
||||
|
||||
|
|
@ -28,6 +58,31 @@
|
|||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-web</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-test</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.junit.jupiter</groupId>
|
||||
<artifactId>junit-jupiter</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.mockito</groupId>
|
||||
<artifactId>mockito-core</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.mockito</groupId>
|
||||
<artifactId>mockito-junit-jupiter</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.assertj</groupId>
|
||||
<artifactId>assertj-core</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework</groupId>
|
||||
<artifactId>spring-tx</artifactId>
|
||||
|
|
@ -88,7 +143,7 @@
|
|||
</repository>
|
||||
</repositories>
|
||||
<build>
|
||||
<finalName>${artifactId}</finalName>
|
||||
<finalName>${project.artifactId}</finalName>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
|
|
@ -113,6 +168,17 @@
|
|||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-surefire-plugin</artifactId>
|
||||
<version>2.22.2</version>
|
||||
<configuration>
|
||||
<includes>
|
||||
<include>**/*Test.java</include>
|
||||
<include>**/*Tests.java</include>
|
||||
</includes>
|
||||
</configuration>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
</project>
|
||||
|
|
|
|||
|
|
@ -99,17 +99,31 @@ public class ApiController {
|
|||
public ResponseEntity<Resource> downloadCSV(@RequestBody RequestParameters request) throws IOException, SQLException {
|
||||
logger.debug("Starting downloadCSV process for request: {}", request.getType());
|
||||
|
||||
File csvFile = apiService.download(ConfigType.DOWNLOAD_CSV, request);
|
||||
InputStreamResource resource = new InputStreamResource(new FileInputStream(csvFile));
|
||||
File downloadFile = apiService.download(ConfigType.DOWNLOAD_CSV, request);
|
||||
InputStreamResource resource = new InputStreamResource(new FileInputStream(downloadFile));
|
||||
|
||||
logger.debug("Finished downloadCSV process for request: {}. Sending to user...", request.getType());
|
||||
|
||||
return ResponseEntity.ok()
|
||||
.header(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=" + csvFile.getName())
|
||||
.contentType(MediaType.parseMediaType("text/csv; charset=UTF-8"))
|
||||
.header(HttpHeaders.CONTENT_ENCODING, "UTF-8")
|
||||
.contentLength(csvFile.length())
|
||||
.body(resource);
|
||||
// Determine content type based on file extension
|
||||
String fileName = downloadFile.getName();
|
||||
MediaType contentType;
|
||||
if (fileName.toLowerCase().endsWith(".zip")) {
|
||||
contentType = MediaType.APPLICATION_OCTET_STREAM;
|
||||
} else {
|
||||
contentType = MediaType.parseMediaType("text/csv; charset=UTF-8");
|
||||
}
|
||||
|
||||
ResponseEntity.BodyBuilder response = ResponseEntity.ok()
|
||||
.header(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=" + fileName)
|
||||
.contentType(contentType)
|
||||
.contentLength(downloadFile.length());
|
||||
|
||||
// Only add content encoding for CSV files
|
||||
if (!fileName.endsWith(".zip")) {
|
||||
response.header(HttpHeaders.CONTENT_ENCODING, "UTF-8");
|
||||
}
|
||||
|
||||
return response.body(resource);
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -25,11 +25,14 @@ import java.sql.SQLException;
|
|||
import java.time.LocalDate;
|
||||
import java.util.*;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.zip.ZipEntry;
|
||||
import java.util.zip.ZipOutputStream;
|
||||
|
||||
@Service
|
||||
public class DownloadService {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(DownloadService.class);
|
||||
private static final int MAX_ROWS_PER_CSV = 600000;
|
||||
|
||||
public File download(BaseDownloadRequest selectedRequest, List<String> ids, RequestParameters parameters, Map<String, Boolean> validationResults) throws SQLException {
|
||||
LocalDate startDate = parameters.getStartDate();
|
||||
|
|
@ -113,23 +116,16 @@ public class DownloadService {
|
|||
|
||||
List<String[]> results = executeSqlQuery(connection, requestURL);
|
||||
|
||||
File csvFile = File.createTempFile("download-", ".csv");
|
||||
|
||||
try (PrintWriter writer = new PrintWriter(
|
||||
new OutputStreamWriter(new FileOutputStream(csvFile), StandardCharsets.UTF_8))) {
|
||||
String lineSeparator = "\r\n";
|
||||
|
||||
for (String[] row : results) {
|
||||
writer.print(formatCsvRow(row));
|
||||
writer.print(lineSeparator);
|
||||
}
|
||||
} catch (IOException e) {
|
||||
logger.error("Failed to write to CSV file", e);
|
||||
// Check if we need to split into multiple files
|
||||
// Note: results includes header row, so we check size - 1 for actual data rows
|
||||
if (results.size() - 1 <= MAX_ROWS_PER_CSV) {
|
||||
// Single CSV file
|
||||
return writeSingleSqlCsvFile(results, "download-", ".csv");
|
||||
} else {
|
||||
// Multiple CSV files in ZIP
|
||||
return writeSqlResultsToZip(results);
|
||||
}
|
||||
|
||||
|
||||
return csvFile;
|
||||
|
||||
}
|
||||
catch (SQLException | IOException e) {
|
||||
logger.error("SQL execution failed for query: {}", query, e);
|
||||
|
|
@ -137,6 +133,55 @@ public class DownloadService {
|
|||
return null;
|
||||
}
|
||||
|
||||
private File writeSingleSqlCsvFile(List<String[]> results, String prefix, String suffix) throws IOException {
|
||||
File csvFile = File.createTempFile(prefix, suffix);
|
||||
|
||||
try (PrintWriter writer = new PrintWriter(
|
||||
new OutputStreamWriter(new FileOutputStream(csvFile), StandardCharsets.UTF_8))) {
|
||||
String lineSeparator = "\r\n";
|
||||
|
||||
for (String[] row : results) {
|
||||
writer.print(formatCsvRow(row));
|
||||
writer.print(lineSeparator);
|
||||
}
|
||||
}
|
||||
|
||||
return csvFile;
|
||||
}
|
||||
|
||||
private File writeSqlResultsToZip(List<String[]> results) throws IOException {
|
||||
List<File> csvFiles = new ArrayList<>();
|
||||
|
||||
// Extract header
|
||||
String[] headers = results.isEmpty() ? new String[0] : results.get(0);
|
||||
|
||||
int fileIndex = 1;
|
||||
int currentRowIndex = 1; // Start from 1 to skip header in original results
|
||||
|
||||
while (currentRowIndex < results.size()) {
|
||||
List<String[]> chunk = new ArrayList<>();
|
||||
// Always add headers as first row
|
||||
chunk.add(headers);
|
||||
|
||||
// Add data rows up to MAX_ROWS_PER_CSV (including the header)
|
||||
int chunkEndIndex = Math.min(currentRowIndex + MAX_ROWS_PER_CSV - 1, results.size());
|
||||
for (int i = currentRowIndex; i < chunkEndIndex; i++) {
|
||||
chunk.add(results.get(i));
|
||||
}
|
||||
|
||||
File csvFile = writeSingleSqlCsvFile(chunk, "download-part" + fileIndex + "-", ".csv");
|
||||
if (csvFile != null) {
|
||||
csvFiles.add(csvFile);
|
||||
fileIndex++;
|
||||
}
|
||||
|
||||
currentRowIndex = chunkEndIndex;
|
||||
}
|
||||
|
||||
// Create ZIP archive
|
||||
return createZipArchive(csvFiles, "download-");
|
||||
}
|
||||
|
||||
private List<Map<String, Object>> executeSelectAqlRequest(ArangoDatabase arangoDb,
|
||||
String downloadRequestEntitySelectorQuery,
|
||||
List<String> ids, LocalDate startDate, LocalDate endDate, Boolean emptyIdsAllowed, Boolean emptyDatesAllowed) {
|
||||
|
|
@ -272,33 +317,88 @@ public class DownloadService {
|
|||
}
|
||||
|
||||
private File writeResultsToCsv(List<Map<String, Object>> results) {
|
||||
File csvFile;
|
||||
try {
|
||||
csvFile = File.createTempFile("arango-download-", ".csv");
|
||||
// If results fit in a single file, create one CSV
|
||||
if (results.size() <= MAX_ROWS_PER_CSV) {
|
||||
return writeSingleCsvFile(results, "arango-download-", ".csv");
|
||||
}
|
||||
|
||||
try (BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(csvFile), StandardCharsets.UTF_8))) {
|
||||
// Otherwise, create multiple CSV files and zip them
|
||||
List<File> csvFiles = new ArrayList<>();
|
||||
int fileIndex = 1;
|
||||
|
||||
if (!results.isEmpty()) {
|
||||
List<String> headers = new ArrayList<>(results.get(0).keySet());
|
||||
writer.write(String.join(",", headers));
|
||||
writer.newLine();
|
||||
for (int i = 0; i < results.size(); i += MAX_ROWS_PER_CSV) {
|
||||
int endIndex = Math.min(i + MAX_ROWS_PER_CSV, results.size());
|
||||
List<Map<String, Object>> chunk = results.subList(i, endIndex);
|
||||
|
||||
for (Map<String, Object> row : results) {
|
||||
List<String> rowValues = headers.stream()
|
||||
.map(header -> formatCsvField(row.get(header)))
|
||||
.collect(Collectors.toList());
|
||||
writer.write(String.join(",", rowValues));
|
||||
writer.newLine();
|
||||
}
|
||||
File csvFile = writeSingleCsvFile(chunk, "arango-download-part" + fileIndex + "-", ".csv");
|
||||
if (csvFile != null) {
|
||||
csvFiles.add(csvFile);
|
||||
fileIndex++;
|
||||
}
|
||||
}
|
||||
|
||||
// Create ZIP archive
|
||||
return createZipArchive(csvFiles, "arango-download-");
|
||||
|
||||
} catch (IOException e) {
|
||||
logger.error("Failed to write results to CSV", e);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private File writeSingleCsvFile(List<Map<String, Object>> results, String prefix, String suffix) throws IOException {
|
||||
File csvFile = File.createTempFile(prefix, suffix);
|
||||
|
||||
try (BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(csvFile), StandardCharsets.UTF_8))) {
|
||||
if (!results.isEmpty()) {
|
||||
List<String> headers = new ArrayList<>(results.get(0).keySet());
|
||||
writer.write(String.join(",", headers));
|
||||
writer.newLine();
|
||||
|
||||
for (Map<String, Object> row : results) {
|
||||
List<String> rowValues = headers.stream()
|
||||
.map(header -> formatCsvField(row.get(header)))
|
||||
.collect(Collectors.toList());
|
||||
writer.write(String.join(",", rowValues));
|
||||
writer.newLine();
|
||||
}
|
||||
}
|
||||
}
|
||||
return csvFile;
|
||||
}
|
||||
|
||||
private File createZipArchive(List<File> files, String prefix) throws IOException {
|
||||
File zipFile = File.createTempFile(prefix, ".zip");
|
||||
|
||||
try (ZipOutputStream zos = new ZipOutputStream(new FileOutputStream(zipFile))) {
|
||||
for (File file : files) {
|
||||
addFileToZip(file, zos);
|
||||
// Delete temporary CSV files after adding to ZIP
|
||||
if (!file.delete()) {
|
||||
logger.warn("Failed to delete temporary file: {}", file.getName());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return zipFile;
|
||||
}
|
||||
|
||||
private void addFileToZip(File file, ZipOutputStream zos) throws IOException {
|
||||
try (FileInputStream fis = new FileInputStream(file)) {
|
||||
ZipEntry zipEntry = new ZipEntry(file.getName());
|
||||
zos.putNextEntry(zipEntry);
|
||||
|
||||
byte[] buffer = new byte[8192];
|
||||
int length;
|
||||
while ((length = fis.read(buffer)) >= 0) {
|
||||
zos.write(buffer, 0, length);
|
||||
}
|
||||
|
||||
zos.closeEntry();
|
||||
}
|
||||
}
|
||||
|
||||
private String formatCsvField(Object value) {
|
||||
if (value == null) {
|
||||
return "\"\"";
|
||||
|
|
|
|||
|
|
@ -0,0 +1,283 @@
|
|||
package org.micord.controller;
|
||||
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
import org.micord.enums.ConfigType;
|
||||
import org.micord.models.requests.RequestParameters;
|
||||
import org.micord.service.ApiService;
|
||||
import org.mockito.InjectMocks;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import org.springframework.core.io.Resource;
|
||||
import org.springframework.http.HttpHeaders;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.time.LocalDate;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import java.util.zip.ZipEntry;
|
||||
import java.util.zip.ZipOutputStream;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.*;
|
||||
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class ApiControllerTest {
|
||||
|
||||
@Mock
|
||||
private ApiService apiService;
|
||||
|
||||
@InjectMocks
|
||||
private ApiController apiController;
|
||||
|
||||
@TempDir
|
||||
Path tempDir;
|
||||
|
||||
private RequestParameters testRequest;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
testRequest = new RequestParameters();
|
||||
testRequest.setType("test-download-type");
|
||||
testRequest.setIds(Arrays.asList("id1", "id2", "id3"));
|
||||
testRequest.setStartDate(LocalDate.of(2024, 1, 1));
|
||||
testRequest.setEndDate(LocalDate.of(2024, 12, 31));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testDownloadCSV_SingleCsvFile() throws Exception {
|
||||
// Given: Service returns a single CSV file
|
||||
File csvFile = createTempCsvFile("test-download.csv");
|
||||
when(apiService.download(eq(ConfigType.DOWNLOAD_CSV), any(RequestParameters.class)))
|
||||
.thenReturn(csvFile);
|
||||
|
||||
// When: Calling downloadCSV endpoint
|
||||
ResponseEntity<Resource> response = apiController.downloadCSV(testRequest);
|
||||
|
||||
// Then: Response should have CSV content type and headers
|
||||
assertThat(response.getStatusCodeValue()).isEqualTo(200);
|
||||
assertThat(response.getHeaders().getContentType())
|
||||
.isEqualTo(MediaType.parseMediaType("text/csv; charset=UTF-8"));
|
||||
assertThat(response.getHeaders().getFirst(HttpHeaders.CONTENT_DISPOSITION))
|
||||
.startsWith("attachment; filename=")
|
||||
.endsWith(".csv");
|
||||
assertThat(response.getHeaders().getFirst(HttpHeaders.CONTENT_ENCODING))
|
||||
.isEqualTo("UTF-8");
|
||||
assertThat(response.getHeaders().getContentLength()).isEqualTo(csvFile.length());
|
||||
|
||||
verify(apiService).download(eq(ConfigType.DOWNLOAD_CSV), any(RequestParameters.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testDownloadCSV_ZipFile() throws Exception {
|
||||
// Given: Service returns a ZIP file (for large datasets)
|
||||
File zipFile = createTempZipFile("test-download.zip");
|
||||
when(apiService.download(eq(ConfigType.DOWNLOAD_CSV), any(RequestParameters.class)))
|
||||
.thenReturn(zipFile);
|
||||
|
||||
// When: Calling downloadCSV endpoint
|
||||
ResponseEntity<Resource> response = apiController.downloadCSV(testRequest);
|
||||
|
||||
// Then: Response should have ZIP content type and headers
|
||||
assertThat(response.getStatusCodeValue()).isEqualTo(200);
|
||||
assertThat(response.getHeaders().getContentType())
|
||||
.isEqualTo(MediaType.APPLICATION_OCTET_STREAM);
|
||||
assertThat(response.getHeaders().getFirst(HttpHeaders.CONTENT_DISPOSITION))
|
||||
.startsWith("attachment; filename=")
|
||||
.endsWith(".zip");
|
||||
// Content-Encoding should NOT be set for ZIP files
|
||||
assertThat(response.getHeaders().getFirst(HttpHeaders.CONTENT_ENCODING))
|
||||
.isNull();
|
||||
assertThat(response.getHeaders().getContentLength()).isEqualTo(zipFile.length());
|
||||
|
||||
verify(apiService).download(eq(ConfigType.DOWNLOAD_CSV), any(RequestParameters.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testDownloadCSV_HandlesIOException() throws Exception {
|
||||
// Given: Service throws IOException
|
||||
when(apiService.download(eq(ConfigType.DOWNLOAD_CSV), any(RequestParameters.class)))
|
||||
.thenThrow(new IOException("Test IO error"));
|
||||
|
||||
// When/Then: Should propagate the exception
|
||||
assertThrows(IOException.class, () -> {
|
||||
apiController.downloadCSV(testRequest);
|
||||
});
|
||||
}
|
||||
|
||||
@Test
|
||||
void testDownloadCSV_EmptyRequest() throws Exception {
|
||||
// Given: Empty request parameters
|
||||
RequestParameters emptyRequest = new RequestParameters();
|
||||
emptyRequest.setType("empty-type");
|
||||
|
||||
File csvFile = createTempCsvFile("empty.csv");
|
||||
when(apiService.download(eq(ConfigType.DOWNLOAD_CSV), any(RequestParameters.class)))
|
||||
.thenReturn(csvFile);
|
||||
|
||||
// When: Calling with empty request
|
||||
ResponseEntity<Resource> response = apiController.downloadCSV(emptyRequest);
|
||||
|
||||
// Then: Should still process successfully
|
||||
assertThat(response.getStatusCodeValue()).isEqualTo(200);
|
||||
assertThat(response.getBody()).isNotNull();
|
||||
}
|
||||
|
||||
@Test
|
||||
void testListDownloadTypes() throws Exception {
|
||||
// Given: Service returns list of download types
|
||||
List<String> downloadTypes = Arrays.asList(
|
||||
"TYPE_1", "TYPE_2", "TYPE_3", "TYPE_4"
|
||||
);
|
||||
when(apiService.getDownloadTypes(ConfigType.DOWNLOAD_CSV))
|
||||
.thenReturn(downloadTypes);
|
||||
|
||||
// When: Calling listDownloadTypes endpoint
|
||||
ResponseEntity<List<String>> response = apiController.listDownloadTypes();
|
||||
|
||||
// Then: Should return the list
|
||||
assertThat(response.getStatusCodeValue()).isEqualTo(200);
|
||||
assertThat(response.getBody()).isEqualTo(downloadTypes);
|
||||
assertThat(response.getBody()).hasSize(4);
|
||||
|
||||
verify(apiService).getDownloadTypes(ConfigType.DOWNLOAD_CSV);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testRemoveMilitaryDraftNotices() throws Exception {
|
||||
// Given: Request with IDs
|
||||
RequestParameters request = new RequestParameters();
|
||||
request.setIds(Arrays.asList("notice1", "notice2"));
|
||||
|
||||
doNothing().when(apiService).process(eq(ConfigType.REMOVE_MILITARY_DRAFT_NOTICES), any(RequestParameters.class));
|
||||
|
||||
// When: Calling removeMilitaryDraftNotices
|
||||
ResponseEntity<?> response = apiController.removeMilitaryDraftNotices(request);
|
||||
|
||||
// Then: Should return success message
|
||||
assertThat(response.getStatusCodeValue()).isEqualTo(200);
|
||||
assertThat(response.getBody()).isEqualTo("Операция \"Удаление повесток\" завершена успешно.");
|
||||
|
||||
verify(apiService).process(eq(ConfigType.REMOVE_MILITARY_DRAFT_NOTICES), any(RequestParameters.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testDeleteFiles() throws Exception {
|
||||
// Given: List of file IDs
|
||||
List<String> fileIds = Arrays.asList("file1", "file2", "file3");
|
||||
|
||||
doNothing().when(apiService).process(eq(ConfigType.DELETE_FILES), eq(fileIds));
|
||||
|
||||
// When: Calling deleteFiles
|
||||
ResponseEntity<?> response = apiController.deleteFiles(fileIds);
|
||||
|
||||
// Then: Should return success message
|
||||
assertThat(response.getStatusCodeValue()).isEqualTo(200);
|
||||
assertThat(response.getBody()).isEqualTo("Операция \"Удаление файлов\" завершена успешно.");
|
||||
|
||||
verify(apiService).process(ConfigType.DELETE_FILES, fileIds);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testBlock() throws Exception {
|
||||
// Given: List of IDs to block
|
||||
List<String> ids = Arrays.asList("user1", "user2");
|
||||
|
||||
doNothing().when(apiService).process(eq(ConfigType.BLOCK), eq(ids));
|
||||
|
||||
// When: Calling block
|
||||
ResponseEntity<?> response = apiController.block(ids);
|
||||
|
||||
// Then: Should return success message
|
||||
assertThat(response.getStatusCodeValue()).isEqualTo(200);
|
||||
assertThat(response.getBody()).isEqualTo("Операция \"Блокировка\" завершена успешно.");
|
||||
|
||||
verify(apiService).process(ConfigType.BLOCK, ids);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testUnblock() throws Exception {
|
||||
// Given: List of IDs to unblock
|
||||
List<String> ids = Arrays.asList("user1", "user2");
|
||||
|
||||
doNothing().when(apiService).process(eq(ConfigType.UNBLOCK), eq(ids));
|
||||
|
||||
// When: Calling unblock
|
||||
ResponseEntity<?> response = apiController.unblock(ids);
|
||||
|
||||
// Then: Should return success message
|
||||
assertThat(response.getStatusCodeValue()).isEqualTo(200);
|
||||
assertThat(response.getBody()).isEqualTo("Операция \"Разблокировка\" завершена успешно.");
|
||||
|
||||
verify(apiService).process(ConfigType.UNBLOCK, ids);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testContentTypeDetection() throws Exception {
|
||||
// Test various file extensions
|
||||
testFileContentType("data.csv", MediaType.parseMediaType("text/csv; charset=UTF-8"));
|
||||
testFileContentType("archive.zip", MediaType.APPLICATION_OCTET_STREAM);
|
||||
testFileContentType("report.CSV", MediaType.parseMediaType("text/csv; charset=UTF-8"));
|
||||
testFileContentType("bundle.ZIP", MediaType.APPLICATION_OCTET_STREAM);
|
||||
}
|
||||
|
||||
private void testFileContentType(String filename, MediaType expectedType) throws Exception {
|
||||
// Create file with specific extension
|
||||
File file = tempDir.resolve(filename).toFile();
|
||||
file.createNewFile();
|
||||
|
||||
when(apiService.download(eq(ConfigType.DOWNLOAD_CSV), any(RequestParameters.class)))
|
||||
.thenReturn(file);
|
||||
|
||||
ResponseEntity<Resource> response = apiController.downloadCSV(testRequest);
|
||||
|
||||
assertThat(response.getHeaders().getContentType()).isEqualTo(expectedType);
|
||||
}
|
||||
|
||||
// Helper methods
|
||||
|
||||
private File createTempCsvFile(String filename) throws IOException {
|
||||
File csvFile = Files.createTempFile(tempDir, filename.replace(".csv", ""), ".csv").toFile();
|
||||
|
||||
try (FileOutputStream fos = new FileOutputStream(csvFile)) {
|
||||
String csvContent = "header1,header2,header3\n" +
|
||||
"value1,value2,value3\n" +
|
||||
"value4,value5,value6\n";
|
||||
fos.write(csvContent.getBytes(StandardCharsets.UTF_8));
|
||||
}
|
||||
|
||||
return csvFile;
|
||||
}
|
||||
|
||||
private File createTempZipFile(String filename) throws IOException {
|
||||
File zipFile = Files.createTempFile(tempDir, filename.replace(".zip", ""), ".zip").toFile();
|
||||
|
||||
try (ZipOutputStream zos = new ZipOutputStream(new FileOutputStream(zipFile))) {
|
||||
// Add first CSV
|
||||
ZipEntry entry1 = new ZipEntry("part1.csv");
|
||||
zos.putNextEntry(entry1);
|
||||
zos.write("header1,header2\ndata1,data2\n".getBytes(StandardCharsets.UTF_8));
|
||||
zos.closeEntry();
|
||||
|
||||
// Add second CSV
|
||||
ZipEntry entry2 = new ZipEntry("part2.csv");
|
||||
zos.putNextEntry(entry2);
|
||||
zos.write("header1,header2\ndata3,data4\n".getBytes(StandardCharsets.UTF_8));
|
||||
zos.closeEntry();
|
||||
}
|
||||
|
||||
return zipFile;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,290 @@
|
|||
package org.micord.service;
|
||||
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.InjectMocks;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
|
||||
import java.io.*;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.*;
|
||||
import java.util.zip.ZipEntry;
|
||||
import java.util.zip.ZipFile;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class DownloadServiceTest {
|
||||
|
||||
@InjectMocks
|
||||
private DownloadService downloadService;
|
||||
|
||||
@TempDir
|
||||
Path tempDir;
|
||||
|
||||
private static final int MAX_ROWS_PER_CSV = 600000;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
downloadService = new DownloadService();
|
||||
}
|
||||
|
||||
@Test
|
||||
void testSingleCsvFileWhenRowsUnderLimit() throws Exception {
|
||||
// Given: Dataset with less than 600k rows
|
||||
List<Map<String, Object>> testData = generateTestData(100000);
|
||||
|
||||
// When: Writing results to CSV
|
||||
File result = invokeWriteResultsToCsv(testData);
|
||||
|
||||
// Then: Should create single CSV file, not ZIP
|
||||
assertThat(result).isNotNull();
|
||||
assertThat(result.getName()).endsWith(".csv");
|
||||
assertThat(result.getName()).doesNotEndWith(".zip");
|
||||
|
||||
// Verify CSV content
|
||||
List<String> lines = Files.readAllLines(result.toPath());
|
||||
// Header + 100k data rows
|
||||
assertThat(lines).hasSize(100001);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testMultipleCsvFilesWhenRowsOverLimit() throws Exception {
|
||||
// Given: Dataset with more than 600k rows (1.5 million)
|
||||
List<Map<String, Object>> testData = generateTestData(1500000);
|
||||
|
||||
// When: Writing results to CSV
|
||||
File result = invokeWriteResultsToCsv(testData);
|
||||
|
||||
// Then: Should create ZIP file
|
||||
assertThat(result).isNotNull();
|
||||
assertThat(result.getName()).endsWith(".zip");
|
||||
|
||||
// Verify ZIP contains 3 CSV files (600k + 600k + 300k)
|
||||
try (ZipFile zipFile = new ZipFile(result)) {
|
||||
assertThat(Collections.list(zipFile.entries())).hasSize(3);
|
||||
|
||||
zipFile.entries().asIterator().forEachRemaining(entry -> {
|
||||
assertThat(entry.getName()).endsWith(".csv");
|
||||
assertThat(entry.getName()).contains("part");
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void testExactlyMaxRowsPerCsvFile() throws Exception {
|
||||
// Given: Exactly 600k rows
|
||||
List<Map<String, Object>> testData = generateTestData(MAX_ROWS_PER_CSV);
|
||||
|
||||
// When: Writing results to CSV
|
||||
File result = invokeWriteResultsToCsv(testData);
|
||||
|
||||
// Then: Should create single CSV file
|
||||
assertThat(result).isNotNull();
|
||||
assertThat(result.getName()).endsWith(".csv");
|
||||
|
||||
List<String> lines = Files.readAllLines(result.toPath());
|
||||
assertThat(lines).hasSize(MAX_ROWS_PER_CSV + 1); // +1 for header
|
||||
}
|
||||
|
||||
@Test
|
||||
void testChunkingPreservesHeaders() throws Exception {
|
||||
// Given: Dataset that requires chunking
|
||||
List<Map<String, Object>> testData = generateTestData(1200000);
|
||||
|
||||
// When: Writing results to CSV
|
||||
File result = invokeWriteResultsToCsv(testData);
|
||||
|
||||
// Then: Each CSV in ZIP should have headers
|
||||
try (ZipFile zipFile = new ZipFile(result)) {
|
||||
zipFile.entries().asIterator().forEachRemaining(entry -> {
|
||||
try (BufferedReader reader = new BufferedReader(
|
||||
new InputStreamReader(zipFile.getInputStream(entry)))) {
|
||||
String firstLine = reader.readLine();
|
||||
// Verify header exists
|
||||
assertThat(firstLine).contains("column1,column2,column3");
|
||||
} catch (IOException e) {
|
||||
fail("Failed to read ZIP entry: " + e.getMessage());
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void testEmptyResultsHandling() throws Exception {
|
||||
// Given: Empty dataset
|
||||
List<Map<String, Object>> testData = new ArrayList<>();
|
||||
|
||||
// When: Writing results to CSV
|
||||
File result = invokeWriteResultsToCsv(testData);
|
||||
|
||||
// Then: Should create empty CSV file
|
||||
assertThat(result).isNotNull();
|
||||
assertThat(result.getName()).endsWith(".csv");
|
||||
|
||||
List<String> lines = Files.readAllLines(result.toPath());
|
||||
assertThat(lines).isEmpty();
|
||||
}
|
||||
|
||||
@Test
|
||||
void testLargeDatasetChunking() throws Exception {
|
||||
// Given: 2 million rows (should create 4 files)
|
||||
List<Map<String, Object>> testData = generateTestData(2000000);
|
||||
|
||||
// When: Writing results to CSV
|
||||
File result = invokeWriteResultsToCsv(testData);
|
||||
|
||||
// Then: Should create ZIP with 4 CSV files
|
||||
assertThat(result.getName()).endsWith(".zip");
|
||||
|
||||
try (ZipFile zipFile = new ZipFile(result)) {
|
||||
List<? extends ZipEntry> entries = Collections.list(zipFile.entries());
|
||||
assertThat(entries).hasSize(4);
|
||||
|
||||
// Verify file names are sequential
|
||||
assertThat(entries.get(0).getName()).contains("part1");
|
||||
assertThat(entries.get(1).getName()).contains("part2");
|
||||
assertThat(entries.get(2).getName()).contains("part3");
|
||||
assertThat(entries.get(3).getName()).contains("part4");
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void testCsvFieldFormatting() throws Exception {
|
||||
// Given: Data with special characters that need escaping
|
||||
List<Map<String, Object>> testData = new ArrayList<>();
|
||||
// Use LinkedHashMap to preserve field order
|
||||
Map<String, Object> row = new LinkedHashMap<>();
|
||||
row.put("normal", "value");
|
||||
row.put("withQuote", "value\"with\"quotes");
|
||||
row.put("withComma", "value,with,commas");
|
||||
row.put("withNewline", "value\nwith\nnewlines");
|
||||
row.put("nullValue", null);
|
||||
testData.add(row);
|
||||
|
||||
// When: Writing results to CSV
|
||||
File result = invokeWriteResultsToCsv(testData);
|
||||
|
||||
// Then: Fields should be properly escaped
|
||||
String content = Files.readString(result.toPath());
|
||||
// Verify proper escaping
|
||||
assertThat(content).contains("\"value\"\"with\"\"quotes\"");
|
||||
assertThat(content).contains("\"value,with,commas\"");
|
||||
assertThat(content).contains("\"\""); // null value
|
||||
}
|
||||
|
||||
@Test
|
||||
void testSqlResultsToSingleFile() throws Exception {
|
||||
// Given: SQL results under the limit
|
||||
List<String[]> sqlResults = generateSqlTestData(50000);
|
||||
|
||||
// When: Writing SQL results to CSV
|
||||
File result = invokeWriteSingleSqlCsvFile(sqlResults, "test-", ".csv");
|
||||
|
||||
// Then: Should create single CSV file
|
||||
assertThat(result).isNotNull();
|
||||
assertThat(result.getName()).startsWith("test-");
|
||||
assertThat(result.getName()).endsWith(".csv");
|
||||
|
||||
List<String> lines = Files.readAllLines(result.toPath());
|
||||
assertThat(lines).hasSize(50000);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testSqlResultsChunking() throws Exception {
|
||||
// Given: SQL results that will create exactly 2 chunks
|
||||
// generateSqlTestData(1199999) creates 1 header + 1,199,998 data rows = 1,199,999 total
|
||||
// With MAX_ROWS_PER_CSV = 600,000:
|
||||
// File 1: header + 599,999 data rows = 600,000 total
|
||||
// File 2: header + 599,999 data rows = 600,000 total
|
||||
List<String[]> sqlResults = generateSqlTestData(1199999);
|
||||
|
||||
// When: Writing SQL results with chunking
|
||||
File result = invokeWriteSqlResultsToZip(sqlResults);
|
||||
|
||||
// Then: Should create ZIP with 2 CSV files
|
||||
assertThat(result).isNotNull();
|
||||
assertThat(result.getName()).endsWith(".zip");
|
||||
|
||||
try (ZipFile zipFile = new ZipFile(result)) {
|
||||
assertThat(Collections.list(zipFile.entries())).hasSize(2);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void testZipFileContentsIntegrity() throws Exception {
|
||||
// Given: Large dataset
|
||||
List<Map<String, Object>> testData = generateTestData(1200000);
|
||||
|
||||
// When: Creating ZIP file
|
||||
File result = invokeWriteResultsToCsv(testData);
|
||||
|
||||
// Then: Verify total row count across all files in ZIP
|
||||
int totalRows = 0;
|
||||
try (ZipFile zipFile = new ZipFile(result)) {
|
||||
for (ZipEntry entry : Collections.list(zipFile.entries())) {
|
||||
try (BufferedReader reader = new BufferedReader(
|
||||
new InputStreamReader(zipFile.getInputStream(entry)))) {
|
||||
// Count lines, excluding header
|
||||
long lines = reader.lines().count() - 1;
|
||||
totalRows += lines;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
assertThat(totalRows).isEqualTo(1200000);
|
||||
}
|
||||
|
||||
// Helper methods
|
||||
|
||||
private List<Map<String, Object>> generateTestData(int rows) {
|
||||
List<Map<String, Object>> data = new ArrayList<>();
|
||||
for (int i = 0; i < rows; i++) {
|
||||
// Use LinkedHashMap to preserve order
|
||||
Map<String, Object> row = new LinkedHashMap<>();
|
||||
row.put("column1", "value" + i);
|
||||
row.put("column2", i);
|
||||
row.put("column3", "test" + i);
|
||||
data.add(row);
|
||||
}
|
||||
return data;
|
||||
}
|
||||
|
||||
private List<String[]> generateSqlTestData(int rows) {
|
||||
List<String[]> data = new ArrayList<>();
|
||||
// Add header
|
||||
data.add(new String[]{"column1", "column2", "column3"});
|
||||
// Add data rows
|
||||
for (int i = 0; i < rows - 1; i++) {
|
||||
data.add(new String[]{"value" + i, String.valueOf(i), "test" + i});
|
||||
}
|
||||
return data;
|
||||
}
|
||||
|
||||
// Reflection-based invocation of private methods for testing
|
||||
private File invokeWriteResultsToCsv(List<Map<String, Object>> results) throws Exception {
|
||||
java.lang.reflect.Method method = DownloadService.class.getDeclaredMethod(
|
||||
"writeResultsToCsv", List.class);
|
||||
method.setAccessible(true);
|
||||
return (File) method.invoke(downloadService, results);
|
||||
}
|
||||
|
||||
private File invokeWriteSingleSqlCsvFile(List<String[]> results, String prefix, String suffix) throws Exception {
|
||||
java.lang.reflect.Method method = DownloadService.class.getDeclaredMethod(
|
||||
"writeSingleSqlCsvFile", List.class, String.class, String.class);
|
||||
method.setAccessible(true);
|
||||
return (File) method.invoke(downloadService, results, prefix, suffix);
|
||||
}
|
||||
|
||||
private File invokeWriteSqlResultsToZip(List<String[]> results) throws Exception {
|
||||
java.lang.reflect.Method method = DownloadService.class.getDeclaredMethod(
|
||||
"writeSqlResultsToZip", List.class);
|
||||
method.setAccessible(true);
|
||||
return (File) method.invoke(downloadService, results);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,200 @@
|
|||
package org.micord.utils;
|
||||
|
||||
import java.io.*;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.zip.ZipEntry;
|
||||
import java.util.zip.ZipFile;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
/**
|
||||
* Utility class for file operations in tests.
|
||||
*/
|
||||
public class FileTestUtils {
|
||||
|
||||
/**
|
||||
* Count the number of lines in a CSV file
|
||||
*/
|
||||
public static long countCsvLines(File csvFile) throws IOException {
|
||||
try (BufferedReader reader = new BufferedReader(
|
||||
new InputStreamReader(new FileInputStream(csvFile), StandardCharsets.UTF_8))) {
|
||||
return reader.lines().count();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Read all lines from a CSV file
|
||||
*/
|
||||
public static List<String> readCsvLines(File csvFile) throws IOException {
|
||||
return Files.readAllLines(csvFile.toPath(), StandardCharsets.UTF_8);
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse CSV line (simple implementation for testing)
|
||||
*/
|
||||
public static String[] parseCsvLine(String line) {
|
||||
List<String> result = new ArrayList<>();
|
||||
StringBuilder current = new StringBuilder();
|
||||
boolean inQuotes = false;
|
||||
|
||||
for (int i = 0; i < line.length(); i++) {
|
||||
char c = line.charAt(i);
|
||||
|
||||
if (c == '"') {
|
||||
if (i + 1 < line.length() && line.charAt(i + 1) == '"') {
|
||||
current.append('"');
|
||||
i++; // Skip next quote
|
||||
} else {
|
||||
inQuotes = !inQuotes;
|
||||
}
|
||||
} else if (c == ',' && !inQuotes) {
|
||||
result.add(current.toString());
|
||||
current = new StringBuilder();
|
||||
} else {
|
||||
current.append(c);
|
||||
}
|
||||
}
|
||||
|
||||
result.add(current.toString());
|
||||
return result.toArray(new String[0]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Verify CSV file structure
|
||||
*/
|
||||
public static void verifyCsvStructure(File csvFile, int expectedColumns) throws IOException {
|
||||
List<String> lines = readCsvLines(csvFile);
|
||||
assertThat(lines).isNotEmpty();
|
||||
|
||||
// Check header
|
||||
String[] header = parseCsvLine(lines.get(0));
|
||||
assertThat(header).hasSize(expectedColumns);
|
||||
|
||||
// Check data rows have same number of columns
|
||||
for (int i = 1; i < Math.min(10, lines.size()); i++) {
|
||||
String[] row = parseCsvLine(lines.get(i));
|
||||
assertThat(row).hasSize(expectedColumns);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Count total rows across all CSV files in a ZIP
|
||||
*/
|
||||
public static long countTotalRowsInZip(File zipFile) throws IOException {
|
||||
long totalRows = 0;
|
||||
|
||||
try (ZipFile zip = new ZipFile(zipFile)) {
|
||||
for (ZipEntry entry : java.util.Collections.list(zip.entries())) {
|
||||
if (entry.getName().endsWith(".csv")) {
|
||||
try (BufferedReader reader = new BufferedReader(
|
||||
new InputStreamReader(zip.getInputStream(entry), StandardCharsets.UTF_8))) {
|
||||
// Count lines excluding header
|
||||
long lines = reader.lines().count() - 1;
|
||||
totalRows += lines;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return totalRows;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract all CSV files from a ZIP to a temporary directory
|
||||
*/
|
||||
public static List<File> extractCsvFromZip(File zipFile, Path tempDir) throws IOException {
|
||||
List<File> extractedFiles = new ArrayList<>();
|
||||
|
||||
try (ZipFile zip = new ZipFile(zipFile)) {
|
||||
for (ZipEntry entry : java.util.Collections.list(zip.entries())) {
|
||||
if (entry.getName().endsWith(".csv")) {
|
||||
Path outputPath = tempDir.resolve(entry.getName());
|
||||
|
||||
try (InputStream is = zip.getInputStream(entry);
|
||||
OutputStream os = Files.newOutputStream(outputPath)) {
|
||||
byte[] buffer = new byte[8192];
|
||||
int len;
|
||||
while ((len = is.read(buffer)) > 0) {
|
||||
os.write(buffer, 0, len);
|
||||
}
|
||||
}
|
||||
|
||||
extractedFiles.add(outputPath.toFile());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return extractedFiles;
|
||||
}
|
||||
|
||||
/**
|
||||
* Verify that each CSV file in ZIP has headers
|
||||
*/
|
||||
public static void verifyZipCsvHeaders(File zipFile, String[] expectedHeaders) throws IOException {
|
||||
try (ZipFile zip = new ZipFile(zipFile)) {
|
||||
for (ZipEntry entry : java.util.Collections.list(zip.entries())) {
|
||||
if (entry.getName().endsWith(".csv")) {
|
||||
try (BufferedReader reader = new BufferedReader(
|
||||
new InputStreamReader(zip.getInputStream(entry), StandardCharsets.UTF_8))) {
|
||||
String firstLine = reader.readLine();
|
||||
assertThat(firstLine).isNotNull();
|
||||
|
||||
String[] headers = parseCsvLine(firstLine);
|
||||
assertThat(headers).containsExactly(expectedHeaders);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a temporary CSV file with test data
|
||||
*/
|
||||
public static File createTempCsvFile(List<String[]> data, Path tempDir) throws IOException {
|
||||
File csvFile = Files.createTempFile(tempDir, "test-", ".csv").toFile();
|
||||
|
||||
try (PrintWriter writer = new PrintWriter(
|
||||
new OutputStreamWriter(new FileOutputStream(csvFile), StandardCharsets.UTF_8))) {
|
||||
for (String[] row : data) {
|
||||
writer.println(formatCsvRow(row));
|
||||
}
|
||||
}
|
||||
|
||||
return csvFile;
|
||||
}
|
||||
|
||||
/**
|
||||
* Format a row for CSV output
|
||||
*/
|
||||
private static String formatCsvRow(String[] row) {
|
||||
StringBuilder result = new StringBuilder();
|
||||
for (int i = 0; i < row.length; i++) {
|
||||
if (i > 0) {
|
||||
result.append(",");
|
||||
}
|
||||
result.append("\"");
|
||||
result.append(row[i] != null ? row[i].replace("\"", "\"\"") : "");
|
||||
result.append("\"");
|
||||
}
|
||||
return result.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* Get file size in MB
|
||||
*/
|
||||
public static double getFileSizeInMB(File file) {
|
||||
return file.length() / (1024.0 * 1024.0);
|
||||
}
|
||||
|
||||
/**
|
||||
* Verify file is within size limits
|
||||
*/
|
||||
public static void verifyFileSize(File file, double maxSizeMB) {
|
||||
double sizeMB = getFileSizeInMB(file);
|
||||
assertThat(sizeMB).isLessThanOrEqualTo(maxSizeMB);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,196 @@
|
|||
package org.micord.utils;
|
||||
|
||||
import java.time.LocalDate;
|
||||
import java.util.*;
|
||||
import java.util.concurrent.ThreadLocalRandom;
|
||||
|
||||
/**
|
||||
* Utility class for generating test data for download service tests.
|
||||
*/
|
||||
public class TestDataGenerator {
|
||||
|
||||
private static final String[] FIRST_NAMES = {
|
||||
"John", "Jane", "Michael", "Sarah", "David", "Emma", "Robert", "Lisa", "William", "Mary"
|
||||
};
|
||||
|
||||
private static final String[] LAST_NAMES = {
|
||||
"Smith", "Johnson", "Williams", "Brown", "Jones", "Garcia", "Miller", "Davis", "Rodriguez", "Martinez"
|
||||
};
|
||||
|
||||
private static final String[] CITIES = {
|
||||
"New York", "Los Angeles", "Chicago", "Houston", "Phoenix", "Philadelphia", "San Antonio", "San Diego"
|
||||
};
|
||||
|
||||
/**
|
||||
* Generate a list of maps representing CSV data rows
|
||||
*/
|
||||
public static List<Map<String, Object>> generateMapData(int rows) {
|
||||
List<Map<String, Object>> data = new ArrayList<>();
|
||||
Random random = new Random();
|
||||
|
||||
for (int i = 0; i < rows; i++) {
|
||||
Map<String, Object> row = new HashMap<>();
|
||||
row.put("id", UUID.randomUUID().toString());
|
||||
row.put("firstName", FIRST_NAMES[random.nextInt(FIRST_NAMES.length)]);
|
||||
row.put("lastName", LAST_NAMES[random.nextInt(LAST_NAMES.length)]);
|
||||
row.put("age", 18 + random.nextInt(60));
|
||||
row.put("city", CITIES[random.nextInt(CITIES.length)]);
|
||||
row.put("email", "user" + i + "@example.com");
|
||||
row.put("phone", generatePhoneNumber());
|
||||
row.put("registrationDate", generateRandomDate());
|
||||
row.put("isActive", random.nextBoolean());
|
||||
row.put("balance", String.format("%.2f", random.nextDouble() * 10000));
|
||||
|
||||
data.add(row);
|
||||
}
|
||||
|
||||
return data;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate SQL-style array data (including headers)
|
||||
*/
|
||||
public static List<String[]> generateSqlData(int rows, boolean includeHeaders) {
|
||||
List<String[]> data = new ArrayList<>();
|
||||
Random random = new Random();
|
||||
|
||||
if (includeHeaders) {
|
||||
data.add(new String[]{
|
||||
"id", "firstName", "lastName", "age", "city",
|
||||
"email", "phone", "registrationDate", "isActive", "balance"
|
||||
});
|
||||
}
|
||||
|
||||
for (int i = 0; i < rows; i++) {
|
||||
String[] row = new String[]{
|
||||
UUID.randomUUID().toString(),
|
||||
FIRST_NAMES[random.nextInt(FIRST_NAMES.length)],
|
||||
LAST_NAMES[random.nextInt(LAST_NAMES.length)],
|
||||
String.valueOf(18 + random.nextInt(60)),
|
||||
CITIES[random.nextInt(CITIES.length)],
|
||||
"user" + i + "@example.com",
|
||||
generatePhoneNumber(),
|
||||
generateRandomDate().toString(),
|
||||
String.valueOf(random.nextBoolean()),
|
||||
String.format("%.2f", random.nextDouble() * 10000)
|
||||
};
|
||||
data.add(row);
|
||||
}
|
||||
|
||||
return data;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate data with special characters that need CSV escaping
|
||||
*/
|
||||
public static List<Map<String, Object>> generateProblematicData() {
|
||||
List<Map<String, Object>> data = new ArrayList<>();
|
||||
|
||||
// Row with quotes
|
||||
Map<String, Object> row1 = new HashMap<>();
|
||||
row1.put("id", "1");
|
||||
row1.put("description", "This has \"quotes\" in it");
|
||||
row1.put("name", "John \"The Boss\" Smith");
|
||||
data.add(row1);
|
||||
|
||||
// Row with commas
|
||||
Map<String, Object> row2 = new HashMap<>();
|
||||
row2.put("id", "2");
|
||||
row2.put("description", "Values, with, many, commas");
|
||||
row2.put("name", "Smith, John");
|
||||
data.add(row2);
|
||||
|
||||
// Row with newlines
|
||||
Map<String, Object> row3 = new HashMap<>();
|
||||
row3.put("id", "3");
|
||||
row3.put("description", "First line\nSecond line\nThird line");
|
||||
row3.put("name", "Multi\nLine\nName");
|
||||
data.add(row3);
|
||||
|
||||
// Row with null values
|
||||
Map<String, Object> row4 = new HashMap<>();
|
||||
row4.put("id", "4");
|
||||
row4.put("description", null);
|
||||
row4.put("name", null);
|
||||
data.add(row4);
|
||||
|
||||
// Row with special characters
|
||||
Map<String, Object> row5 = new HashMap<>();
|
||||
row5.put("id", "5");
|
||||
row5.put("description", "Special chars: @#$%^&*()_+-=[]{}|;':\",./<>?");
|
||||
row5.put("name", "Ñoño José");
|
||||
data.add(row5);
|
||||
|
||||
// Row with tabs
|
||||
Map<String, Object> row6 = new HashMap<>();
|
||||
row6.put("id", "6");
|
||||
row6.put("description", "Tab\tseparated\tvalues");
|
||||
row6.put("name", "Tab\tUser");
|
||||
data.add(row6);
|
||||
|
||||
return data;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate edge case datasets for boundary testing
|
||||
*/
|
||||
public static class EdgeCaseDatasets {
|
||||
public static final int JUST_UNDER_LIMIT = 599999;
|
||||
public static final int EXACTLY_AT_LIMIT = 600000;
|
||||
public static final int JUST_OVER_LIMIT = 600001;
|
||||
public static final int ONE_AND_HALF_CHUNKS = 900000;
|
||||
public static final int EXACTLY_TWO_CHUNKS = 1200000;
|
||||
public static final int MULTIPLE_CHUNKS = 2400000;
|
||||
|
||||
public static List<Map<String, Object>> getDataset(int size) {
|
||||
return generateMapData(size);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate large dataset in chunks to avoid memory issues
|
||||
*/
|
||||
public static Iterator<Map<String, Object>> generateLargeDataIterator(int totalRows) {
|
||||
return new Iterator<Map<String, Object>>() {
|
||||
private int currentRow = 0;
|
||||
private final Random random = new Random();
|
||||
|
||||
@Override
|
||||
public boolean hasNext() {
|
||||
return currentRow < totalRows;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<String, Object> next() {
|
||||
if (!hasNext()) {
|
||||
throw new NoSuchElementException();
|
||||
}
|
||||
|
||||
Map<String, Object> row = new HashMap<>();
|
||||
row.put("id", UUID.randomUUID().toString());
|
||||
row.put("rowNumber", currentRow);
|
||||
row.put("timestamp", System.currentTimeMillis());
|
||||
row.put("data", "Row " + currentRow + " of " + totalRows);
|
||||
row.put("random", random.nextDouble());
|
||||
|
||||
currentRow++;
|
||||
return row;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
private static String generatePhoneNumber() {
|
||||
Random random = new Random();
|
||||
return String.format("(%03d) %03d-%04d",
|
||||
random.nextInt(900) + 100,
|
||||
random.nextInt(900) + 100,
|
||||
random.nextInt(9000) + 1000);
|
||||
}
|
||||
|
||||
private static LocalDate generateRandomDate() {
|
||||
long minDay = LocalDate.of(2020, 1, 1).toEpochDay();
|
||||
long maxDay = LocalDate.of(2024, 12, 31).toEpochDay();
|
||||
long randomDay = ThreadLocalRandom.current().nextLong(minDay, maxDay);
|
||||
return LocalDate.ofEpochDay(randomDay);
|
||||
}
|
||||
}
|
||||
8819
frontend/package-lock.json
generated
8819
frontend/package-lock.json
generated
File diff suppressed because it is too large
Load diff
Loading…
Add table
Add a link
Reference in a new issue