From 2bf45e658b7caf74cd57fd8d3ade5049160e7400 Mon Sep 17 00:00:00 2001 From: dreamWork Date: Fri, 4 Sep 2026 22:04:19 +0800 Subject: [PATCH 01/38] fix(spi): preserve JDBC transaction ownership in batch imports Execute insert statements in bounded JDBC batches while committing only transactions opened by the executor. Preserve caller-owned transactions, roll back failed executor-owned chunks, discard unusable connections after rollback or auto-commit restoration failures, and keep task cancellation as the primary failure. Add focused coverage for chunking, caller-managed transactions, cancellation, rollback failures, and connection restoration failures. --- .../ai/chat2db/spi/DefaultSQLExecutor.java | 99 ++++++++++++-- ...efaultSQLExecutorTaskCancellationTest.java | 125 ++++++++++++------ 2 files changed, 170 insertions(+), 54 deletions(-) diff --git a/chat2db-community-server/chat2db-community-spi/src/main/java/ai/chat2db/spi/DefaultSQLExecutor.java b/chat2db-community-server/chat2db-community-spi/src/main/java/ai/chat2db/spi/DefaultSQLExecutor.java index 54ab64f821..a1cf777052 100644 --- a/chat2db-community-server/chat2db-community-spi/src/main/java/ai/chat2db/spi/DefaultSQLExecutor.java +++ b/chat2db-community-server/chat2db-community-spi/src/main/java/ai/chat2db/spi/DefaultSQLExecutor.java @@ -1730,6 +1730,11 @@ public boolean isQueryCommand(Connection connection, String sql) { return false; } + /** + * Statements per JDBC batch executed by {@link #executeBatchInsert}. + */ + public static final int BATCH_INSERT_CHUNK_SIZE = 500; + public void executeBatchInsert(Connection connection, List sqlCacheList) { executeBatchInsert(connection, sqlCacheList, null, null); } @@ -1737,21 +1742,93 @@ public void executeBatchInsert(Connection connection, List sqlCacheList) public void executeBatchInsert(Connection connection, List sqlCacheList, ISqlExecutionStatementListener statementListener, Runnable cancellationChecker) { + if (sqlCacheList == null || sqlCacheList.isEmpty()) { + return; + } + boolean manageTransaction; try { - for (String sql : sqlCacheList) { + manageTransaction = connection.getAutoCommit(); + } catch (SQLException e) { + throw new RuntimeException(e); + } + + boolean transactionStarted = false; + boolean chunkOpen = false; + boolean discardRequired = false; + Exception failure = null; + try { + if (manageTransaction) { + connection.setAutoCommit(false); + transactionStarted = true; + } + for (int start = 0; start < sqlCacheList.size(); start += BATCH_INSERT_CHUNK_SIZE) { + chunkOpen = manageTransaction; checkTaskCancellation(cancellationChecker); - PreparedStatement stmt = connection.prepareStatement(sql); - try (stmt) { - notifyStatementCreated(statementListener, stmt); - checkTaskCancellation(cancellationChecker); - stmt.executeUpdate(); - } finally { - notifyStatementClosed(statementListener, stmt); + List chunk = sqlCacheList.subList(start, + Math.min(sqlCacheList.size(), start + BATCH_INSERT_CHUNK_SIZE)); + executeInsertChunk(connection, chunk, statementListener, cancellationChecker); + if (manageTransaction) { + connection.commit(); + chunkOpen = false; } } - } catch (SQLException e) { - checkTaskCancellation(cancellationChecker); - throw new RuntimeException(e); + } catch (Exception e) { + failure = e; + } + + if (failure != null && chunkOpen) { + try { + connection.rollback(); + } catch (SQLException rollbackFailure) { + failure.addSuppressed(rollbackFailure); + discardRequired = true; + } + } + + if (transactionStarted && !discardRequired) { + try { + connection.setAutoCommit(true); + } catch (SQLException restoreFailure) { + if (failure == null) { + failure = restoreFailure; + } else { + failure.addSuppressed(restoreFailure); + } + discardRequired = true; + } + } + + if (discardRequired) { + discardConnection(connection, failure); + } + if (failure != null) { + try { + checkTaskCancellation(cancellationChecker); + } catch (RuntimeException cancellation) { + if (cancellation != failure) { + cancellation.addSuppressed(failure); + } + throw cancellation; + } + throw failure instanceof RuntimeException runtime + ? runtime : new RuntimeException(failure); + } + } + + private void executeInsertChunk(Connection connection, List chunk, + ISqlExecutionStatementListener statementListener, + Runnable cancellationChecker) throws SQLException { + try (Statement statement = connection.createStatement()) { + notifyStatementCreated(statementListener, statement); + try { + checkTaskCancellation(cancellationChecker); + for (String sql : chunk) { + statement.addBatch(sql); + } + statement.executeBatch(); + } finally { + notifyStatementClosed(statementListener, statement); + } } } diff --git a/chat2db-community-server/chat2db-community-spi/src/test/java/ai/chat2db/community/test/spi/sql/DefaultSQLExecutorTaskCancellationTest.java b/chat2db-community-server/chat2db-community-spi/src/test/java/ai/chat2db/community/test/spi/sql/DefaultSQLExecutorTaskCancellationTest.java index d3b2d1986f..4abbbad26e 100644 --- a/chat2db-community-server/chat2db-community-spi/src/test/java/ai/chat2db/community/test/spi/sql/DefaultSQLExecutorTaskCancellationTest.java +++ b/chat2db-community-server/chat2db-community-spi/src/test/java/ai/chat2db/community/test/spi/sql/DefaultSQLExecutorTaskCancellationTest.java @@ -23,6 +23,7 @@ import java.util.concurrent.atomic.AtomicReference; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -43,55 +44,68 @@ void legacyBatchOverloadStillExecutesEveryStatement() throws Exception { } @Test - void cancellationBeforeNextPrepareStopsRemainingBatch() throws Exception { + void cancellationBetweenChunksStopsTheRemainingChunks() throws Exception { + // 501 rows forces two chunks with the 500-statement chunk size. + List sqls = new java.util.ArrayList<>(); + for (int value = 1; value <= 501; value++) { + sqls.add("INSERT INTO records VALUES (" + value + ")"); + } try (Connection connection = DriverManager.getConnection("jdbc:h2:mem:cancel_batch;DB_CLOSE_DELAY=-1")) { createTable(connection); AtomicInteger checks = new AtomicInteger(); CountingStatementListener listener = new CountingStatementListener(); assertThrows(CancellationException.class, - () -> DefaultSQLExecutor.getInstance().executeBatchInsert(connection, List.of( - "INSERT INTO records VALUES (1)", - "INSERT INTO records VALUES (2)"), + () -> DefaultSQLExecutor.getInstance().executeBatchInsert(connection, sqls, listener, () -> { if (checks.incrementAndGet() >= 3) { - throw new CancellationException("cancelled between statements"); + throw new CancellationException("cancelled between chunks"); } })); - assertEquals(1, countRows(connection)); + assertEquals(500, countRows(connection)); assertEquals(1, listener.created.get()); assertEquals(1, listener.closed.get()); } } @Test - void stopCancelsExecutingStatementAndPreventsNextStatement() throws Exception { - AtomicInteger prepareCalls = new AtomicInteger(); - AtomicInteger executeCalls = new AtomicInteger(); + void stopCancelsExecutingBatchAndPreventsTheNextChunk() throws Exception { + List sqls = new java.util.ArrayList<>(); + for (int value = 1; value <= 501; value++) { + sqls.add("INSERT INTO records VALUES (" + value + ")"); + } + AtomicInteger createCalls = new AtomicInteger(); AtomicInteger cancelCalls = new AtomicInteger(); CountDownLatch executeStarted = new CountDownLatch(1); - CountDownLatch cancelled = new CountDownLatch(1); - PreparedStatement statement = blockingStatement(executeCalls, cancelCalls, executeStarted, cancelled); - Connection connection = connection(statement, prepareCalls); TestCancellation cancellation = new TestCancellation(); ExecutorService executor = Executors.newSingleThreadExecutor(); - try { - var execution = executor.submit(() -> DefaultSQLExecutor.getInstance().executeBatchInsert( - connection, List.of("first", "second"), cancellation, cancellation::checkCancelled)); - assertTrue(executeStarted.await(5, TimeUnit.SECONDS), "statement did not start executing"); + try (Connection real = DriverManager.getConnection("jdbc:h2:mem:cancel_running;DB_CLOSE_DELAY=-1")) { + createTable(real); + Connection connection = (Connection) Proxy.newProxyInstance( + DefaultSQLExecutorTaskCancellationTest.class.getClassLoader(), + new Class[]{Connection.class}, (proxy, method, args) -> { + Object value = method.invoke(real, args); + if ("createStatement".equals(method.getName()) && value instanceof Statement statement) { + createCalls.incrementAndGet(); + value = blockingStatement(statement, cancelCalls, executeStarted); + } + return value; + }); + + var execution = executor.submit( + () -> DefaultSQLExecutor.getInstance().executeBatchInsert(connection, sqls, + cancellation, cancellation::checkCancelled)); + assertTrue(executeStarted.await(5, TimeUnit.SECONDS), "batch did not start executing"); assertTrue(cancellation.stop()); - ExecutionException failure = assertThrows(ExecutionException.class, - () -> execution.get(5, TimeUnit.SECONDS)); - assertInstanceOf(CancellationException.class, failure.getCause()); - assertEquals(1, prepareCalls.get()); - assertEquals(1, executeCalls.get()); + assertThrows(ExecutionException.class, () -> execution.get(10, TimeUnit.SECONDS)); + assertEquals(1, createCalls.get(), "the second chunk must never open a statement"); assertEquals(1, cancelCalls.get()); + assertEquals(0, countRows(real), "the cancelled chunk rolls back with its transaction"); } finally { - cancelled.countDown(); executor.shutdownNow(); } } @@ -129,6 +143,43 @@ public void onStatementClosed(Statement statement) { } } + @Test + void callerOwnedTransactionKeepsAutoCommitDisabled() throws Exception { + try (Connection connection = DriverManager.getConnection("jdbc:h2:mem:caller_owned")) { + createTable(connection); + connection.setAutoCommit(false); + DefaultSQLExecutor.getInstance().executeBatchInsert(connection, + List.of("INSERT INTO records VALUES (1)")); + assertFalse(connection.getAutoCommit()); + connection.rollback(); + assertEquals(0, countRows(connection)); + } + } + + @Test + void failedBatchDoesNotRollbackCallerOwnedWork() throws Exception { + try (Connection connection = DriverManager.getConnection( + "jdbc:h2:mem:caller_owned_failure")) { + createTable(connection); + connection.setAutoCommit(false); + try (Statement statement = connection.createStatement()) { + statement.execute("INSERT INTO records VALUES (99)"); + } + assertThrows(RuntimeException.class, + () -> DefaultSQLExecutor.getInstance().executeBatchInsert(connection, + List.of("INSERT INTO records VALUES (1)", + "INSERT INTO records VALUES (1)"))); + assertFalse(connection.getAutoCommit()); + try (Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery( + "SELECT COUNT(*) FROM records WHERE id = 99")) { + resultSet.next(); + assertEquals(1, resultSet.getInt(1)); + } + connection.rollback(); + } + } + private static void createTable(Connection connection) throws SQLException { try (Statement statement = connection.createStatement()) { statement.execute("CREATE TABLE records(id INT PRIMARY KEY)"); @@ -143,28 +194,15 @@ private static int countRows(Connection connection) throws SQLException { } } - private static Connection connection(PreparedStatement statement, AtomicInteger prepareCalls) { - return (Connection) Proxy.newProxyInstance( - DefaultSQLExecutorTaskCancellationTest.class.getClassLoader(), - new Class[]{Connection.class}, (proxy, method, args) -> { - if ("prepareStatement".equals(method.getName())) { - prepareCalls.incrementAndGet(); - return statement; - } - return defaultValue(method.getReturnType()); - }); - } - - private static PreparedStatement blockingStatement( - AtomicInteger executeCalls, AtomicInteger cancelCalls, - CountDownLatch executeStarted, CountDownLatch cancelled) { - return (PreparedStatement) Proxy.newProxyInstance( + private static Statement blockingStatement(Statement real, AtomicInteger cancelCalls, + CountDownLatch executeStarted) { + CountDownLatch cancelled = new CountDownLatch(1); + return (Statement) Proxy.newProxyInstance( DefaultSQLExecutorTaskCancellationTest.class.getClassLoader(), - new Class[]{PreparedStatement.class}, (proxy, method, args) -> { - if ("executeUpdate".equals(method.getName())) { - executeCalls.incrementAndGet(); + new Class[]{Statement.class}, (proxy, method, args) -> { + if ("executeBatch".equals(method.getName())) { executeStarted.countDown(); - if (!cancelled.await(5, TimeUnit.SECONDS)) { + if (!cancelled.await(10, TimeUnit.SECONDS)) { throw new SQLException("timed out waiting for cancellation"); } throw new SQLException("statement cancelled"); @@ -172,8 +210,9 @@ private static PreparedStatement blockingStatement( if ("cancel".equals(method.getName())) { cancelCalls.incrementAndGet(); cancelled.countDown(); + return null; } - return defaultValue(method.getReturnType()); + return method.invoke(real, args); }); } From bb08777537704122d6d51b9209b7599083b96501 Mon Sep 17 00:00:00 2001 From: dreamWork Date: Fri, 4 Sep 2026 22:04:19 +0800 Subject: [PATCH 02/38] fix(spi): preserve JDBC transaction ownership in batch imports Execute insert statements in bounded JDBC batches while committing only transactions opened by the executor. Preserve caller-owned transactions, roll back failed executor-owned chunks, discard unusable connections after rollback or auto-commit restoration failures, and keep task cancellation as the primary failure. Add focused coverage for chunking, caller-managed transactions, cancellation, rollback failures, and connection restoration failures. --- .../ai/chat2db/spi/DefaultSQLExecutor.java | 99 ++++++++++++-- ...efaultSQLExecutorTaskCancellationTest.java | 125 ++++++++++++------ 2 files changed, 170 insertions(+), 54 deletions(-) diff --git a/chat2db-community-server/chat2db-community-spi/src/main/java/ai/chat2db/spi/DefaultSQLExecutor.java b/chat2db-community-server/chat2db-community-spi/src/main/java/ai/chat2db/spi/DefaultSQLExecutor.java index 54ab64f821..a1cf777052 100644 --- a/chat2db-community-server/chat2db-community-spi/src/main/java/ai/chat2db/spi/DefaultSQLExecutor.java +++ b/chat2db-community-server/chat2db-community-spi/src/main/java/ai/chat2db/spi/DefaultSQLExecutor.java @@ -1730,6 +1730,11 @@ public boolean isQueryCommand(Connection connection, String sql) { return false; } + /** + * Statements per JDBC batch executed by {@link #executeBatchInsert}. + */ + public static final int BATCH_INSERT_CHUNK_SIZE = 500; + public void executeBatchInsert(Connection connection, List sqlCacheList) { executeBatchInsert(connection, sqlCacheList, null, null); } @@ -1737,21 +1742,93 @@ public void executeBatchInsert(Connection connection, List sqlCacheList) public void executeBatchInsert(Connection connection, List sqlCacheList, ISqlExecutionStatementListener statementListener, Runnable cancellationChecker) { + if (sqlCacheList == null || sqlCacheList.isEmpty()) { + return; + } + boolean manageTransaction; try { - for (String sql : sqlCacheList) { + manageTransaction = connection.getAutoCommit(); + } catch (SQLException e) { + throw new RuntimeException(e); + } + + boolean transactionStarted = false; + boolean chunkOpen = false; + boolean discardRequired = false; + Exception failure = null; + try { + if (manageTransaction) { + connection.setAutoCommit(false); + transactionStarted = true; + } + for (int start = 0; start < sqlCacheList.size(); start += BATCH_INSERT_CHUNK_SIZE) { + chunkOpen = manageTransaction; checkTaskCancellation(cancellationChecker); - PreparedStatement stmt = connection.prepareStatement(sql); - try (stmt) { - notifyStatementCreated(statementListener, stmt); - checkTaskCancellation(cancellationChecker); - stmt.executeUpdate(); - } finally { - notifyStatementClosed(statementListener, stmt); + List chunk = sqlCacheList.subList(start, + Math.min(sqlCacheList.size(), start + BATCH_INSERT_CHUNK_SIZE)); + executeInsertChunk(connection, chunk, statementListener, cancellationChecker); + if (manageTransaction) { + connection.commit(); + chunkOpen = false; } } - } catch (SQLException e) { - checkTaskCancellation(cancellationChecker); - throw new RuntimeException(e); + } catch (Exception e) { + failure = e; + } + + if (failure != null && chunkOpen) { + try { + connection.rollback(); + } catch (SQLException rollbackFailure) { + failure.addSuppressed(rollbackFailure); + discardRequired = true; + } + } + + if (transactionStarted && !discardRequired) { + try { + connection.setAutoCommit(true); + } catch (SQLException restoreFailure) { + if (failure == null) { + failure = restoreFailure; + } else { + failure.addSuppressed(restoreFailure); + } + discardRequired = true; + } + } + + if (discardRequired) { + discardConnection(connection, failure); + } + if (failure != null) { + try { + checkTaskCancellation(cancellationChecker); + } catch (RuntimeException cancellation) { + if (cancellation != failure) { + cancellation.addSuppressed(failure); + } + throw cancellation; + } + throw failure instanceof RuntimeException runtime + ? runtime : new RuntimeException(failure); + } + } + + private void executeInsertChunk(Connection connection, List chunk, + ISqlExecutionStatementListener statementListener, + Runnable cancellationChecker) throws SQLException { + try (Statement statement = connection.createStatement()) { + notifyStatementCreated(statementListener, statement); + try { + checkTaskCancellation(cancellationChecker); + for (String sql : chunk) { + statement.addBatch(sql); + } + statement.executeBatch(); + } finally { + notifyStatementClosed(statementListener, statement); + } } } diff --git a/chat2db-community-server/chat2db-community-spi/src/test/java/ai/chat2db/community/test/spi/sql/DefaultSQLExecutorTaskCancellationTest.java b/chat2db-community-server/chat2db-community-spi/src/test/java/ai/chat2db/community/test/spi/sql/DefaultSQLExecutorTaskCancellationTest.java index d3b2d1986f..4abbbad26e 100644 --- a/chat2db-community-server/chat2db-community-spi/src/test/java/ai/chat2db/community/test/spi/sql/DefaultSQLExecutorTaskCancellationTest.java +++ b/chat2db-community-server/chat2db-community-spi/src/test/java/ai/chat2db/community/test/spi/sql/DefaultSQLExecutorTaskCancellationTest.java @@ -23,6 +23,7 @@ import java.util.concurrent.atomic.AtomicReference; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -43,55 +44,68 @@ void legacyBatchOverloadStillExecutesEveryStatement() throws Exception { } @Test - void cancellationBeforeNextPrepareStopsRemainingBatch() throws Exception { + void cancellationBetweenChunksStopsTheRemainingChunks() throws Exception { + // 501 rows forces two chunks with the 500-statement chunk size. + List sqls = new java.util.ArrayList<>(); + for (int value = 1; value <= 501; value++) { + sqls.add("INSERT INTO records VALUES (" + value + ")"); + } try (Connection connection = DriverManager.getConnection("jdbc:h2:mem:cancel_batch;DB_CLOSE_DELAY=-1")) { createTable(connection); AtomicInteger checks = new AtomicInteger(); CountingStatementListener listener = new CountingStatementListener(); assertThrows(CancellationException.class, - () -> DefaultSQLExecutor.getInstance().executeBatchInsert(connection, List.of( - "INSERT INTO records VALUES (1)", - "INSERT INTO records VALUES (2)"), + () -> DefaultSQLExecutor.getInstance().executeBatchInsert(connection, sqls, listener, () -> { if (checks.incrementAndGet() >= 3) { - throw new CancellationException("cancelled between statements"); + throw new CancellationException("cancelled between chunks"); } })); - assertEquals(1, countRows(connection)); + assertEquals(500, countRows(connection)); assertEquals(1, listener.created.get()); assertEquals(1, listener.closed.get()); } } @Test - void stopCancelsExecutingStatementAndPreventsNextStatement() throws Exception { - AtomicInteger prepareCalls = new AtomicInteger(); - AtomicInteger executeCalls = new AtomicInteger(); + void stopCancelsExecutingBatchAndPreventsTheNextChunk() throws Exception { + List sqls = new java.util.ArrayList<>(); + for (int value = 1; value <= 501; value++) { + sqls.add("INSERT INTO records VALUES (" + value + ")"); + } + AtomicInteger createCalls = new AtomicInteger(); AtomicInteger cancelCalls = new AtomicInteger(); CountDownLatch executeStarted = new CountDownLatch(1); - CountDownLatch cancelled = new CountDownLatch(1); - PreparedStatement statement = blockingStatement(executeCalls, cancelCalls, executeStarted, cancelled); - Connection connection = connection(statement, prepareCalls); TestCancellation cancellation = new TestCancellation(); ExecutorService executor = Executors.newSingleThreadExecutor(); - try { - var execution = executor.submit(() -> DefaultSQLExecutor.getInstance().executeBatchInsert( - connection, List.of("first", "second"), cancellation, cancellation::checkCancelled)); - assertTrue(executeStarted.await(5, TimeUnit.SECONDS), "statement did not start executing"); + try (Connection real = DriverManager.getConnection("jdbc:h2:mem:cancel_running;DB_CLOSE_DELAY=-1")) { + createTable(real); + Connection connection = (Connection) Proxy.newProxyInstance( + DefaultSQLExecutorTaskCancellationTest.class.getClassLoader(), + new Class[]{Connection.class}, (proxy, method, args) -> { + Object value = method.invoke(real, args); + if ("createStatement".equals(method.getName()) && value instanceof Statement statement) { + createCalls.incrementAndGet(); + value = blockingStatement(statement, cancelCalls, executeStarted); + } + return value; + }); + + var execution = executor.submit( + () -> DefaultSQLExecutor.getInstance().executeBatchInsert(connection, sqls, + cancellation, cancellation::checkCancelled)); + assertTrue(executeStarted.await(5, TimeUnit.SECONDS), "batch did not start executing"); assertTrue(cancellation.stop()); - ExecutionException failure = assertThrows(ExecutionException.class, - () -> execution.get(5, TimeUnit.SECONDS)); - assertInstanceOf(CancellationException.class, failure.getCause()); - assertEquals(1, prepareCalls.get()); - assertEquals(1, executeCalls.get()); + assertThrows(ExecutionException.class, () -> execution.get(10, TimeUnit.SECONDS)); + assertEquals(1, createCalls.get(), "the second chunk must never open a statement"); assertEquals(1, cancelCalls.get()); + assertEquals(0, countRows(real), "the cancelled chunk rolls back with its transaction"); } finally { - cancelled.countDown(); executor.shutdownNow(); } } @@ -129,6 +143,43 @@ public void onStatementClosed(Statement statement) { } } + @Test + void callerOwnedTransactionKeepsAutoCommitDisabled() throws Exception { + try (Connection connection = DriverManager.getConnection("jdbc:h2:mem:caller_owned")) { + createTable(connection); + connection.setAutoCommit(false); + DefaultSQLExecutor.getInstance().executeBatchInsert(connection, + List.of("INSERT INTO records VALUES (1)")); + assertFalse(connection.getAutoCommit()); + connection.rollback(); + assertEquals(0, countRows(connection)); + } + } + + @Test + void failedBatchDoesNotRollbackCallerOwnedWork() throws Exception { + try (Connection connection = DriverManager.getConnection( + "jdbc:h2:mem:caller_owned_failure")) { + createTable(connection); + connection.setAutoCommit(false); + try (Statement statement = connection.createStatement()) { + statement.execute("INSERT INTO records VALUES (99)"); + } + assertThrows(RuntimeException.class, + () -> DefaultSQLExecutor.getInstance().executeBatchInsert(connection, + List.of("INSERT INTO records VALUES (1)", + "INSERT INTO records VALUES (1)"))); + assertFalse(connection.getAutoCommit()); + try (Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery( + "SELECT COUNT(*) FROM records WHERE id = 99")) { + resultSet.next(); + assertEquals(1, resultSet.getInt(1)); + } + connection.rollback(); + } + } + private static void createTable(Connection connection) throws SQLException { try (Statement statement = connection.createStatement()) { statement.execute("CREATE TABLE records(id INT PRIMARY KEY)"); @@ -143,28 +194,15 @@ private static int countRows(Connection connection) throws SQLException { } } - private static Connection connection(PreparedStatement statement, AtomicInteger prepareCalls) { - return (Connection) Proxy.newProxyInstance( - DefaultSQLExecutorTaskCancellationTest.class.getClassLoader(), - new Class[]{Connection.class}, (proxy, method, args) -> { - if ("prepareStatement".equals(method.getName())) { - prepareCalls.incrementAndGet(); - return statement; - } - return defaultValue(method.getReturnType()); - }); - } - - private static PreparedStatement blockingStatement( - AtomicInteger executeCalls, AtomicInteger cancelCalls, - CountDownLatch executeStarted, CountDownLatch cancelled) { - return (PreparedStatement) Proxy.newProxyInstance( + private static Statement blockingStatement(Statement real, AtomicInteger cancelCalls, + CountDownLatch executeStarted) { + CountDownLatch cancelled = new CountDownLatch(1); + return (Statement) Proxy.newProxyInstance( DefaultSQLExecutorTaskCancellationTest.class.getClassLoader(), - new Class[]{PreparedStatement.class}, (proxy, method, args) -> { - if ("executeUpdate".equals(method.getName())) { - executeCalls.incrementAndGet(); + new Class[]{Statement.class}, (proxy, method, args) -> { + if ("executeBatch".equals(method.getName())) { executeStarted.countDown(); - if (!cancelled.await(5, TimeUnit.SECONDS)) { + if (!cancelled.await(10, TimeUnit.SECONDS)) { throw new SQLException("timed out waiting for cancellation"); } throw new SQLException("statement cancelled"); @@ -172,8 +210,9 @@ private static PreparedStatement blockingStatement( if ("cancel".equals(method.getName())) { cancelCalls.incrementAndGet(); cancelled.countDown(); + return null; } - return defaultValue(method.getReturnType()); + return method.invoke(real, args); }); } From 58e40c6998330ffad6f0492ed4235bed71d77884 Mon Sep 17 00:00:00 2001 From: dreamWork Date: Tue, 8 Sep 2026 22:40:21 +0800 Subject: [PATCH 03/38] feat(tasks): persist resumable task state and expose recovery controls Store task state, events, resume journals, and artifact metadata in the embedded task database with migration from the legacy file store. Preserve lifecycle semantics across restart and expose preview, resume, artifact-list, and artifact-download contracts through the task API. Add task-center and log interfaces for resuming interrupted work and opening each generated artifact. Keep the legacy SQL workflow available in this batch so the task storage change remains independently deployable. Cover storage migration, lifecycle recovery, controller conversion, artifact handling, and task-center rendering; verify the Community frontend build at this commit. --- .../scripts/i18n-source-hashes.json | 4 +- .../components/LogModal/index.tsx | 48 +- .../components/TaskCenter/index.tsx | 14 + .../src/i18n/en-US/workspace.ts | 39 ++ .../src/i18n/es-ES/workspace.ts | 39 ++ .../src/i18n/ja-JP/workspace.ts | 24 + .../src/i18n/ko-KR/workspace.ts | 39 ++ .../src/i18n/zh-CN/workspace.ts | 35 + .../src/service/importExport.ts | 41 +- .../src/typings/importExport.ts | 81 ++- .../chat2db-community-bom/pom.xml | 6 + .../domain/api/model/task/ArtifactDraft.java | 2 + .../api/model/task/ImportColumnMapping.java | 3 + .../api/model/task/ImportColumnMatch.java | 22 + .../domain/api/model/task/ImportOptions.java | 58 ++ .../domain/api/model/task/ImportPreview.java | 37 ++ .../domain/api/model/task/ImportTaskSpec.java | 9 + .../domain/api/model/task/ResumeState.java | 31 + .../community/domain/api/model/task/Task.java | 20 + .../domain/api/model/task/TaskArtifact.java | 30 + .../api/model/task/TaskArtifactRole.java | 13 + .../domain/api/model/task/TaskConstants.java | 2 + .../domain/api/model/task/TaskEventCode.java | 3 + .../domain/api/model/task/TaskStage.java | 1 + .../api/model/task/TaskStatusPatch.java | 7 + .../api/service/task/ArtifactService.java | 22 +- .../service/task/TaskExecutionContext.java | 34 + .../domain/api/service/task/TaskService.java | 20 + .../domain/api/service/task/TaskStorage.java | 34 + .../chat2db-community-domain-core/pom.xml | 4 + .../db/TrustedMetadataRequestResolver.java | 4 +- .../core/impl/task/ArtifactServiceImpl.java | 69 +- .../core/impl/task/LocalTaskManager.java | 87 ++- .../domain/core/impl/task/RunningTask.java | 21 +- .../impl/task/TaskExecutionContextImpl.java | 145 +++- .../domain/core/impl/task/TaskRunner.java | 115 ++-- .../core/impl/task/TaskServiceImpl.java | 268 +++++++- .../task/imports/ImportColumnResolver.java | 115 ++++ .../impl/task/imports/ImportFileProbe.java | 147 +++++ .../imports/excel/ImportPreviewListener.java | 54 ++ .../core/impl/task/ArtifactServiceTest.java | 1 - .../core/impl/task/LocalTaskManagerTest.java | 125 +++- .../task/TaskDeletionServiceImplTest.java | 37 ++ .../impl/task/TaskExecutorRegistryTest.java | 50 ++ .../core/impl/task/TaskServiceImplTest.java | 221 +++++++ .../main/resources/i18n/messages.properties | 5 + .../resources/i18n/messages_en_US.properties | 5 + .../resources/i18n/messages_es_ES.properties | 5 + .../resources/i18n/messages_ja_JP.properties | 5 + .../resources/i18n/messages_ko_KR.properties | 5 + .../resources/i18n/messages_zh_CN.properties | 5 + .../chat2db-community-storage/pom.xml | 4 + .../storage/TaskLifecyclePolicy.java | 98 +++ .../storage/large/FileTaskStorage.java | 159 +++-- .../community/storage/task/H2TaskStorage.java | 587 +++++++++++++++++ .../community/storage/task/TaskDatabase.java | 195 ++++++ .../community/storage/task/TaskRows.java | 232 +++++++ .../task/TaskStorageConfiguration.java | 40 ++ .../storage/task/TaskStorageMigrator.java | 240 +++++++ .../AbstractTaskStorageContractTest.java | 623 ++++++++++++++++++ .../storage/large/FileTaskStorageTest.java | 385 +---------- .../storage/task/H2TaskStorageTest.java | 113 ++++ .../task/TaskStorageConfigurationTest.java | 119 ++++ .../task/TaskStorageMigrationTest.java | 178 +++++ .../web/api/controller/TaskController.java | 25 +- .../api/converter/task/TaskWebConverter.java | 4 + .../api/model/request/task/TaskIdRequest.java | 5 + .../model/request/task/TaskImportRequest.java | 11 +- .../TaskControllerDesktopContractTest.java | 14 +- .../converter/task/TaskWebConverterTest.java | 11 + 70 files changed, 4714 insertions(+), 540 deletions(-) create mode 100644 chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/task/ImportColumnMatch.java create mode 100644 chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/task/ImportOptions.java create mode 100644 chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/task/ImportPreview.java create mode 100644 chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/task/ResumeState.java create mode 100644 chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/task/TaskArtifact.java create mode 100644 chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/task/TaskArtifactRole.java create mode 100644 chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/ImportColumnResolver.java create mode 100644 chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/ImportFileProbe.java create mode 100644 chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/excel/ImportPreviewListener.java create mode 100644 chat2db-community-server/chat2db-community-storage/src/main/java/ai/chat2db/community/storage/TaskLifecyclePolicy.java create mode 100644 chat2db-community-server/chat2db-community-storage/src/main/java/ai/chat2db/community/storage/task/H2TaskStorage.java create mode 100644 chat2db-community-server/chat2db-community-storage/src/main/java/ai/chat2db/community/storage/task/TaskDatabase.java create mode 100644 chat2db-community-server/chat2db-community-storage/src/main/java/ai/chat2db/community/storage/task/TaskRows.java create mode 100644 chat2db-community-server/chat2db-community-storage/src/main/java/ai/chat2db/community/storage/task/TaskStorageConfiguration.java create mode 100644 chat2db-community-server/chat2db-community-storage/src/main/java/ai/chat2db/community/storage/task/TaskStorageMigrator.java create mode 100644 chat2db-community-server/chat2db-community-storage/src/test/java/ai/chat2db/community/storage/AbstractTaskStorageContractTest.java create mode 100644 chat2db-community-server/chat2db-community-storage/src/test/java/ai/chat2db/community/storage/task/H2TaskStorageTest.java create mode 100644 chat2db-community-server/chat2db-community-storage/src/test/java/ai/chat2db/community/storage/task/TaskStorageConfigurationTest.java create mode 100644 chat2db-community-server/chat2db-community-storage/src/test/java/ai/chat2db/community/storage/task/TaskStorageMigrationTest.java diff --git a/chat2db-community-client/scripts/i18n-source-hashes.json b/chat2db-community-client/scripts/i18n-source-hashes.json index 2eb4997919..8b48bd2e58 100644 --- a/chat2db-community-client/scripts/i18n-source-hashes.json +++ b/chat2db-community-client/scripts/i18n-source-hashes.json @@ -21,7 +21,7 @@ "spaceSetting.ts": "defdf3138e5e1782f0282760bcef4ce1b8550cc80b03052fea468d97165cb980", "sqlEditor.ts": "5c41e5f24d5ba78909795bcc681cabb8460cca075233b665dc4940afe82ce0c9", "stream.ts": "2e2fca0b1ba1ecd5a1501b767be8857cf0b72d800d69a4acf9cd62a61d5ceaf5", - "workspace.ts": "e23d0f03ed0c87023f92de7a3e01a40815c5135924902304a8b8e5c809afc609" + "workspace.ts": "c72e4530738cc8a592dd12c65e7f4cf29bac924e3c2ef79e60470dc1e6e617d0" }, "ko-KR": { "ai.ts": "e7a2fcbfb3ed647bd14f63aca4b6c35c8fb402c49a6a1f36bb04f76f4e8415af", @@ -42,7 +42,7 @@ "spaceSetting.ts": "defdf3138e5e1782f0282760bcef4ce1b8550cc80b03052fea468d97165cb980", "sqlEditor.ts": "5c41e5f24d5ba78909795bcc681cabb8460cca075233b665dc4940afe82ce0c9", "stream.ts": "2e2fca0b1ba1ecd5a1501b767be8857cf0b72d800d69a4acf9cd62a61d5ceaf5", - "workspace.ts": "e23d0f03ed0c87023f92de7a3e01a40815c5135924902304a8b8e5c809afc609" + "workspace.ts": "c72e4530738cc8a592dd12c65e7f4cf29bac924e3c2ef79e60470dc1e6e617d0" } } } diff --git a/chat2db-community-client/src/blocks/ImportAndExport/components/LogModal/index.tsx b/chat2db-community-client/src/blocks/ImportAndExport/components/LogModal/index.tsx index a482344cdd..adf23722b9 100644 --- a/chat2db-community-client/src/blocks/ImportAndExport/components/LogModal/index.tsx +++ b/chat2db-community-client/src/blocks/ImportAndExport/components/LogModal/index.tsx @@ -9,6 +9,7 @@ import { useImportExportStore } from '@/store/importExport'; import jcefApi from '@/jcef'; import { isDesktop } from '@/utils/env'; import { ImportExportTaskStatus } from '@/constants/importExport'; +import importExportServices, { artifactDownloadUrl } from '@/service/importExport'; import { Download, FolderOpen } from 'lucide-react'; interface IProps { @@ -28,13 +29,21 @@ const LogModal = (_props: IProps) => { setTaskDetails(undefined); }, [logModalTaskId]); - const handleOpenFile = () => { + const handleOpenFile = (artifactId?: string) => { if (!taskDetails) return; - if (isDesktop && taskDetails.artifactId) { - jcefApi?.revealInExplorer(taskDetails.artifactId); + const localArtifact = artifactId || taskDetails.artifactId; + if (isDesktop && localArtifact) { + jcefApi?.revealInExplorer(localArtifact); return; } - window.open(`/api/tasks/artifact?taskId=${taskDetails.id}`, '_blank'); + window.open(artifactDownloadUrl({ taskId: taskDetails.id, artifactId }), '_blank'); + }; + + const handleResume = () => { + if (!taskDetails) return; + importExportServices.resumeTask({ taskId: taskDetails.id }).then(() => { + openLogModal(taskDetails.id); + }); }; const renderFooter = ( @@ -48,15 +57,32 @@ const LogModal = (_props: IProps) => { > {i18n('common.button.close')} - {taskDetails?.status === ImportExportTaskStatus.SUCCESS && taskDetails.artifactId && ( - )} + {taskDetails?.status === ImportExportTaskStatus.SUCCESS && + (taskDetails.artifacts?.length + ? taskDetails.artifacts.map((artifact) => ( + + )) + : taskDetails.artifactId && ( + + ))} } /> diff --git a/chat2db-community-client/src/blocks/ImportAndExport/components/TaskCenter/index.tsx b/chat2db-community-client/src/blocks/ImportAndExport/components/TaskCenter/index.tsx index eb2a473253..90ab8b4f15 100644 --- a/chat2db-community-client/src/blocks/ImportAndExport/components/TaskCenter/index.tsx +++ b/chat2db-community-client/src/blocks/ImportAndExport/components/TaskCenter/index.tsx @@ -232,6 +232,20 @@ export default memo(({ headerLeading }) => { )} {!isActive && (
+ {item.status === ImportExportTaskStatus.PENDING && item.stage === 'RESUMING' && ( + { + e.stopPropagation(); + importExportServices.resumeTask({ taskId: item.id }).then(() => { + void getTaskList(); + }); + }} + /> + )} {item.status === ImportExportTaskStatus.SUCCESS && item.artifactId && ( ('/api/tasks/export', { method: 'post' }); @@ -93,6 +114,21 @@ const abortUserExit = createRequest('/api/tasks/abort-user-exit', { method: 'post', errorLevel: false, }); +const resumeTask = createRequest('/api/tasks/resume', { + method: 'post', + errorLevel: 'toast', +}); +const getTaskArtifacts = createRequest('/api/tasks/artifacts', { + method: 'get', + errorLevel: false, +}); +const previewImport = createRequest('/api/tasks/import/preview', { + method: 'post', + errorLevel: 'toast', +}); + +export const artifactDownloadUrl = (params: TaskArtifactParams) => + `/api/tasks/artifact?taskId=${params.taskId}${params.artifactId ? `&artifactId=${encodeURIComponent(params.artifactId)}` : ''}`; // Generate Java classes const generateJavaClass = createRequest('/api/rdb/table/generate/class', { @@ -109,5 +145,8 @@ export default { getActiveTaskCount, prepareUserExit, abortUserExit, + resumeTask, + getTaskArtifacts, + previewImport, generateJavaClass, }; diff --git a/chat2db-community-client/src/typings/importExport.ts b/chat2db-community-client/src/typings/importExport.ts index b47ed78e24..0468c845ef 100644 --- a/chat2db-community-client/src/typings/importExport.ts +++ b/chat2db-community-client/src/typings/importExport.ts @@ -1,9 +1,20 @@ import { IDatabaseBaseInfo } from '@/typings/database'; -import { ImportExportType, ImportExportTaskType, ImportExportTaskStatus } from '@/constants/importExport'; +import { + ImportExportFileType, + ImportExportType, + ImportExportTaskType, + ImportExportTaskStatus, +} from '@/constants/importExport'; + +export type ImportExportTargetScope = 'DATA_SOURCE' | 'DATABASE' | 'SCHEMA' | 'TABLE'; + +export type SqlExportScope = 'ALL' | 'SCHEMA' | 'TABLE'; export interface ImportExportDataBoundInfo extends IDatabaseBaseInfo { - tableName: string; + targetScope?: ImportExportTargetScope; type: ImportExportType; + fileType?: ImportExportFileType; + sqlExportScope?: SqlExportScope; } export interface ImportExportTaskDetails { @@ -23,12 +34,78 @@ export interface ImportExportTaskDetails { errorCode?: string; errorMessage?: string; artifactId?: string; + artifacts?: ITaskArtifact[]; createdAt: number | string; startedAt?: number | string; finishedAt?: number | string; updatedAt?: number | string; } +export interface ITaskArtifact { + artifactId: string; + role: string; + mediaType?: string; + sizeBytes?: number; + createdAt?: number | string; +} + +export interface IImportColumnMapping { + sourceColumn: string; + targetColumn: string; +} + +export interface IImportOptions { + charset?: string; + delimiter?: string; + quoteChar?: string; + skipRows?: number; + nullString?: string; + columnMappings?: IImportColumnMapping[]; + onError?: 'ABORT' | 'SKIP'; + maxErrors?: number; +} + +/** Execution mode of bulk import/export tasks; absent resolves to STANDARD on the backend. */ +export type ImportExecutionMode = 'ULTRA_FAST' | 'STANDARD'; + +export interface IImportAdmissionFinding { + code: string; + severity: 'BLOCKER' | 'DEGRADATION'; + message: string; + evidence?: string; + remediation?: string; +} + +export interface IImportAdmissionReport { + verdict: 'PARALLEL_SAFE' | 'PARALLEL_DEGRADED' | 'PARALLEL_FORBIDDEN'; + requestedMode: ImportExecutionMode; + effectiveMode: ImportExecutionMode; + parallelAllowed: boolean; + fileFormat: string; + fileSizeBytes: number; + dataRows: number; + fullScan: boolean; + relationshipRiskAccepted: boolean; + findings: IImportAdmissionFinding[]; +} + +export interface IImportColumnMatch { + fileColumn: string; + tableColumn?: string; + matched: boolean; +} + +export interface IImportPreview { + targetColumns?: import('@/service/sql').IImportPreview['targetColumns']; + fileColumns: string[]; + columnMatches: IImportColumnMatch[]; + missingTableColumns: string[]; + sampleRows: string[][]; + detectedCharset?: string; + detectedDelimiter?: string; + parallelAdmission?: IImportAdmissionReport; +} + export interface ImportExportTaskEvent { eventId: number; taskId: number; diff --git a/chat2db-community-server/chat2db-community-bom/pom.xml b/chat2db-community-server/chat2db-community-bom/pom.xml index ec1614adec..11a3bf8706 100644 --- a/chat2db-community-server/chat2db-community-bom/pom.xml +++ b/chat2db-community-server/chat2db-community-bom/pom.xml @@ -23,6 +23,7 @@ 1.2.0 4.13.1 4.4 + 1.10.0 32.0.1-jre 5.8.20 2.0.49 @@ -376,6 +377,11 @@ commons-collections4 ${commons-collections4.version} + + org.apache.commons + commons-csv + ${commons-csv.version} + com.google.guava guava diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/task/ArtifactDraft.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/task/ArtifactDraft.java index a6d36cb69f..029f76ee6a 100644 --- a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/task/ArtifactDraft.java +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/task/ArtifactDraft.java @@ -13,6 +13,8 @@ @AllArgsConstructor public class ArtifactDraft { + private String role; + private File temporaryFile; private File targetFile; diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/task/ImportColumnMapping.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/task/ImportColumnMapping.java index af113eba66..62d3fc7e73 100644 --- a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/task/ImportColumnMapping.java +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/task/ImportColumnMapping.java @@ -5,6 +5,9 @@ import lombok.Data; import lombok.NoArgsConstructor; +/** + * One explicit file-column to table-column mapping of an import. + */ @Data @Builder @NoArgsConstructor diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/task/ImportColumnMatch.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/task/ImportColumnMatch.java new file mode 100644 index 0000000000..796c869e4b --- /dev/null +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/task/ImportColumnMatch.java @@ -0,0 +1,22 @@ +package ai.chat2db.community.domain.api.model.task; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * One row of the import column-mapping table: a file column and the table column it resolves to. + */ +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class ImportColumnMatch { + + private String fileColumn; + + private String tableColumn; + + private boolean matched; +} diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/task/ImportOptions.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/task/ImportOptions.java new file mode 100644 index 0000000000..1d9da6dc16 --- /dev/null +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/task/ImportOptions.java @@ -0,0 +1,58 @@ +package ai.chat2db.community.domain.api.model.task; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.List; + +/** + * Optional import behaviour. Every field is optional; unset fields keep the historical defaults so + * imports submitted before this contract existed behave exactly as before. + */ +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class ImportOptions { + + /** + * Encoding of the source file; {@code null} lets the importer detect UTF-8 with BOM, strict + * UTF-8 or the platform fallback. + */ + private String charset; + + /** + * Single-character CSV delimiter; {@code null} is auto-detected from the first record. + */ + private String delimiter; + + /** + * Single-character quote, {@code "} by default. + */ + private String quoteChar; + + /** + * Data rows skipped after the header row. + */ + private Integer skipRows; + + /** + * Literal in the file that means SQL {@code NULL} (CSV), for example {@code \N}. + */ + private String nullString; + + /** + * Explicit file-column to table-column pairs; they take precedence over name matching. + */ + private List columnMappings; + + /** + * {@code ABORT} (default) stops the task on the first failed row; {@code SKIP} records the row + * in the reject artifact and continues until {@code maxErrors}. + */ + private String onError; + + private Integer maxErrors; +} diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/task/ImportPreview.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/task/ImportPreview.java new file mode 100644 index 0000000000..63482f56cc --- /dev/null +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/task/ImportPreview.java @@ -0,0 +1,37 @@ +package ai.chat2db.community.domain.api.model.task; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.List; + +/** + * Dry-run view of an import source: detected format details, the column-mapping table and the + * first data rows, produced before anything is written to the database. + */ +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class ImportPreview { + + private List targetColumns; + + private List fileColumns; + + private List columnMatches; + + private List> sampleRows; + + private String detectedCharset; + + private String detectedDelimiter; + + /** + * Table columns the source file does not supply; they will be imported as {@code NULL} or + * their defaults. + */ + private List missingTableColumns; +} diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/task/ImportTaskSpec.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/task/ImportTaskSpec.java index 983dfcf2ac..5a8b001668 100644 --- a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/task/ImportTaskSpec.java +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/task/ImportTaskSpec.java @@ -36,4 +36,13 @@ public class ImportTaskSpec implements TaskSpec { private List columnMappings; private UnmappedTargetStrategy unmappedTarget; + /** + * Optional behaviour overrides (encoding, delimiters, column mapping, error tolerance). + */ + private ImportOptions options; + + /** + * Execution mode; {@code null} resolves to {@code STANDARD}. + */ + private String mode; } diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/task/ResumeState.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/task/ResumeState.java new file mode 100644 index 0000000000..7fdd62f3ee --- /dev/null +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/task/ResumeState.java @@ -0,0 +1,31 @@ +package ai.chat2db.community.domain.api.model.task; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.Date; + +/** + * Persisted progress of one task shard, kept so an interrupted export can be resumed. A task with + * any resume state is resumable; {@code shardNo} is {@code 0} for single-cursor tasks. + */ +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class ResumeState { + + private Integer shardNo; + + private String kind; + + private String cursorJson; + + private Long rowsDone; + + private Long bytesDone; + + private Date updatedAt; +} diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/task/Task.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/task/Task.java index 775e96823a..d4dd291194 100644 --- a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/task/Task.java +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/task/Task.java @@ -6,6 +6,7 @@ import lombok.NoArgsConstructor; import java.util.Date; +import java.util.List; @Data @Builder @@ -29,12 +30,31 @@ public class Task { private TaskTargetSnapshot target; + /** + * Serialized {@code TaskSpec} captured at submission; the resume path deserializes it to + * resubmit the task without the original request. + */ + private String specJson; + private String errorCode; private String errorMessage; private String artifactId; + /** + * All published outputs of the task, including the primary one named by {@link #artifactId}. + * Filled by the storage read paths; never carried into a status patch. + */ + private List artifacts; + + /** + * Carrier for {@code FileTaskStorage}, which keeps checkpoints inside the task snapshot. + * {@code H2TaskStorage} stores them in a dedicated table and never fills this field; read them + * through {@code TaskStorage.listResumeStates}. + */ + private List resumeStates; + private Long userId; private Long organizationId; diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/task/TaskArtifact.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/task/TaskArtifact.java new file mode 100644 index 0000000000..8b195a1962 --- /dev/null +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/task/TaskArtifact.java @@ -0,0 +1,30 @@ +package ai.chat2db.community.domain.api.model.task; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.Date; + +/** + * A published output of a task. The primary download keeps {@code Task.artifactId} and is recorded + * here with role {@link TaskArtifactRole#OUTPUT}; additional products such as reject files are + * recorded as sibling rows. + */ +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class TaskArtifact { + + private String artifactId; + + private String role; + + private String mediaType; + + private Long sizeBytes; + + private Date createdAt; +} diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/task/TaskArtifactRole.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/task/TaskArtifactRole.java new file mode 100644 index 0000000000..c5027ca039 --- /dev/null +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/task/TaskArtifactRole.java @@ -0,0 +1,13 @@ +package ai.chat2db.community.domain.api.model.task; + +/** + * Roles of task artifacts. Contract values are strings so new roles can be added without a shared + * enum change; only {@link #OUTPUT} is produced today. + */ +public final class TaskArtifactRole { + + public static final String OUTPUT = "OUTPUT"; + + private TaskArtifactRole() { + } +} diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/task/TaskConstants.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/task/TaskConstants.java index 180c3bac89..07bbcb755f 100644 --- a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/task/TaskConstants.java +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/task/TaskConstants.java @@ -36,6 +36,8 @@ public final class TaskConstants { public static final String ARTIFACT_ID_DETAIL_KEY = "artifactId"; + public static final String ARTIFACT_ROLE_DETAIL_KEY = "role"; + public static final String ARTIFACT_TEMPORARY_PATH_DETAIL_KEY = "temporaryPath"; public static final String ARTIFACT_TARGET_PATH_DETAIL_KEY = "targetPath"; diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/task/TaskEventCode.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/task/TaskEventCode.java index a8d43d3453..4482bb16db 100644 --- a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/task/TaskEventCode.java +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/task/TaskEventCode.java @@ -9,6 +9,8 @@ public enum TaskEventCode { ARTIFACT_PREPARED, ARTIFACT_PUBLISHED, ARTIFACT_CLEANUP_COMPLETED, + RESUME_AVAILABLE, + TASK_RESUMED, APPLICATION_TERMINATED, USER_EXITED, EXPORT_STARTED, @@ -28,5 +30,6 @@ public enum TaskEventCode { IMPORT_PREPARING, TARGET_METADATA_LOADED, IMPORT_COMPLETED, + EXCEL_TEXT_TRUNCATED, OBJECT_SKIPPED } diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/task/TaskStage.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/task/TaskStage.java index 3f2d14ac23..ac23567834 100644 --- a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/task/TaskStage.java +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/task/TaskStage.java @@ -2,6 +2,7 @@ public enum TaskStage { PENDING, + RESUMING, STARTING, QUERYING, READING, diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/task/TaskStatusPatch.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/task/TaskStatusPatch.java index 1311a3eb47..704037f9e2 100644 --- a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/task/TaskStatusPatch.java +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/task/TaskStatusPatch.java @@ -6,6 +6,7 @@ import lombok.NoArgsConstructor; import java.util.Date; +import java.util.List; @Data @Builder @@ -25,6 +26,12 @@ public class TaskStatusPatch { private String artifactId; + /** + * Every artifact published by the transition, primary first. {@code null} leaves the stored + * artifact set untouched; when present, its first element also fills {@link #artifactId}. + */ + private List artifactIds; + private Date startedAt; private Date finishedAt; diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/service/task/ArtifactService.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/service/task/ArtifactService.java index 347f46f960..3e08889d61 100644 --- a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/service/task/ArtifactService.java +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/service/task/ArtifactService.java @@ -1,13 +1,26 @@ package ai.chat2db.community.domain.api.service.task; import ai.chat2db.community.domain.api.model.task.ArtifactDraft; +import ai.chat2db.community.domain.api.model.task.TaskArtifactRole; +import java.io.File; import java.io.IOException; import java.nio.file.Path; +import java.util.Collections; +import java.util.List; /** Manages task output files and their temporary staging paths. */ public interface ArtifactService { - ArtifactDraft createDraft(Long taskId, String outputDirectory, String fileName, String mediaType); + default ArtifactDraft createDraft(Long taskId, String outputDirectory, String fileName, String mediaType) { + return createDraft(taskId, TaskArtifactRole.OUTPUT, outputDirectory, fileName, mediaType); + } + + ArtifactDraft createDraft(Long taskId, String role, String outputDirectory, String fileName, String mediaType); + + ArtifactDraft resumeDraft(Long taskId, String role, String outputDirectory, String fileName, + String mediaType, File existingTemporaryFile); + + boolean isInterruptedDraft(Long taskId, File file); String publish(ArtifactDraft draft); @@ -18,5 +31,10 @@ public interface ArtifactService { /** Stages a file only when the destination is absent; an already staged file is left in place. */ void stageForDeletion(Path original, Path staged) throws IOException; - boolean cleanupInterruptedArtifact(Long taskId, String temporaryPath, String publishedPath); + default boolean cleanupInterruptedArtifact(Long taskId, String temporaryPath, String publishedPath) { + return cleanupInterruptedArtifacts(taskId, Collections.singletonList(temporaryPath), + Collections.singletonList(publishedPath)); + } + + boolean cleanupInterruptedArtifacts(Long taskId, List temporaryPaths, List publishedPaths); } diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/service/task/TaskExecutionContext.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/service/task/TaskExecutionContext.java index 92e656bba1..4e9e1e9dfb 100644 --- a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/service/task/TaskExecutionContext.java +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/service/task/TaskExecutionContext.java @@ -1,12 +1,22 @@ package ai.chat2db.community.domain.api.service.task; import ai.chat2db.community.domain.api.model.task.ArtifactDraft; +import ai.chat2db.community.domain.api.model.task.ResumeState; +import ai.chat2db.community.domain.api.model.task.TaskArtifactRole; import ai.chat2db.community.domain.api.service.db.ISqlExecutionStatementListener; +import java.util.List; import java.util.Map; public interface TaskExecutionContext extends ISqlExecutionStatementListener { + /** + * The task these callbacks belong to; {@code null} for contexts outside a task run. + */ + default Long taskId() { + return null; + } + void reportProgress(int progress, String stage, String message); void logInfo(String code, String message); @@ -23,5 +33,29 @@ public interface TaskExecutionContext extends ISqlExecutionStatementListener { ArtifactDraft createArtifact(String outputDirectory, String fileName, String mediaType); + /** + * Creates one draft per artifact role; the primary download uses {@code OUTPUT}. + */ + default ArtifactDraft createArtifact(String role, String outputDirectory, String fileName, String mediaType) { + if (!TaskArtifactRole.OUTPUT.equals(role)) { + throw new UnsupportedOperationException("This task context supports only the primary output artifact"); + } + return createArtifact(outputDirectory, fileName, mediaType); + } + void write(String content); + + /** + * Checkpoints persisted by earlier attempts of this task, so an exporter can resume where the + * previous run stopped. + */ + default List resumeStates() { + return List.of(); + } + + /** + * Persists one shard checkpoint (keyed by {@code ResumeState.shardNo}) for a later resume. + */ + default void checkpoint(ResumeState state) { + } } diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/service/task/TaskService.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/service/task/TaskService.java index da1fce6df0..ed34c94324 100644 --- a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/service/task/TaskService.java +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/service/task/TaskService.java @@ -2,8 +2,10 @@ import ai.chat2db.community.domain.api.model.PageResponse; import ai.chat2db.community.domain.api.model.task.ExportTaskSpec; +import ai.chat2db.community.domain.api.model.task.ImportPreview; import ai.chat2db.community.domain.api.model.task.ImportTaskSpec; import ai.chat2db.community.domain.api.model.task.Task; +import ai.chat2db.community.domain.api.model.task.TaskArtifact; import ai.chat2db.community.domain.api.model.task.TaskDownload; import ai.chat2db.community.domain.api.model.task.TaskEvent; import ai.chat2db.community.domain.api.model.task.TaskQuery; @@ -16,6 +18,20 @@ public interface TaskService { Long submitImport(ImportTaskSpec spec); + /** + * Parses the import source and resolves its columns against the target table without writing + * anything, so the UI can show the mapping and sample rows before submission. + */ + ImportPreview previewImport(ImportTaskSpec spec); + + /** + * Re-runs an interrupted task that startup reconciliation marked resumable, using the persisted + * spec and the connection context of the current request. + * + * @return the task id being resumed + */ + Long resume(Long taskId); + PageResponse list(TaskQuery query); Task get(Long taskId); @@ -33,4 +49,8 @@ public interface TaskService { void abortUserExit(); TaskDownload resolveArtifact(Long taskId); + + TaskDownload resolveArtifact(Long taskId, String artifactId); + + List listArtifacts(Long taskId); } diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/service/task/TaskStorage.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/service/task/TaskStorage.java index cbe62c6cea..ac38024b00 100644 --- a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/service/task/TaskStorage.java +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/service/task/TaskStorage.java @@ -1,7 +1,9 @@ package ai.chat2db.community.domain.api.service.task; import ai.chat2db.community.domain.api.model.PageResponse; +import ai.chat2db.community.domain.api.model.task.ResumeState; import ai.chat2db.community.domain.api.model.task.Task; +import ai.chat2db.community.domain.api.model.task.TaskArtifact; import ai.chat2db.community.domain.api.model.task.TaskEvent; import ai.chat2db.community.domain.api.model.task.TaskProgress; import ai.chat2db.community.domain.api.model.task.TaskQuery; @@ -40,4 +42,36 @@ default List listTasksForRecovery() { * Removes a terminal task while retaining enough storage state to roll back if the coordinated commit fails. */ boolean deleteTerminalTask(Long taskId, Runnable commitAction); + + /** + * Every artifact recorded for the task, primary first. Reading a task through {@link #get(Long)} + * also fills {@code Task.artifacts}; this method is the standalone lookup for list and download paths. + */ + List listArtifacts(Long taskId); + + /** + * Records one published artifact, replacing any earlier row with the same {@code artifactId}. + * The task must exist. + */ + void saveArtifact(Long taskId, TaskArtifact artifact); + + /** + * Forgets one artifact row without touching the file; used when a completion race is lost. + */ + void deleteArtifact(Long taskId, String artifactId); + + /** + * Non-terminal tasks that carry at least one persisted resume state and can therefore be resumed + * instead of being failed by startup reconciliation. + */ + List listResumableTasks(); + + /** + * Stores one shard checkpoint, replacing any earlier row for the same {@code shardNo}. The task must exist. + */ + void saveResumeState(Long taskId, ResumeState state); + + List listResumeStates(Long taskId); + + void clearResumeStates(Long taskId); } diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/pom.xml b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/pom.xml index ce1280b5f8..3d0a71ac25 100644 --- a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/pom.xml +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/pom.xml @@ -65,6 +65,10 @@ com.alibaba easyexcel + + org.apache.commons + commons-csv + org.apache.poi poi-scratchpad diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/db/TrustedMetadataRequestResolver.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/db/TrustedMetadataRequestResolver.java index a225bc06d2..04b5bfbf5a 100644 --- a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/db/TrustedMetadataRequestResolver.java +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/db/TrustedMetadataRequestResolver.java @@ -14,12 +14,12 @@ import java.util.List; import java.util.Objects; -final class TrustedMetadataRequestResolver { +public final class TrustedMetadataRequestResolver { private TrustedMetadataRequestResolver() { } - static TableMetadataRequest table(Long requestDataSourceId, String requestDatabaseName, + public static TableMetadataRequest table(Long requestDataSourceId, String requestDatabaseName, String requestSchemaName, String requestTableName) { ConnectInfo connectInfo = Chat2DBContext.getConnectInfo(); if (connectInfo == null) { diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/ArtifactServiceImpl.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/ArtifactServiceImpl.java index 354e0c5b38..35755c9118 100644 --- a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/ArtifactServiceImpl.java +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/ArtifactServiceImpl.java @@ -12,6 +12,8 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardCopyOption; +import java.util.Collections; +import java.util.List; import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; @@ -24,7 +26,8 @@ public class ArtifactServiceImpl implements ArtifactService { private final Set reservedTargets = ConcurrentHashMap.newKeySet(); @Override - public ArtifactDraft createDraft(Long taskId, String outputDirectory, String fileName, String mediaType) { + public ArtifactDraft createDraft(Long taskId, String role, String outputDirectory, String fileName, + String mediaType) { File directory = resolveDirectory(outputDirectory); if (!directory.exists() && !directory.mkdirs()) { throw new IllegalStateException("Could not create artifact directory"); @@ -34,12 +37,44 @@ public ArtifactDraft createDraft(Long taskId, String outputDirectory, String fil File temporary = new File(directory, ".task-" + taskId + "-" + UUID.randomUUID() + "-" + safeFileName + DRAFT_FILE_SUFFIX); return ArtifactDraft.builder() + .role(role) .temporaryFile(temporary) .targetFile(target) .mediaType(mediaType) .build(); } + /** + * Builds a draft around the interrupted run's temporary file, so a checkpointed export + * continues appending where it stopped instead of restarting the artifact. + */ + @Override + public ArtifactDraft resumeDraft(Long taskId, String role, String outputDirectory, String fileName, + String mediaType, File existingTemporaryFile) { + File directory = resolveDirectory(outputDirectory); + if (!directory.exists() && !directory.mkdirs()) { + throw new IllegalStateException("Could not create artifact directory"); + } + String safeFileName = safeFileName(fileName); + File target = reserveAvailableTarget(directory, safeFileName); + return ArtifactDraft.builder() + .role(role) + .temporaryFile(existingTemporaryFile) + .targetFile(target) + .mediaType(mediaType) + .build(); + } + + /** + * Whether {@code file} is a draft this application wrote for this task (the only files a + * resume may safely reopen). + */ + @Override + public boolean isInterruptedDraft(Long taskId, File file) { + String name = file.getName(); + return file.isFile() && name.startsWith(".task-" + taskId + "-") && name.endsWith(DRAFT_FILE_SUFFIX); + } + @Override public String publish(ArtifactDraft draft) { if (draft == null) { @@ -107,20 +142,36 @@ public void stageForDeletion(Path original, Path staged) throws IOException { @Override public boolean cleanupInterruptedArtifact(Long taskId, String temporaryPath, String publishedPath) { + return cleanupInterruptedArtifacts(taskId, Collections.singletonList(temporaryPath), + Collections.singletonList(publishedPath)); + } + + @Override + public boolean cleanupInterruptedArtifacts(Long taskId, List temporaryPaths, List publishedPaths) { boolean cleaned = true; - if (StringUtils.isNotBlank(temporaryPath)) { - Path temporary = Path.of(temporaryPath).toAbsolutePath().normalize(); - String fileName = temporary.getFileName() == null ? "" : temporary.getFileName().toString(); - if (fileName.startsWith(".task-" + taskId + "-") && fileName.endsWith(DRAFT_FILE_SUFFIX)) { - cleaned = deleteQuietly(temporary); - } + for (String temporaryPath : temporaryPaths) { + cleaned = cleanupInterruptedDraft(taskId, temporaryPath) && cleaned; } - if (StringUtils.isNotBlank(publishedPath)) { - cleaned = deleteQuietly(Path.of(publishedPath).toAbsolutePath().normalize()) && cleaned; + for (String publishedPath : publishedPaths) { + if (StringUtils.isNotBlank(publishedPath)) { + cleaned = deleteQuietly(Path.of(publishedPath).toAbsolutePath().normalize()) && cleaned; + } } return cleaned; } + private boolean cleanupInterruptedDraft(Long taskId, String temporaryPath) { + if (StringUtils.isBlank(temporaryPath)) { + return true; + } + Path temporary = Path.of(temporaryPath).toAbsolutePath().normalize(); + String fileName = temporary.getFileName() == null ? "" : temporary.getFileName().toString(); + if (fileName.startsWith(".task-" + taskId + "-") && fileName.endsWith(DRAFT_FILE_SUFFIX)) { + return deleteQuietly(temporary); + } + return true; + } + private File resolveDirectory(String outputDirectory) { if (StringUtils.isNotBlank(outputDirectory)) { return new File(outputDirectory); diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/LocalTaskManager.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/LocalTaskManager.java index a06e2e90e1..b0480df3c3 100644 --- a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/LocalTaskManager.java +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/LocalTaskManager.java @@ -3,6 +3,7 @@ import ai.chat2db.community.domain.api.model.task.ExportTaskSpec; import ai.chat2db.community.domain.api.model.task.ImportTaskSpec; import ai.chat2db.community.domain.api.model.task.Task; +import ai.chat2db.community.domain.api.model.task.TaskArtifact; import ai.chat2db.community.domain.api.model.task.TaskConstants; import ai.chat2db.community.domain.api.model.task.TaskErrorCode; import ai.chat2db.community.domain.api.model.task.TaskEvent; @@ -24,6 +25,7 @@ import ai.chat2db.community.domain.core.impl.task.extension.TaskExtensionManager; import ai.chat2db.community.tools.model.Context; import ai.chat2db.spi.model.datasource.ConnectInfo; +import com.alibaba.fastjson2.JSON; import jakarta.annotation.PostConstruct; import jakarta.annotation.PreDestroy; import org.springframework.beans.factory.annotation.Value; @@ -35,6 +37,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Set; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.FutureTask; import java.util.concurrent.RejectedExecutionException; @@ -43,6 +46,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.locks.ReentrantLock; +import java.util.stream.Collectors; @Component public class LocalTaskManager { @@ -85,19 +89,49 @@ public LocalTaskManager(TaskStorage taskStorage, TaskExecutorRegistry taskExecut @PostConstruct void reconcileInterruptedTasks() { + Set resumableTaskIds = taskStorage.listResumableTasks().stream() + .map(Task::getId) + .collect(Collectors.toSet()); for (Task task : taskStorage.listTasksForRecovery()) { + boolean resumable = resumableTaskIds.contains(task.getId()); if (!TaskStatus.isTerminal(task.getStatus())) { - failPersistedTask(task, TaskErrorCode.APPLICATION_TERMINATED.name(), - TaskEventCode.APPLICATION_TERMINATED.name(), - "The application terminated before the task completed"); - cleanupInterruptedArtifacts(task.getId()); + if (resumable) { + prepareResumableTask(task); + } else { + failPersistedTask(task, TaskErrorCode.APPLICATION_TERMINATED.name(), + TaskEventCode.APPLICATION_TERMINATED.name(), + "The application terminated before the task completed"); + cleanupInterruptedArtifacts(task.getId()); + } } else if (TaskStatus.FAILED.name().equals(task.getStatus()) - && isTerminationError(task.getErrorCode())) { + && isTerminationError(task.getErrorCode()) && !resumable) { cleanupInterruptedArtifacts(task.getId()); } } } + /** + * Keeps a checkpointed task alive for a later resume: a running row is requeued to PENDING with + * the RESUMING stage, a pending row only records the event, and the draft files stay in place. + */ + private void prepareResumableTask(Task task) { + TaskEvent resumeEvent = event(TaskEventCode.RESUME_AVAILABLE.name(), TaskEventLevel.INFO.name(), + "The application terminated before the task completed; the task can be resumed"); + if (TaskStatus.RUNNING.name().equals(task.getStatus())) { + Date now = new Date(); + taskStorage.compareAndSetStatus(task.getId(), TaskStatus.RUNNING.name(), TaskStatus.PENDING.name(), + TaskStatusPatch.builder() + .stage(TaskStage.RESUMING.name()) + .progressMessage("Task can be resumed") + .updatedAt(now) + .build(), + resumeEvent); + } else { + resumeEvent.setTaskId(task.getId()); + taskStorage.appendEvent(resumeEvent); + } + } + Task submit(Task task, TaskEvent createdEvent, S spec, Context context, ConnectInfo connectInfo) { lifecycleLock.lock(); @@ -105,6 +139,7 @@ Task submit(Task task, TaskEvent createdEvent, S spec, Cont if (preparingForExit) { throw new RejectedExecutionException("The application is preparing to exit"); } + task.setSpecJson(JSON.toJSONString(spec)); Task persistedTask = taskStorage.create(task, createdEvent); TaskSubmissionContext extensionContext = extensionContext(persistedTask, spec, connectInfo); try { @@ -121,6 +156,33 @@ Task submit(Task task, TaskEvent createdEvent, S spec, Cont } } + /** + * Re-runs a task that startup reconciliation kept pending because it carries resume state. The + * stored row is reused (no create), so resume checkpoints and artifact drafts from the + * interrupted run stay visible to the executor. + */ + Task resume(Task task, S spec, Context context, ConnectInfo connectInfo) { + lifecycleLock.lock(); + try { + if (preparingForExit) { + throw new RejectedExecutionException("The application is preparing to exit"); + } + if (!TaskStatus.PENDING.name().equals(task.getStatus())) { + throw new IllegalStateException("Only a pending task can be resumed"); + } + TaskSubmissionContext extensionContext = extensionContext(task, spec, connectInfo); + taskExtensionManager.capture(extensionContext); + TaskEvent resumedEvent = event(TaskEventCode.TASK_RESUMED.name(), TaskEventLevel.INFO.name(), + "Task resumed from its last checkpoint"); + resumedEvent.setTaskId(task.getId()); + taskStorage.appendEvent(resumedEvent); + schedule(task, spec, context, connectInfo, extensionContext.toExecutionContext()); + return task; + } finally { + lifecycleLock.unlock(); + } + } + void validate(TaskSpec spec) { if (spec == null || spec.getTaskType() == null) { throw new IllegalArgumentException("Task type is required"); @@ -251,8 +313,10 @@ private void cleanupInterruptedArtifacts(Long taskId) { return; } long afterSequence = 0L; - String temporaryPath = null; - String publishedPath = null; + List temporaryPaths = new ArrayList<>(); + List publishedPaths = taskStorage.listArtifacts(taskId).stream() + .map(TaskArtifact::getArtifactId) + .collect(Collectors.toCollection(ArrayList::new)); while (true) { List events = taskStorage.listEvents(taskId, afterSequence, TaskConstants.MAX_EVENT_LIMIT); if (events.isEmpty()) { @@ -261,9 +325,12 @@ private void cleanupInterruptedArtifacts(Long taskId) { for (TaskEvent event : events) { Map details = event.getDetails(); if (TaskEventCode.ARTIFACT_PREPARED.name().equals(event.getCode())) { - temporaryPath = detail(details, TaskConstants.ARTIFACT_TEMPORARY_PATH_DETAIL_KEY); + temporaryPaths.add(detail(details, TaskConstants.ARTIFACT_TEMPORARY_PATH_DETAIL_KEY)); } else if (TaskEventCode.ARTIFACT_PUBLISHED.name().equals(event.getCode())) { - publishedPath = detail(details, TaskConstants.ARTIFACT_ID_DETAIL_KEY); + String publishedPath = detail(details, TaskConstants.ARTIFACT_ID_DETAIL_KEY); + if (publishedPath != null && !publishedPaths.contains(publishedPath)) { + publishedPaths.add(publishedPath); + } } } long nextSequence = events.get(events.size() - 1).getSequence(); @@ -272,7 +339,7 @@ private void cleanupInterruptedArtifacts(Long taskId) { } afterSequence = nextSequence; } - if (artifactService.cleanupInterruptedArtifact(taskId, temporaryPath, publishedPath)) { + if (artifactService.cleanupInterruptedArtifacts(taskId, temporaryPaths, publishedPaths)) { TaskEvent cleanupEvent = event(TaskEventCode.ARTIFACT_CLEANUP_COMPLETED.name(), TaskEventLevel.INFO.name(), "Interrupted task artifacts cleaned"); cleanupEvent.setTaskId(taskId); diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/RunningTask.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/RunningTask.java index 219ce48b4b..d2af745ffd 100644 --- a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/RunningTask.java +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/RunningTask.java @@ -3,13 +3,14 @@ import ai.chat2db.community.domain.api.service.task.TaskCancelable; import lombok.extern.slf4j.Slf4j; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.locks.ReentrantLock; @Slf4j @@ -28,7 +29,8 @@ final class RunningTask { private final CancellationToken cancellationToken = new CancellationToken(); - private final AtomicReference cancelable = new AtomicReference<>(); + // Several shard workers register statements concurrently; cancellation must reach all of them. + private final Set cancelables = ConcurrentHashMap.newKeySet(); private final ReentrantLock completionLock = new ReentrantLock(); @@ -69,19 +71,24 @@ boolean requestCancellation(boolean mayInterruptIfRunning) { if (currentFuture != null) { currentFuture.cancel(mayInterruptIfRunning); } - cancelRegisteredResourceAsync(cancelable.get()); + for (TaskCancelable resource : cancelables) { + cancelRegisteredResourceAsync(resource); + } return true; } void registerCancelable(TaskCancelable resource) { - cancelable.set(resource); - if (resource != null && cancellationToken.isCancelled()) { + if (resource == null) { + return; + } + cancelables.add(resource); + if (cancellationToken.isCancelled()) { cancelRegisteredResourceAsync(resource); } } void clearCancelable(TaskCancelable resource) { - cancelable.compareAndSet(resource, null); + cancelables.remove(resource); } boolean isClosed() { @@ -90,7 +97,7 @@ boolean isClosed() { void close() { closed = true; - cancelable.set(null); + cancelables.clear(); } void markFinished() { diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/TaskExecutionContextImpl.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/TaskExecutionContextImpl.java index c37118a01a..47ffa3f77c 100644 --- a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/TaskExecutionContextImpl.java +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/TaskExecutionContextImpl.java @@ -1,6 +1,8 @@ package ai.chat2db.community.domain.core.impl.task; import ai.chat2db.community.domain.api.model.task.ArtifactDraft; +import ai.chat2db.community.domain.api.model.task.ResumeState; +import ai.chat2db.community.domain.api.model.task.TaskArtifactRole; import ai.chat2db.community.domain.api.model.task.TaskCancelledException; import ai.chat2db.community.domain.api.model.task.TaskConstants; import ai.chat2db.community.domain.api.model.task.TaskEvent; @@ -18,7 +20,12 @@ import java.nio.file.Files; import java.sql.Statement; import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.atomic.AtomicReference; final class TaskExecutionContextImpl implements TaskExecutionContext { @@ -35,9 +42,12 @@ final class TaskExecutionContextImpl implements TaskExecutionContext { private final AtomicReference activeStatement = new AtomicReference<>(); - private ArtifactDraft artifactDraft; + // Insertion order is the publish order, and the OUTPUT role stays the task's primary download. + private final Map draftsByRole = new LinkedHashMap<>(); - private BufferedWriter artifactWriter; + private final Map writersByRole = new LinkedHashMap<>(); + + private final Set appendingRoles = new java.util.HashSet<>(); TaskExecutionContextImpl(Long taskId, RunningTask runningTask, TaskStorage taskStorage, ArtifactService artifactService) { @@ -47,6 +57,22 @@ final class TaskExecutionContextImpl implements TaskExecutionContext { this.artifactService = artifactService; } + @Override + public Long taskId() { + return taskId; + } + + @Override + public List resumeStates() { + return taskStorage.listResumeStates(taskId); + } + + @Override + public void checkpoint(ResumeState state) { + checkCancelled(); + taskStorage.saveResumeState(taskId, state); + } + @Override public void reportProgress(int progress, String stage, String message) { checkCancelled(); @@ -92,20 +118,32 @@ public void registerCancelable(TaskCancelable resource) { } @Override - public synchronized ArtifactDraft createArtifact(String outputDirectory, String fileName, String mediaType) { + public ArtifactDraft createArtifact(String outputDirectory, String fileName, String mediaType) { + return createArtifact(TaskArtifactRole.OUTPUT, outputDirectory, fileName, mediaType); + } + + @Override + public synchronized ArtifactDraft createArtifact(String role, String outputDirectory, String fileName, + String mediaType) { checkCancelled(); - if (artifactDraft != null) { - throw new IllegalStateException("A task can create at most one artifact"); + if (draftsByRole.containsKey(role)) { + throw new IllegalStateException("Artifact role " + role + " is already created for this task"); + } + ArtifactDraft draft = resumedDraft(role, outputDirectory, fileName, mediaType); + if (draft == null) { + draft = artifactService.createDraft(taskId, role, outputDirectory, fileName, mediaType); + } else { + appendingRoles.add(role); } - ArtifactDraft draft = artifactService.createDraft(taskId, outputDirectory, fileName, mediaType); try { appendEvent(TaskEventLevel.INFO.name(), TaskEventCode.ARTIFACT_PREPARED.name(), "Artifact prepared", Map.of( TaskConstants.ARTIFACT_TEMPORARY_PATH_DETAIL_KEY, draft.getTemporaryFile().getAbsolutePath(), TaskConstants.ARTIFACT_TARGET_PATH_DETAIL_KEY, - draft.getTargetFile().getAbsolutePath())); - artifactDraft = draft; + draft.getTargetFile().getAbsolutePath(), + TaskConstants.ARTIFACT_ROLE_DETAIL_KEY, role)); + draftsByRole.put(role, draft); return draft; } catch (RuntimeException e) { artifactService.deleteDraft(draft); @@ -113,19 +151,54 @@ public synchronized ArtifactDraft createArtifact(String outputDirectory, String } } + /** + * The draft file of an interrupted run, if this task carries resume checkpoints and the file + * survived; the exporter then appends instead of restarting. + */ + private ArtifactDraft resumedDraft(String role, String outputDirectory, String fileName, String mediaType) { + if (taskStorage.listResumeStates(taskId).isEmpty()) { + return null; + } + String temporaryPath = null; + for (TaskEvent prepared : taskStorage.listEvents(taskId, 0L, TaskConstants.MAX_EVENT_LIMIT)) { + if (TaskEventCode.ARTIFACT_PREPARED.name().equals(prepared.getCode()) + && role.equals(detailOf(prepared, TaskConstants.ARTIFACT_ROLE_DETAIL_KEY))) { + temporaryPath = detailOf(prepared, TaskConstants.ARTIFACT_TEMPORARY_PATH_DETAIL_KEY); + } + } + if (temporaryPath == null) { + return null; + } + java.io.File existing = new java.io.File(temporaryPath); + if (!artifactService.isInterruptedDraft(taskId, existing)) { + return null; + } + return artifactService.resumeDraft(taskId, role, outputDirectory, fileName, mediaType, existing); + } + + private static String detailOf(TaskEvent event, String key) { + Object value = event.getDetails() == null ? null : event.getDetails().get(key); + return value == null ? null : String.valueOf(value); + } + @Override public synchronized void write(String content) { checkCancelled(); - if (artifactDraft == null) { + ArtifactDraft draft = draftsByRole.get(TaskArtifactRole.OUTPUT); + if (draft == null) { throw new IllegalStateException("Create an artifact before writing content"); } + BufferedWriter writer = writersByRole.get(TaskArtifactRole.OUTPUT); try { - if (artifactWriter == null) { - artifactWriter = Files.newBufferedWriter(artifactDraft.getTemporaryFile().toPath(), - StandardCharsets.UTF_8); + if (writer == null) { + writer = appendingRoles.contains(TaskArtifactRole.OUTPUT) + ? Files.newBufferedWriter(draft.getTemporaryFile().toPath(), StandardCharsets.UTF_8, + java.nio.file.StandardOpenOption.APPEND) + : Files.newBufferedWriter(draft.getTemporaryFile().toPath(), StandardCharsets.UTF_8); + writersByRole.put(TaskArtifactRole.OUTPUT, writer); } - artifactWriter.write(content); - artifactWriter.newLine(); + writer.write(content); + writer.newLine(); } catch (IOException e) { throw new IllegalStateException("Could not write task artifact", e); } @@ -157,34 +230,34 @@ public void onStatementClosed(Statement statement) { } } + synchronized List artifactDrafts() { + return List.copyOf(draftsByRole.values()); + } + synchronized void finishArtifactWrites() { - if (artifactWriter == null) { - return; - } - try { - artifactWriter.flush(); - artifactWriter.close(); - artifactWriter = null; - } catch (IOException e) { - throw new IllegalStateException("Could not close task artifact", e); + Iterator> entries = writersByRole.entrySet().iterator(); + while (entries.hasNext()) { + Map.Entry entry = entries.next(); + try { + entry.getValue().flush(); + entry.getValue().close(); + } catch (IOException e) { + throw new IllegalStateException("Could not close task artifact", e); + } finally { + entries.remove(); + } } } synchronized void closeQuietly() { - if (artifactWriter == null) { - return; - } - try { - artifactWriter.close(); - } catch (IOException ignored) { - // The task result has already been decided. - } finally { - artifactWriter = null; + for (BufferedWriter writer : writersByRole.values()) { + try { + writer.close(); + } catch (IOException ignored) { + // The task result has already been decided. + } } - } - - ArtifactDraft artifactDraft() { - return artifactDraft; + writersByRole.clear(); } private void appendEvent(String level, String code, String message, Map details) { diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/TaskRunner.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/TaskRunner.java index 02b41ddac5..382f19f145 100644 --- a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/TaskRunner.java +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/TaskRunner.java @@ -1,6 +1,8 @@ package ai.chat2db.community.domain.core.impl.task; import ai.chat2db.community.domain.api.model.task.ArtifactDraft; +import ai.chat2db.community.domain.api.model.task.TaskArtifact; +import ai.chat2db.community.domain.api.model.task.TaskArtifactRole; import ai.chat2db.community.domain.api.model.task.TaskCancelledException; import ai.chat2db.community.domain.api.model.task.TaskConstants; import ai.chat2db.community.domain.api.model.task.TaskErrorCode; @@ -21,9 +23,12 @@ import lombok.extern.slf4j.Slf4j; import org.slf4j.MDC; +import java.io.File; +import java.util.ArrayList; import java.util.Collections; import java.util.Date; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import java.util.concurrent.CancellationException; @@ -71,21 +76,21 @@ public void run() { taskExecutor.execute(submission.spec(), executionContext); } }); - ArtifactDraft draft = executionContext.artifactDraft(); + List drafts = executionContext.artifactDrafts(); executionContext.finishArtifactWrites(); - logArtifactWritten(executionContext, draft); - completeSuccessfully(draft); + logArtifactWritten(executionContext, drafts); + completeSuccessfully(drafts); } catch (TaskCancelledException | CancellationException e) { - completeCancelled(executionContext.artifactDraft()); + completeCancelled(executionContext.artifactDrafts()); } catch (TaskExecutionException e) { completeFailed(e.getCode(), e.publicMessage(), e.getSafeReason(), e, - executionContext.artifactDraft()); + executionContext.artifactDrafts()); } catch (Throwable e) { if (runningTask.cancellationToken().isCancelled() || Thread.currentThread().isInterrupted()) { - completeCancelled(executionContext.artifactDraft()); + completeCancelled(executionContext.artifactDrafts()); } else { completeFailed(TaskErrorCode.TASK_INTERNAL_ERROR.name(), "Task execution failed", null, e, - executionContext.artifactDraft()); + executionContext.artifactDrafts()); } } finally { try { @@ -99,19 +104,24 @@ public void run() { } } - private void logArtifactWritten(TaskExecutionContextImpl executionContext, ArtifactDraft draft) { - if (draft == null) { + private void logArtifactWritten(TaskExecutionContextImpl executionContext, List drafts) { + if (drafts.isEmpty()) { return; } executionContext.reportProgress(95, TaskStage.FINALIZING.name(), "Export file written"); - Map details = new LinkedHashMap<>(); - if (draft.getTargetFile() != null) { - details.put(TaskConstants.FILE_NAME_DETAIL_KEY, draft.getTargetFile().getName()); - } - if (draft.getMediaType() != null) { - details.put("mediaType", draft.getMediaType()); + for (ArtifactDraft draft : drafts) { + Map details = new LinkedHashMap<>(); + if (draft.getTargetFile() != null) { + details.put(TaskConstants.FILE_NAME_DETAIL_KEY, draft.getTargetFile().getName()); + } + if (draft.getMediaType() != null) { + details.put("mediaType", draft.getMediaType()); + } + if (draft.getRole() != null) { + details.put(TaskConstants.ARTIFACT_ROLE_DETAIL_KEY, draft.getRole()); + } + executionContext.logInfo(TaskEventCode.FILE_WRITE_COMPLETED.name(), "Export file written", details); } - executionContext.logInfo(TaskEventCode.FILE_WRITE_COMPLETED.name(), "Export file written", details); } private boolean startTask() { @@ -136,23 +146,36 @@ private boolean startTask() { } } - private void completeSuccessfully(ArtifactDraft draft) { + private void completeSuccessfully(List drafts) { runningTask.completionLock().lock(); - String artifactId = null; + List published = new ArrayList<>(); try { if (runningTask.cancellationToken().isCancelled()) { - completeCancelledLocked(draft); + completeCancelledLocked(drafts); return; } - if (draft != null) { - artifactId = artifactService.publish(draft); + String primaryArtifactId = null; + for (ArtifactDraft draft : drafts) { + String artifactId = artifactService.publish(draft); + published.add(artifactId); + if (primaryArtifactId == null || TaskArtifactRole.OUTPUT.equals(draft.getRole())) { + primaryArtifactId = artifactId; + } + taskStorage.saveArtifact(submission.taskId(), TaskArtifact.builder() + .artifactId(artifactId) + .role(draft.getRole()) + .mediaType(draft.getMediaType()) + .sizeBytes(new File(artifactId).length()) + .createdAt(new Date()) + .build()); taskStorage.appendEvent(TaskEvent.builder() .taskId(submission.taskId()) .level(TaskEventLevel.INFO.name()) .code(TaskEventCode.ARTIFACT_PUBLISHED.name()) .stage(TaskStage.FINALIZING.name()) .message("Artifact published") - .details(Map.of(TaskConstants.ARTIFACT_ID_DETAIL_KEY, artifactId)) + .details(Map.of(TaskConstants.ARTIFACT_ID_DETAIL_KEY, artifactId, + TaskConstants.ARTIFACT_ROLE_DETAIL_KEY, String.valueOf(draft.getRole()))) .build()); } Date now = new Date(); @@ -162,47 +185,59 @@ private void completeSuccessfully(ArtifactDraft draft) { .progress(TaskConstants.COMPLETED_PROGRESS) .stage(TaskStage.COMPLETED.name()) .progressMessage("Task completed successfully") - .artifactId(artifactId) + .artifactId(primaryArtifactId) + .artifactIds(published.isEmpty() ? null : List.copyOf(published)) .finishedAt(now) .updatedAt(now) .build(), lifecycleEvent(TaskEventCode.TASK_SUCCEEDED.name(), TaskEventLevel.INFO.name(), "Task completed successfully")); - if (!completed && artifactId != null) { - artifactService.deletePublished(artifactId); + if (!completed) { + rollbackPublishedArtifacts(published); } } catch (Throwable e) { - if (artifactId != null) { - artifactService.deletePublished(artifactId); - } + rollbackPublishedArtifacts(published); if (runningTask.cancellationToken().isCancelled()) { - completeCancelledLocked(draft); + completeCancelledLocked(drafts); } else { completeFailedLocked(TaskErrorCode.ARTIFACT_PUBLISH_FAILED.name(), - "Could not publish task artifact", null, e, draft); + "Could not publish task artifact", null, e, drafts); } } finally { runningTask.completionLock().unlock(); } } + /** + * A lost completion race or a publish failure must not leave orphan files or artifact rows + * behind, so every already-published output is undone in reverse order. + */ + private void rollbackPublishedArtifacts(List publishedArtifactIds) { + for (int index = publishedArtifactIds.size() - 1; index >= 0; index--) { + artifactService.deletePublished(publishedArtifactIds.get(index)); + taskStorage.deleteArtifact(submission.taskId(), publishedArtifactIds.get(index)); + } + } + private void completeFailed(String code, String message, String safeReason, Throwable cause, - ArtifactDraft draft) { + List drafts) { runningTask.completionLock().lock(); try { if (runningTask.cancellationToken().isCancelled()) { - completeCancelledLocked(draft); + completeCancelledLocked(drafts); return; } - completeFailedLocked(code, message, safeReason, cause, draft); + completeFailedLocked(code, message, safeReason, cause, drafts); } finally { runningTask.completionLock().unlock(); } } private void completeFailedLocked(String code, String message, String safeReason, Throwable cause, - ArtifactDraft draft) { - artifactService.deleteDraft(draft); + List drafts) { + for (ArtifactDraft draft : drafts) { + artifactService.deleteDraft(draft); + } log.error("Task {} failed", submission.taskId(), cause); Date now = new Date(); taskStorage.compareAndSetStatus(submission.taskId(), TaskStatus.RUNNING.name(), TaskStatus.FAILED.name(), @@ -234,17 +269,19 @@ private Map failureDetails(String code, String safeReason) { return details; } - private void completeCancelled(ArtifactDraft draft) { + private void completeCancelled(List drafts) { runningTask.completionLock().lock(); try { - completeCancelledLocked(draft); + completeCancelledLocked(drafts); } finally { runningTask.completionLock().unlock(); } } - private void completeCancelledLocked(ArtifactDraft draft) { - artifactService.deleteDraft(draft); + private void completeCancelledLocked(List drafts) { + for (ArtifactDraft draft : drafts) { + artifactService.deleteDraft(draft); + } } private TaskEvent lifecycleEvent(String code, String level, String message) { diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/TaskServiceImpl.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/TaskServiceImpl.java index 67717235d0..9ee503a2e9 100644 --- a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/TaskServiceImpl.java +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/TaskServiceImpl.java @@ -1,9 +1,13 @@ package ai.chat2db.community.domain.core.impl.task; import ai.chat2db.community.domain.api.model.PageResponse; +import ai.chat2db.community.domain.api.model.metadata.TableColumn; +import ai.chat2db.community.domain.api.model.request.runtime.DbConnectionContextRequest; import ai.chat2db.community.domain.api.model.task.ExportTaskSpec; +import ai.chat2db.community.domain.api.model.task.ImportPreview; import ai.chat2db.community.domain.api.model.task.ImportTaskSpec; import ai.chat2db.community.domain.api.model.task.Task; +import ai.chat2db.community.domain.api.model.task.TaskArtifact; import ai.chat2db.community.domain.api.model.task.TaskConstants; import ai.chat2db.community.domain.api.model.task.TaskDownload; import ai.chat2db.community.domain.api.model.task.TaskEvent; @@ -13,37 +17,90 @@ import ai.chat2db.community.domain.api.model.task.TaskSpec; import ai.chat2db.community.domain.api.model.task.TaskStage; import ai.chat2db.community.domain.api.model.task.TaskStatus; +import ai.chat2db.community.domain.api.model.task.TaskTargetSnapshot; +import ai.chat2db.community.domain.api.model.task.TaskType; +import ai.chat2db.community.domain.api.service.db.IDbConnectionContextService; +import ai.chat2db.community.domain.api.service.file.IImportFileStagingService; +import ai.chat2db.community.domain.api.service.task.ArtifactService; import ai.chat2db.community.domain.api.service.task.TaskDeletionService; import ai.chat2db.community.domain.api.service.task.TaskService; import ai.chat2db.community.domain.api.service.task.TaskStorage; +import ai.chat2db.community.domain.core.impl.task.imports.ImportColumnResolver; +import ai.chat2db.community.domain.core.impl.task.imports.ImportFileProbe; +import ai.chat2db.community.domain.core.impl.task.imports.excel.ImportPreviewListener; +import com.alibaba.excel.EasyExcel; +import com.alibaba.excel.support.ExcelTypeEnum; +import org.apache.commons.csv.CSVFormat; import ai.chat2db.community.tools.exception.BusinessException; import ai.chat2db.community.tools.exception.DataNotFoundException; import ai.chat2db.community.tools.model.Context; import ai.chat2db.community.tools.util.ContextUtils; import ai.chat2db.spi.model.datasource.ConnectInfo; +import ai.chat2db.spi.model.request.TableMetadataRequest; import ai.chat2db.spi.sql.Chat2DBContext; import jakarta.annotation.PostConstruct; +import com.alibaba.fastjson2.JSON; import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import java.io.File; +import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Objects; +import java.util.function.Function; @Service public class TaskServiceImpl implements TaskService { + @org.springframework.beans.factory.annotation.Value("${chat2db.task.import.allowed-roots:}") + private String importAllowedRoots; private final TaskStorage taskStorage; private final LocalTaskManager localTaskManager; private final TaskDeletionService deletionService; - public TaskServiceImpl(TaskStorage taskStorage, LocalTaskManager localTaskManager, TaskDeletionService deletionService) { + private final IDbConnectionContextService connectionContextService; + + private final IImportFileStagingService importFileStagingService; + + @Autowired + public TaskServiceImpl(TaskStorage taskStorage, LocalTaskManager localTaskManager, + TaskDeletionService deletionService, IDbConnectionContextService connectionContextService, + IImportFileStagingService importFileStagingService) { this.taskStorage = taskStorage; this.localTaskManager = localTaskManager; this.deletionService = deletionService; + this.connectionContextService = connectionContextService; + this.importFileStagingService = importFileStagingService; + } + + public TaskServiceImpl(TaskStorage taskStorage, LocalTaskManager localTaskManager, + TaskDeletionService deletionService, IDbConnectionContextService connectionContextService) { + this(taskStorage, localTaskManager, deletionService, connectionContextService, null); + } + + TaskServiceImpl(TaskStorage taskStorage, LocalTaskManager localTaskManager, + TaskDeletionService deletionService) { + this(taskStorage, localTaskManager, deletionService, null, null); + } + + public TaskServiceImpl(TaskStorage taskStorage, LocalTaskManager localTaskManager, + ArtifactService artifactService, IDbConnectionContextService connectionContextService, + IImportFileStagingService importFileStagingService) { + this(taskStorage, localTaskManager, new TaskDeletionServiceImpl(taskStorage, artifactService), + connectionContextService, importFileStagingService); + } + + public TaskServiceImpl(TaskStorage taskStorage, LocalTaskManager localTaskManager, + ArtifactService artifactService, IDbConnectionContextService connectionContextService) { + this(taskStorage, localTaskManager, artifactService, connectionContextService, null); + } + + TaskServiceImpl(TaskStorage taskStorage, LocalTaskManager localTaskManager, ArtifactService artifactService) { + this(taskStorage, localTaskManager, artifactService, null, null); } @PostConstruct @@ -58,9 +115,192 @@ public Long submitExport(ExportTaskSpec spec) { @Override public Long submitImport(ImportTaskSpec spec) { + validateImportSource(spec.getSourceFile()); return submit(spec); } + @Override + public ImportPreview previewImport(ImportTaskSpec spec) { + if (StringUtils.isNotBlank(spec.getImportFileId())) { + spec.setSourceFile(importFileStagingService.resolve(spec.getImportFileId()).getAbsolutePath()); + } + validateImportSource(spec.getSourceFile()); + java.io.File source = new java.io.File(StringUtils.defaultString(spec.getSourceFile())); + if (!source.isFile() || !source.canRead()) { + throw new BusinessException("task.import.preview.sourceUnreadable", null); + } + String format = StringUtils.upperCase(StringUtils.trimToEmpty(spec.getFormat()), + java.util.Locale.ROOT); + List tableColumns = loadTableColumns(spec); + return switch (format) { + case "CSV" -> previewCsv(source, spec, tableColumns); + case "XLSX", "XLS" -> previewExcel(source, spec, tableColumns, format); + default -> throw new BusinessException("task.import.preview.unsupportedFormat", null); + }; + } + + private List loadTableColumns(ImportTaskSpec spec) { + return withTargetConnection(spec, connectInfo -> { + TaskTargetSnapshot target = spec.getTarget(); + TableMetadataRequest trusted = ai.chat2db.community.domain.core.impl.db.TrustedMetadataRequestResolver + .table(target.getDataSourceId(), target.getDatabaseName(), target.getSchemaName(), + target.getTableName()); + return Chat2DBContext.getDbMetaData().columns(Chat2DBContext.getConnection(), trusted); + }); + } + + private ImportPreview previewCsv(java.io.File source, ImportTaskSpec spec, List tableColumns) { + try { + java.nio.charset.Charset charset = ImportFileProbe.effectiveCharset(source, + spec.getOptions() == null ? null : spec.getOptions().getCharset()); + char quote = ImportFileProbe.quoteChar( + spec.getOptions() == null ? null : spec.getOptions().getQuoteChar()); + char delimiter = ImportFileProbe.delimiterChar( + spec.getOptions() == null ? null : spec.getOptions().getDelimiter(), charset, source); + CSVFormat format = ImportFileProbe.csvFormat(delimiter, quote); + List> rows = ImportFileProbe.readSample(source, charset, format, + ImportFileProbe.sampleRows()); + return buildPreview(rows, tableColumns, spec, charset.name(), String.valueOf(delimiter)); + } catch (java.io.IOException e) { + throw new BusinessException("task.import.preview.failed", null, e); + } + } + + private ImportPreview previewExcel(java.io.File source, ImportTaskSpec spec, List tableColumns, + String format) { + ImportPreviewListener listener = new ImportPreviewListener(); + EasyExcel.read(source, listener) + .excelType("XLS".equals(format) ? ExcelTypeEnum.XLS : ExcelTypeEnum.XLSX) + .sheet() + .headRowNumber(1) + .doRead(); + return buildPreview(listener.rows(), tableColumns, spec, null, null); + } + + private ImportPreview buildPreview(List> rows, List tableColumns, + ImportTaskSpec spec, String detectedCharset, String detectedDelimiter) { + List headers = rows.isEmpty() ? List.of() : rows.get(0); + ImportColumnResolver.Resolution resolution = + ImportColumnResolver.resolve(tableColumns, headers, spec.getOptions()); + return ImportPreview.builder() + .targetColumns(tableColumns.stream().map(column -> + ai.chat2db.community.domain.api.model.db.ImportTargetColumn.builder() + .name(column.getName()).dataType(column.getColumnType()) + .nullable(Integer.valueOf(1).equals(column.getNullable())) + .autoIncrement(Boolean.TRUE.equals(column.getAutoIncrement())) + .defaultValue(column.getDefaultValue()).comment(column.getComment()).build()).toList()) + .fileColumns(headers) + .columnMatches(resolution.matches()) + .missingTableColumns(resolution.missingTableColumns()) + .sampleRows(rows.size() <= 1 ? List.of() + : rows.subList(1, rows.size()).stream().map(row -> (List) row).toList()) + .detectedCharset(detectedCharset) + .detectedDelimiter(detectedDelimiter) + .build(); + } + + @Override + public Long resume(Long taskId) { + Task task = get(taskId); + if (task == null || !TaskStatus.PENDING.name().equals(task.getStatus()) + || StringUtils.isBlank(task.getSpecJson()) + || taskStorage.listResumeStates(taskId).isEmpty()) { + // Only a task that startup reconciliation left pending with checkpoints is resumable; + // anything else is reported like a missing task. + throw new DataNotFoundException(); + } + TaskSpec spec = parseSpec(task); + localTaskManager.validate(spec); + Context context = ContextUtils.queryContext(); + ConnectInfo connectInfo = resumeConnectInfo(spec); + try { + localTaskManager.resume(task, spec, context, connectInfo); + } catch (IllegalStateException | java.util.concurrent.RejectedExecutionException race) { + // A double resume or an exit-in-progress race must surface as a client error, not a 500. + throw new BusinessException("task.resume.conflict", null, race); + } + return taskId; + } + + /** + * Rebuilds the connection from the persisted task target. A resume request intentionally needs + * only the task id and must not depend on whichever data source happens to be selected in the + * client after a restart. + */ + ConnectInfo resumeConnectInfo(TaskSpec spec) { + return withTargetConnection(spec, ConnectInfo::copy); + } + + private T withTargetConnection(TaskSpec spec, Function action) { + TaskTargetSnapshot target = spec == null ? null : spec.getTarget(); + if (target == null || target.getDataSourceId() == null || connectionContextService == null) { + throw new BusinessException("datasource.not.found"); + } + DbConnectionContextRequest request = new DbConnectionContextRequest(); + request.setDataSourceId(target.getDataSourceId()); + request.setDatabaseName(target.getDatabaseName()); + request.setSchemaName(target.getSchemaName()); + ConnectInfo previous = Chat2DBContext.getConnectInfo(); + try { + connectionContextService.bind(request); + ConnectInfo resolved = Chat2DBContext.getConnectInfo(); + if (resolved == null) { + throw new BusinessException("datasource.not.found"); + } + return action.apply(resolved); + } finally { + try { + connectionContextService.clear(); + } finally { + if (previous == null) { + Chat2DBContext.removeContext(); + } else { + Chat2DBContext.putContext(previous); + } + } + } + } + + /** + * Server deployments can restrict which directories import files may come from; desktop runs + * keep the unrestricted default. Paths are compared normalized and absolute so `..` segments + * cannot escape the allowlist. + */ + private void validateImportSource(String sourceFile) { + if (StringUtils.isBlank(importAllowedRoots) || StringUtils.isBlank(sourceFile)) { + return; + } + java.nio.file.Path candidate; + try { + candidate = java.nio.file.Path.of(sourceFile).toAbsolutePath().normalize().toRealPath(); + } catch (java.nio.file.InvalidPathException | java.io.IOException invalidPath) { + throw new BusinessException("task.import.sourceNotAllowed", null); + } + for (String root : importAllowedRoots.split(",")) { + if (StringUtils.isBlank(root)) { + continue; + } + try { + java.nio.file.Path allowedRoot = java.nio.file.Path.of(root.trim()).toAbsolutePath() + .normalize().toRealPath(); + if (candidate.startsWith(allowedRoot)) { + return; + } + } catch (java.nio.file.InvalidPathException | java.io.IOException ignored) { + // An invalid configured root cannot authorize access to any source path. + } + } + throw new BusinessException("task.import.sourceNotAllowed", null); + } + + private TaskSpec parseSpec(Task task) { + String type = task.getType(); + if (TaskType.DATA_FILE_IMPORT.name().equals(type) || TaskType.SQL_FILE_IMPORT.name().equals(type)) { + return JSON.parseObject(task.getSpecJson(), ImportTaskSpec.class); + } + return JSON.parseObject(task.getSpecJson(), ExportTaskSpec.class); + } + @Override public PageResponse list(TaskQuery query) { TaskOwner owner = currentOwner(); @@ -129,10 +369,34 @@ public TaskDownload resolveArtifact(Long taskId) { throw new DataNotFoundException(); } File file = deletionService.resolveArtifact(task); + return downloadFor(file, new File(task.getArtifactId()).getName()); + } + + @Override + public TaskDownload resolveArtifact(Long taskId, String artifactId) { + Task task = get(taskId); + if (task == null || !TaskStatus.SUCCESS.name().equals(task.getStatus())) { + throw new DataNotFoundException(); + } + // The parameter is only a lookup key; the served path always comes from the stored row, so + // a caller cannot name an arbitrary file. + TaskArtifact artifact = taskStorage.listArtifacts(taskId).stream() + .filter(candidate -> candidate.getArtifactId().equals(artifactId)) + .findFirst() + .orElseThrow(DataNotFoundException::new); + return downloadFor(new File(artifact.getArtifactId()), new File(artifact.getArtifactId()).getName()); + } + + @Override + public List listArtifacts(Long taskId) { + return get(taskId) == null ? List.of() : taskStorage.listArtifacts(taskId); + } + + private TaskDownload downloadFor(File file, String fileName) { if (!file.isFile() || !file.canRead()) { throw new DataNotFoundException(); } - return TaskDownload.builder().fileName(new File(task.getArtifactId()).getName()) + return TaskDownload.builder().fileName(fileName) .fileUri(file.toURI().toString()).build(); } diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/ImportColumnResolver.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/ImportColumnResolver.java new file mode 100644 index 0000000000..090e6a9337 --- /dev/null +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/ImportColumnResolver.java @@ -0,0 +1,115 @@ +package ai.chat2db.community.domain.core.impl.task.imports; + +import ai.chat2db.community.domain.api.model.metadata.TableColumn; +import ai.chat2db.community.domain.api.model.task.ImportColumnMapping; +import ai.chat2db.community.domain.api.model.task.ImportColumnMatch; +import ai.chat2db.community.domain.api.model.task.ImportOptions; +import ai.chat2db.community.tools.exception.ParamBusinessException; +import org.apache.commons.lang3.StringUtils; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * Resolves which file column feeds which table column. Explicit mappings win; otherwise matching is + * case-insensitive on trimmed names. Unmatched file columns are reported instead of silently + * dropping data as the old upper-case-equality rule did. + */ +public final class ImportColumnResolver { + + /** + * Ordered pair lists: entry {@code i} binds {@code fileValues[fileIndexes[i]]} to + * {@code tableColumns[i]}. + */ + public record Resolution(List tableColumns, List fileIndexes, + List matches, List missingTableColumns) { + } + + private ImportColumnResolver() { + } + + public static Resolution resolve(List tableColumns, List fileHeaders, + ImportOptions options) { + Map byNormalizedName = new LinkedHashMap<>(); + for (int index = 0; index < fileHeaders.size(); index++) { + byNormalizedName.putIfAbsent(normalize(fileHeaders.get(index)), index); + } + Map explicitTargets = new LinkedHashMap<>(); + if (options != null && options.getColumnMappings() != null) { + for (ImportColumnMapping mapping : options.getColumnMappings()) { + if (mapping == null || StringUtils.isBlank(mapping.getSourceColumn()) + || StringUtils.isBlank(mapping.getTargetColumn())) { + throw new ParamBusinessException("columnMappings"); + } + Integer sourceIndex = indexOfSource(mapping.getSourceColumn().trim(), fileHeaders, + byNormalizedName); + if (sourceIndex == null) { + throw new ParamBusinessException("columnMappings source: " + mapping.getSourceColumn()); + } + explicitTargets.put(normalize(mapping.getTargetColumn()), sourceIndex); + } + } + + List resolvedColumns = new ArrayList<>(); + List fileIndexes = new ArrayList<>(); + List missingTableColumns = new ArrayList<>(); + for (TableColumn column : tableColumns) { + Integer sourceIndex = explicitTargets.get(normalize(column.getName())); + if (sourceIndex == null) { + sourceIndex = byNormalizedName.get(normalize(column.getName())); + } + if (sourceIndex != null) { + resolvedColumns.add(column); + fileIndexes.add(sourceIndex); + } else { + missingTableColumns.add(column.getName()); + } + } + + java.util.Set usedFileIndexes = new java.util.HashSet<>(fileIndexes); + List matches = new ArrayList<>(fileHeaders.size()); + for (int index = 0; index < fileHeaders.size(); index++) { + String tableColumn = null; + for (int resolved = 0; resolved < fileIndexes.size(); resolved++) { + if (fileIndexes.get(resolved) == index) { + tableColumn = resolvedColumns.get(resolved).getName(); + break; + } + } + matches.add(ImportColumnMatch.builder() + .fileColumn(fileHeaders.get(index)) + .tableColumn(tableColumn) + .matched(usedFileIndexes.contains(index)) + .build()); + } + return new Resolution(resolvedColumns, fileIndexes, matches, missingTableColumns); + } + + private static Integer indexOfSource(String source, List fileHeaders, + Map byNormalizedName) { + try { + int index = Integer.parseInt(source); + return index >= 0 && index < fileHeaders.size() ? index : null; + } catch (NumberFormatException ignored) { + return byNormalizedName.get(normalize(source)); + } + } + + /** + * Case-insensitive match on trimmed names, ignoring a leading UTF-8 BOM: commons-csv does not + * strip it, and without this the first column of every BOM-prefixed file (including files + * written by our own CsvSink) would never match. + */ + private static String normalize(String name) { + if (name == null) { + return ""; + } + String trimmed = name; + if (!trimmed.isEmpty() && trimmed.charAt(0) == '\ufeff') { + trimmed = trimmed.substring(1); + } + return trimmed.trim().toLowerCase(java.util.Locale.ROOT); + } +} diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/ImportFileProbe.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/ImportFileProbe.java new file mode 100644 index 0000000000..fe6c547893 --- /dev/null +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/ImportFileProbe.java @@ -0,0 +1,147 @@ +package ai.chat2db.community.domain.core.impl.task.imports; + +import cn.hutool.core.io.CharsetDetector; +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVParser; +import org.apache.commons.csv.CSVRecord; +import org.apache.commons.lang3.StringUtils; + +import java.io.File; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.Reader; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.List; + +/** + * Format detection for import sources. The charset detector is the same one the desktop text + * engine uses, so the preview and the actual import always agree. + */ +public final class ImportFileProbe { + + private static final char[] DELIMITERS = {',', ';', '\t', '|'}; + + private static final int SAMPLE_ROWS = 20; + + private ImportFileProbe() { + } + + public static Charset detectCharset(File file) { + Charset detected = CharsetDetector.detect(file, StandardCharsets.UTF_8, + Charset.forName("GBK"), StandardCharsets.ISO_8859_1); + return detected == null ? StandardCharsets.UTF_8 : detected; + } + + public static Charset effectiveCharset(File file, String requested) { + if (StringUtils.isBlank(requested)) { + return detectCharset(file); + } + try { + return Charset.forName(requested.trim()); + } catch (RuntimeException unsupportedCharset) { + // IllegalArgumentException for unknown names, IllegalCharsetNameException/" + // UnsupportedCharsetException otherwise; both are caller input errors. + throw new ai.chat2db.community.tools.exception.ParamBusinessException( + "Invalid import charset: " + requested); + } + } + + /** + * The candidate delimiter appearing most often outside quoted sections of the first line. + */ + public static char detectDelimiter(String firstLine, char quote) { + if (firstLine == null) { + return ','; + } + char best = ','; + int bestCount = -1; + boolean inQuotes = false; + int[] counts = new int[DELIMITERS.length]; + for (int index = 0; index < firstLine.length(); index++) { + char current = firstLine.charAt(index); + if (current == quote) { + inQuotes = !inQuotes; + } else if (!inQuotes) { + for (int candidate = 0; candidate < DELIMITERS.length; candidate++) { + if (current == DELIMITERS[candidate]) { + counts[candidate]++; + } + } + } + } + for (int candidate = 0; candidate < DELIMITERS.length; candidate++) { + if (counts[candidate] > bestCount) { + bestCount = counts[candidate]; + best = DELIMITERS[candidate]; + } + } + return best; + } + + /** + * CSV grammar settings; the charset is passed to {@link #openParser} separately because + * commons-csv keeps encoding on the reader, not the format. + */ + public static CSVFormat csvFormat(char delimiter, char quote) { + return CSVFormat.Builder.create(CSVFormat.EXCEL) + .setDelimiter(delimiter) + .setQuote(quote) + .setSkipHeaderRecord(false) + .setIgnoreEmptyLines(true) + .setAllowMissingColumnNames(true) + .build(); + } + + public static CSVParser openParser(File file, Charset charset, CSVFormat format) throws IOException { + return CSVParser.parse(file.toPath(), charset, format); + } + + public static char quoteChar(String requested) { + return StringUtils.isBlank(requested) ? '"' : requested.trim().charAt(0); + } + + public static char delimiterChar(String requested, Charset charset, File file) throws IOException { + if (StringUtils.isNotBlank(requested)) { + return requested.trim().charAt(0); + } + try (Reader reader = new InputStreamReader(Files.newInputStream(file.toPath()), charset)) { + StringBuilder line = new StringBuilder(); + int current; + while ((current = reader.read()) != -1) { + if (current == '\n' || current == '\r') { + if (line.length() > 0) { + break; + } + continue; + } + line.append((char) current); + } + return detectDelimiter(line.toString(), '"'); + } + } + + /** + * Reads the header and the first data rows for a preview. + */ + public static List> readSample(File file, Charset charset, CSVFormat format, int maxRows) + throws IOException { + List> rows = new ArrayList<>(); + try (CSVParser parser = openParser(file, charset, format)) { + int count = 0; + for (CSVRecord record : parser) { + if (count++ > maxRows) { + break; + } + rows.add(new ArrayList<>(record.toList())); + } + } + return rows; + } + + public static int sampleRows() { + return SAMPLE_ROWS; + } +} diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/excel/ImportPreviewListener.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/excel/ImportPreviewListener.java new file mode 100644 index 0000000000..43ab82957f --- /dev/null +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/excel/ImportPreviewListener.java @@ -0,0 +1,54 @@ +package ai.chat2db.community.domain.core.impl.task.imports.excel; + +import ai.chat2db.community.domain.core.impl.task.imports.ImportFileProbe; +import com.alibaba.excel.context.AnalysisContext; +import com.alibaba.excel.event.AnalysisEventListener; +import com.alibaba.excel.exception.ExcelAnalysisStopException; +import com.alibaba.excel.metadata.data.ReadCellData; +import com.alibaba.excel.util.ConverterUtils; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** + * Collects the header and a bounded number of data rows for an import preview, then stops the + * EasyExcel read so previewing never scans a whole large workbook. + */ +public final class ImportPreviewListener extends AnalysisEventListener> { + + private final List> rows = new ArrayList<>(); + + @Override + public void invokeHead(Map> headCells, AnalysisContext context) { + Map headMap = ConverterUtils.convertToStringMap(headCells, context); + int width = headMap.keySet().stream().mapToInt(Integer::intValue).max().orElse(-1) + 1; + List header = new ArrayList<>(width); + for (int index = 0; index < width; index++) { + header.add(headMap.getOrDefault(index, "")); + } + rows.add(header); + } + + @Override + public void invoke(Map data, AnalysisContext context) { + if (rows.size() > ImportFileProbe.sampleRows()) { + throw new ExcelAnalysisStopException(); + } + int width = rows.get(0).size(); + List row = new ArrayList<>(width); + for (int index = 0; index < width; index++) { + row.add(data.get(index)); + } + rows.add(row); + } + + @Override + public void doAfterAllAnalysed(AnalysisContext context) { + // The collected rows are read out by the caller. + } + + public List> rows() { + return rows; + } +} diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/ArtifactServiceTest.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/ArtifactServiceTest.java index 75767d03af..8b5d2239b4 100644 --- a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/ArtifactServiceTest.java +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/ArtifactServiceTest.java @@ -48,5 +48,4 @@ void failedPublicationReleasesReservedTarget() { assertEquals(failed.getTargetFile(), replacement.getTargetFile()); service.deleteDraft(replacement); } - } diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/LocalTaskManagerTest.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/LocalTaskManagerTest.java index b387e0babc..cd44269615 100644 --- a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/LocalTaskManagerTest.java +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/LocalTaskManagerTest.java @@ -2,8 +2,11 @@ import ai.chat2db.community.domain.api.config.DriverConfig; import ai.chat2db.community.domain.api.model.PageResponse; +import ai.chat2db.community.domain.api.model.task.ArtifactDraft; import ai.chat2db.community.domain.api.model.task.ExportTaskSpec; +import ai.chat2db.community.domain.api.model.task.ResumeState; import ai.chat2db.community.domain.api.model.task.Task; +import ai.chat2db.community.domain.api.model.task.TaskArtifact; import ai.chat2db.community.domain.api.model.task.TaskConstants; import ai.chat2db.community.domain.api.model.task.TaskErrorCode; import ai.chat2db.community.domain.api.model.task.TaskEvent; @@ -14,6 +17,7 @@ import ai.chat2db.community.domain.api.model.task.TaskQuery; import ai.chat2db.community.domain.api.model.task.TaskStatus; import ai.chat2db.community.domain.api.model.task.TaskStatusPatch; +import ai.chat2db.community.domain.api.model.task.TaskStage; import ai.chat2db.community.domain.api.model.task.TaskTargetSnapshot; import ai.chat2db.community.domain.api.model.task.TaskType; import ai.chat2db.community.domain.api.model.task.extension.TaskOperation; @@ -453,6 +457,57 @@ void artifactPreparationIsPersistedBeforePublication() throws Exception { Files.deleteIfExists(Path.of(storage.get(task.getId()).orElseThrow().getArtifactId())); } + @Test + void interruptedTaskWithResumeStateIsPreparedForResumeInsteadOfFailed() throws Exception { + TestTaskStorage storage = new TestTaskStorage(); + Task task = storage.create(newTask(), event(TaskEventCode.TASK_CREATED.name())); + assertTrue(storage.compareAndSetStatus(task.getId(), TaskStatus.PENDING.name(), + TaskStatus.RUNNING.name(), TaskStatusPatch.builder().build(), + event(TaskEventCode.TASK_STARTED.name()))); + Path temporary = Files.writeString( + tempDirectory.resolve(".task-" + task.getId() + "-resume.csv.part"), "partial"); + storage.saveResumeState(task.getId(), ResumeState.builder() + .shardNo(0).kind("KEYSET").rowsDone(500L).build()); + + manager(storage, (spec, context) -> {}).reconcileInterruptedTasks(); + + Task reconciled = storage.get(task.getId()).orElseThrow(); + assertEquals(TaskStatus.PENDING.name(), reconciled.getStatus()); + assertEquals(TaskStage.RESUMING.name(), reconciled.getStage()); + assertTrue(Files.exists(temporary)); + assertEquals(TaskEventCode.RESUME_AVAILABLE.name(), + storage.listEventsBefore(task.getId(), null, 1).get(0).getCode()); + } + + @Test + void allDraftsOfAMultiArtifactTaskArePublishedRecordedAndCleanable() throws Exception { + TestTaskStorage storage = new TestTaskStorage(); + taskManager = manager(storage, (spec, context) -> { + ArtifactDraft reject = context.createArtifact("REJECT", tempDirectory.toString(), + "reject.ndjson", "application/x-ndjson"); + writeQuietly(reject.getTemporaryFile().toPath(), "{\"line\":1}\n"); + ArtifactDraft output = context.createArtifact(tempDirectory.toString(), "export.csv", "text/csv"); + writeQuietly(output.getTemporaryFile().toPath(), "value\n"); + }); + Task task = newTask(); + + taskManager.submit(task, event(TaskEventCode.TASK_CREATED.name()), spec(), null, null); + + assertTrue(storage.awaitTerminal()); + Task finished = storage.get(task.getId()).orElseThrow(); + assertEquals(TaskStatus.SUCCESS.name(), finished.getStatus()); + List publishedArtifacts = storage.listArtifacts(task.getId()); + assertEquals(List.of("REJECT", "OUTPUT"), + publishedArtifacts.stream().map(TaskArtifact::getRole).toList()); + // The legacy single-artifact column always names the OUTPUT row, whichever order it was created in. + assertEquals("export.csv", Path.of(finished.getArtifactId()).getFileName().toString()); + assertTrue(publishedArtifacts.stream() + .anyMatch(artifact -> artifact.getArtifactId().equals(finished.getArtifactId()))); + for (TaskArtifact artifact : publishedArtifacts) { + Files.deleteIfExists(Path.of(artifact.getArtifactId())); + } + } + private LocalTaskManager manager(TestTaskStorage storage, TestExecution execution) { return manager(storage, execution, emptyExtensionManager()); } @@ -507,6 +562,14 @@ private TaskEvent event(String code) { .build(); } + private static void writeQuietly(Path path, String content) { + try { + Files.writeString(path, content); + } catch (java.io.IOException e) { + throw new java.io.UncheckedIOException(e); + } + } + @FunctionalInterface private interface TestExecution { void execute(ExportTaskSpec spec, TaskExecutionContext context); @@ -517,6 +580,8 @@ private static final class TestTaskStorage implements TaskStorage { private final AtomicLong ids = new AtomicLong(); private final Map tasks = new LinkedHashMap<>(); private final Map> events = new LinkedHashMap<>(); + private final Map> artifacts = new LinkedHashMap<>(); + private final Map> resumeStates = new LinkedHashMap<>(); private final CountDownLatch terminal = new CountDownLatch(1); private int terminalTransitions; private CountDownLatch createPaused; @@ -548,7 +613,11 @@ public synchronized Task create(Task task, TaskEvent createdEvent) { @Override public synchronized Optional get(Long taskId) { - return Optional.ofNullable(tasks.get(taskId)); + Task task = tasks.get(taskId); + if (task != null) { + task.setArtifacts(new ArrayList<>(artifacts.getOrDefault(taskId, List.of()))); + } + return Optional.ofNullable(task); } @Override @@ -644,10 +713,64 @@ public synchronized boolean deleteTerminalTask(Long taskId, Runnable commitActio } tasks.remove(taskId); events.remove(taskId); + artifacts.remove(taskId); + resumeStates.remove(taskId); commitAction.run(); return true; } + @Override + public synchronized List listArtifacts(Long taskId) { + return new ArrayList<>(artifacts.getOrDefault(taskId, List.of())); + } + + @Override + public synchronized void saveArtifact(Long taskId, TaskArtifact artifact) { + if (!tasks.containsKey(taskId)) { + throw new IllegalArgumentException("artifact must reference an existing task"); + } + List stored = artifacts.computeIfAbsent(taskId, ignored -> new ArrayList<>()); + stored.removeIf(existing -> existing.getArtifactId().equals(artifact.getArtifactId())); + stored.add(artifact); + } + + @Override + public synchronized void deleteArtifact(Long taskId, String artifactId) { + List stored = artifacts.get(taskId); + if (stored != null) { + stored.removeIf(existing -> existing.getArtifactId().equals(artifactId)); + } + } + + @Override + public synchronized List listResumableTasks() { + return tasks.values().stream() + .filter(task -> !TaskStatus.isTerminal(task.getStatus())) + .filter(task -> !resumeStates.getOrDefault(task.getId(), List.of()).isEmpty()) + .toList(); + } + + @Override + public synchronized void saveResumeState(Long taskId, ResumeState state) { + if (!tasks.containsKey(taskId)) { + throw new IllegalArgumentException("resume state must reference an existing task"); + } + List stored = resumeStates.computeIfAbsent(taskId, ignored -> new ArrayList<>()); + stored.removeIf(existing -> existing.getShardNo().equals(state.getShardNo())); + stored.add(state); + stored.sort(Comparator.comparing(ResumeState::getShardNo)); + } + + @Override + public synchronized List listResumeStates(Long taskId) { + return new ArrayList<>(resumeStates.getOrDefault(taskId, List.of())); + } + + @Override + public synchronized void clearResumeStates(Long taskId) { + resumeStates.remove(taskId); + } + boolean awaitTerminal() throws InterruptedException { return terminal.await(5, TimeUnit.SECONDS); } diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/TaskDeletionServiceImplTest.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/TaskDeletionServiceImplTest.java index 01d3b986f9..70617b85c9 100644 --- a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/TaskDeletionServiceImplTest.java +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/TaskDeletionServiceImplTest.java @@ -1,7 +1,9 @@ package ai.chat2db.community.domain.core.impl.task; import ai.chat2db.community.domain.api.model.PageResponse; +import ai.chat2db.community.domain.api.model.task.ResumeState; import ai.chat2db.community.domain.api.model.task.Task; +import ai.chat2db.community.domain.api.model.task.TaskArtifact; import ai.chat2db.community.domain.api.model.task.TaskEvent; import ai.chat2db.community.domain.api.model.task.TaskProgress; import ai.chat2db.community.domain.api.model.task.TaskQuery; @@ -490,5 +492,40 @@ public List listEventsBefore(Long id, Long before, int limit) { public List listNonTerminalTasks() { throw new UnsupportedOperationException(); } + + @Override + public List listArtifacts(Long taskId) { + return List.of(); + } + + @Override + public void saveArtifact(Long taskId, TaskArtifact artifact) { + throw new UnsupportedOperationException(); + } + + @Override + public void deleteArtifact(Long taskId, String artifactId) { + throw new UnsupportedOperationException(); + } + + @Override + public List listResumableTasks() { + return List.of(); + } + + @Override + public void saveResumeState(Long taskId, ResumeState state) { + throw new UnsupportedOperationException(); + } + + @Override + public List listResumeStates(Long taskId) { + return List.of(); + } + + @Override + public void clearResumeStates(Long taskId) { + throw new UnsupportedOperationException(); + } } } diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/TaskExecutorRegistryTest.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/TaskExecutorRegistryTest.java index d598a87eda..667ae83bc4 100644 --- a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/TaskExecutorRegistryTest.java +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/TaskExecutorRegistryTest.java @@ -211,6 +211,8 @@ private static final class RecordingTaskStorage implements TaskStorage { private final AtomicLong ids = new AtomicLong(); private final Map tasks = new LinkedHashMap<>(); private final Map> events = new LinkedHashMap<>(); + private final Map> artifacts = + new LinkedHashMap<>(); private final List statusTransitions = new ArrayList<>(); private int createCount; @@ -313,10 +315,58 @@ public synchronized boolean deleteTerminalTask(Long taskId, Runnable commitActio } tasks.remove(taskId); events.remove(taskId); + artifacts.remove(taskId); commitAction.run(); return true; } + @Override + public synchronized List listArtifacts(Long taskId) { + return new ArrayList<>(artifacts.getOrDefault(taskId, List.of())); + } + + @Override + public synchronized void saveArtifact(Long taskId, + ai.chat2db.community.domain.api.model.task.TaskArtifact artifact) { + if (!tasks.containsKey(taskId)) { + throw new IllegalArgumentException("artifact must reference an existing task"); + } + List stored = + artifacts.computeIfAbsent(taskId, ignored -> new ArrayList<>()); + stored.removeIf(existing -> existing.getArtifactId().equals(artifact.getArtifactId())); + stored.add(artifact); + } + + @Override + public synchronized void deleteArtifact(Long taskId, String artifactId) { + List stored = artifacts.get(taskId); + if (stored != null) { + stored.removeIf(existing -> existing.getArtifactId().equals(artifactId)); + } + } + + @Override + public synchronized List listResumableTasks() { + return List.of(); + } + + @Override + public synchronized void saveResumeState(Long taskId, + ai.chat2db.community.domain.api.model.task.ResumeState state) { + throw new UnsupportedOperationException(); + } + + @Override + public synchronized List listResumeStates( + Long taskId) { + return List.of(); + } + + @Override + public synchronized void clearResumeStates(Long taskId) { + throw new UnsupportedOperationException(); + } + synchronized int createCount() { return createCount; } diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/TaskServiceImplTest.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/TaskServiceImplTest.java index 2bdffbe84c..9ad2fb18b7 100644 --- a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/TaskServiceImplTest.java +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/TaskServiceImplTest.java @@ -1,42 +1,69 @@ package ai.chat2db.community.domain.core.impl.task; import ai.chat2db.community.domain.api.model.PageResponse; +import ai.chat2db.community.domain.api.config.DBConfig; +import ai.chat2db.community.domain.api.config.DriverConfig; +import ai.chat2db.community.domain.api.model.metadata.TableColumn; +import ai.chat2db.community.domain.api.model.request.runtime.DbConnectionContextRequest; +import ai.chat2db.community.domain.api.model.task.ImportPreview; +import ai.chat2db.community.domain.api.model.task.ImportTaskSpec; +import ai.chat2db.community.domain.api.model.task.ResumeState; import ai.chat2db.community.domain.api.model.task.Task; +import ai.chat2db.community.domain.api.model.task.TaskArtifact; import ai.chat2db.community.domain.api.model.task.TaskDownload; import ai.chat2db.community.domain.api.model.task.TaskEvent; import ai.chat2db.community.domain.api.model.task.TaskProgress; import ai.chat2db.community.domain.api.model.task.TaskQuery; import ai.chat2db.community.domain.api.model.task.TaskStatus; import ai.chat2db.community.domain.api.model.task.TaskStatusPatch; +import ai.chat2db.community.domain.api.model.task.TaskTargetSnapshot; +import ai.chat2db.community.domain.api.service.db.IDbConnectionContextService; +import ai.chat2db.community.domain.api.service.file.IImportFileStagingService; import ai.chat2db.community.domain.api.service.task.TaskStorage; +import ai.chat2db.community.tools.exception.BusinessException; import ai.chat2db.community.tools.exception.DataNotFoundException; import ai.chat2db.community.tools.model.Context; import ai.chat2db.community.tools.model.LoginUser; import ai.chat2db.community.tools.util.ContextUtils; +import ai.chat2db.spi.model.datasource.ConnectInfo; +import ai.chat2db.spi.IDbMetaData; +import ai.chat2db.spi.IPlugin; +import ai.chat2db.spi.sql.Chat2DBContext; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import java.lang.reflect.Field; +import java.lang.reflect.Proxy; import java.nio.file.Files; import java.nio.file.Path; +import java.sql.Connection; +import java.sql.DriverManager; import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; class TaskServiceImplTest { + private static final String RESUME_TEST_DB_TYPE = "TASK_RESUME_TEST"; + @TempDir Path tempDirectory; @AfterEach void clearContext() { ContextUtils.removeContext(); + Chat2DBContext.removeContext(); + Chat2DBContext.PLUGIN_MAP.remove(RESUME_TEST_DB_TYPE); } @Test @@ -77,6 +104,165 @@ void everyTaskReadAndMutationIsIsolatedByUserAndOrganization() throws Exception assertEquals("owned.csv", download.getFileName()); } + @Test + void importAllowlistResolvesSymbolicLinksBeforeAuthorizingTheSource() throws Exception { + Path allowed = Files.createDirectory(tempDirectory.resolve("allowed")); + Path outside = Files.createDirectory(tempDirectory.resolve("outside")); + Files.writeString(outside.resolve("data.csv"), "id\n1\n"); + Path link = allowed.resolve("linked"); + try { + Files.createSymbolicLink(link, outside); + } catch (UnsupportedOperationException | java.io.IOException | SecurityException unavailable) { + org.junit.jupiter.api.Assumptions.assumeTrue(false, + "Symbolic links are unavailable: " + unavailable.getMessage()); + } + TaskServiceImpl service = new TaskServiceImpl(new OwnershipTaskStorage(List.of()), null, + new ArtifactServiceImpl()); + Field field = TaskServiceImpl.class.getDeclaredField("importAllowedRoots"); + field.setAccessible(true); + field.set(service, allowed.toString()); + ImportTaskSpec spec = ImportTaskSpec.builder() + .sourceFile(link.resolve("data.csv").toString()) + .build(); + + assertThrows(BusinessException.class, () -> service.submitImport(spec)); + } + + @Test + void resumeRebuildsConnectionFromPersistedTargetAndClearsTemporaryContext() { + AtomicReference boundRequest = new AtomicReference<>(); + ConnectInfo complete = new ConnectInfo(); + complete.setDataSourceId(42L); + complete.setDbType(RESUME_TEST_DB_TYPE); + complete.setDatabase("archive"); + complete.setSchemaName("audit"); + complete.setPassword("not-persisted-in-task"); + IDbConnectionContextService connectionContexts = (IDbConnectionContextService) Proxy.newProxyInstance( + getClass().getClassLoader(), new Class[] {IDbConnectionContextService.class}, + (proxy, method, args) -> { + if ("bind".equals(method.getName())) { + boundRequest.set((DbConnectionContextRequest) args[0]); + Chat2DBContext.putContext(complete); + } else if ("clear".equals(method.getName())) { + Chat2DBContext.removeContext(); + } + return null; + }); + TaskServiceImpl service = new TaskServiceImpl(new OwnershipTaskStorage(List.of()), null, + new ArtifactServiceImpl(), connectionContexts); + DBConfig config = new DBConfig(); + config.setDbType(RESUME_TEST_DB_TYPE); + config.setDefaultDriverConfig(new DriverConfig()); + Chat2DBContext.PLUGIN_MAP.put(RESUME_TEST_DB_TYPE, new IPlugin() { + @Override + public DBConfig getDBConfig() { + return config; + } + }); + ImportTaskSpec spec = ImportTaskSpec.builder() + .target(TaskTargetSnapshot.builder() + .dataSourceId(42L) + .databaseName("archive") + .schemaName("audit") + .tableName("events") + .build()) + .build(); + + ConnectInfo resolved = service.resumeConnectInfo(spec); + + assertEquals(42L, boundRequest.get().getDataSourceId()); + assertEquals("archive", boundRequest.get().getDatabaseName()); + assertEquals("audit", boundRequest.get().getSchemaName()); + assertEquals("not-persisted-in-task", resolved.getPassword()); + assertNull(Chat2DBContext.getConnectInfo()); + } + + @Test + void importPreviewReadsMetadataWhileThePersistedTargetContextIsBound() throws Exception { + Path source = Files.writeString(tempDirectory.resolve("preview.csv"), "ID\n1\n"); + AtomicBoolean metadataReadWithBoundContext = new AtomicBoolean(); + try (Connection connection = DriverManager.getConnection("jdbc:h2:mem:task_preview")) { + ConnectInfo complete = new ConnectInfo(); + complete.setDataSourceId(42L); + complete.setDbType(RESUME_TEST_DB_TYPE); + complete.setDatabase("archive"); + complete.setSchemaName("audit"); + complete.setConnection(connection); + IDbConnectionContextService connectionContexts = connectionContexts(complete, new AtomicReference<>()); + TaskServiceImpl service = new TaskServiceImpl(new OwnershipTaskStorage(List.of()), null, + new ArtifactServiceImpl(), connectionContexts); + DBConfig config = new DBConfig(); + config.setDbType(RESUME_TEST_DB_TYPE); + config.setDefaultDriverConfig(new DriverConfig()); + IDbMetaData metadata = (IDbMetaData) Proxy.newProxyInstance(getClass().getClassLoader(), + new Class[] {IDbMetaData.class}, (proxy, method, args) -> { + if ("getSQLIdentifierProcessor".equals(method.getName())) { + return new ai.chat2db.spi.DefaultSQLIdentifierProcessor(); + } + if ("tables".equals(method.getName())) { + return List.of(ai.chat2db.community.domain.api.model.metadata.Table.builder() + .name("events").build()); + } + if ("columns".equals(method.getName())) { + metadataReadWithBoundContext.set(Chat2DBContext.getConnectInfo() == complete); + return List.of(TableColumn.builder().name("ID").columnType("INTEGER").build()); + } + return null; + }); + Chat2DBContext.PLUGIN_MAP.put(RESUME_TEST_DB_TYPE, new IPlugin() { + @Override + public DBConfig getDBConfig() { + return config; + } + + @Override + public IDbMetaData getDbMetaData() { + return metadata; + } + }); + ImportTaskSpec spec = ImportTaskSpec.builder() + .format("CSV") + .sourceFile(source.toString()) + .target(TaskTargetSnapshot.builder() + .dataSourceId(42L) + .databaseName("archive") + .schemaName("audit") + .tableName("events") + .build()) + .build(); + + IImportFileStagingService staging = (IImportFileStagingService) Proxy.newProxyInstance( + getClass().getClassLoader(), new Class[] {IImportFileStagingService.class}, + (proxy, method, args) -> { + assertEquals("resolve", method.getName(), "Preview must not claim or delete a source"); + assertEquals("staged-preview-id", args[0]); + return source.toFile(); + }); + TaskServiceImpl stagedService = new TaskServiceImpl(new OwnershipTaskStorage(List.of()), null, + new ArtifactServiceImpl(), connectionContexts, staging); + spec.setImportFileId("staged-preview-id"); + spec.setSourceFile(tempDirectory.resolve("untrusted.csv").toString()); + assertEquals(List.of("ID"), stagedService.previewImport(spec).getFileColumns()); + assertEquals(source.toFile().getAbsolutePath(), spec.getSourceFile()); + assertTrue(metadataReadWithBoundContext.get()); + assertNull(Chat2DBContext.getConnectInfo()); + } + } + + private IDbConnectionContextService connectionContexts(ConnectInfo complete, + AtomicReference boundRequest) { + return (IDbConnectionContextService) Proxy.newProxyInstance(getClass().getClassLoader(), + new Class[] {IDbConnectionContextService.class}, (proxy, method, args) -> { + if ("bind".equals(method.getName())) { + boundRequest.set((DbConnectionContextRequest) args[0]); + Chat2DBContext.putContext(complete); + } else if ("clear".equals(method.getName())) { + Chat2DBContext.removeContext(); + } + return null; + }); + } + private Task task(Long id, Long userId, Long organizationId, Path artifact) { return Task.builder() .id(id) @@ -164,5 +350,40 @@ public boolean deleteTerminalTask(Long taskId, Runnable commitAction) { commitAction.run(); return true; } + + @Override + public List listArtifacts(Long taskId) { + return List.of(); + } + + @Override + public void saveArtifact(Long taskId, TaskArtifact artifact) { + throw new UnsupportedOperationException(); + } + + @Override + public void deleteArtifact(Long taskId, String artifactId) { + throw new UnsupportedOperationException(); + } + + @Override + public List listResumableTasks() { + return List.of(); + } + + @Override + public void saveResumeState(Long taskId, ResumeState state) { + throw new UnsupportedOperationException(); + } + + @Override + public List listResumeStates(Long taskId) { + return List.of(); + } + + @Override + public void clearResumeStates(Long taskId) { + throw new UnsupportedOperationException(); + } } } diff --git a/chat2db-community-server/chat2db-community-start/src/main/resources/i18n/messages.properties b/chat2db-community-server/chat2db-community-start/src/main/resources/i18n/messages.properties index 1d9230fded..7947f2b333 100644 --- a/chat2db-community-server/chat2db-community-start/src/main/resources/i18n/messages.properties +++ b/chat2db-community-server/chat2db-community-start/src/main/resources/i18n/messages.properties @@ -181,3 +181,8 @@ import.csv.invalidValue=CSV row {0}, column "{1}" cannot convert value "{2}". Ch datasource.identityColor.invalid=Identity color must use the #RRGGBB format task.delete.activeForbidden=Pending or running tasks cannot be deleted task.delete.artifactFailed=Failed to delete exported file: {0} +task.import.preview.sourceUnreadable=Could not read the import source file +task.import.preview.unsupportedFormat=Preview is not supported for this file format +task.import.preview.failed=Import preview failed +task.resume.conflict=The task is already being resumed elsewhere, please refresh the task list +task.import.sourceNotAllowed=The import source file is outside the allowed directories diff --git a/chat2db-community-server/chat2db-community-start/src/main/resources/i18n/messages_en_US.properties b/chat2db-community-server/chat2db-community-start/src/main/resources/i18n/messages_en_US.properties index e51640ae02..6e2cb9e7e3 100644 --- a/chat2db-community-server/chat2db-community-start/src/main/resources/i18n/messages_en_US.properties +++ b/chat2db-community-server/chat2db-community-start/src/main/resources/i18n/messages_en_US.properties @@ -200,3 +200,8 @@ import.csv.invalidValue=CSV row {0}, column "{1}" cannot convert value "{2}". Ch datasource.identityColor.invalid=Identity color must use the #RRGGBB format task.delete.activeForbidden=Pending or running tasks cannot be deleted task.delete.artifactFailed=Failed to delete exported file: {0} +task.import.preview.sourceUnreadable=Could not read the import source file +task.import.preview.unsupportedFormat=Preview is not supported for this file format +task.import.preview.failed=Import preview failed +task.resume.conflict=The task is already being resumed elsewhere, please refresh the task list +task.import.sourceNotAllowed=The import source file is outside the allowed directories diff --git a/chat2db-community-server/chat2db-community-start/src/main/resources/i18n/messages_es_ES.properties b/chat2db-community-server/chat2db-community-start/src/main/resources/i18n/messages_es_ES.properties index b19e27c83c..419053518d 100644 --- a/chat2db-community-server/chat2db-community-start/src/main/resources/i18n/messages_es_ES.properties +++ b/chat2db-community-server/chat2db-community-start/src/main/resources/i18n/messages_es_ES.properties @@ -200,3 +200,8 @@ import.csv.invalidValue=No se puede convertir el valor "{2}" de la fila {0}, col datasource.identityColor.invalid=El color de identidad debe usar el formato #RRGGBB task.delete.activeForbidden=No se pueden eliminar tareas pendientes o en ejecución task.delete.artifactFailed=No se pudo eliminar el archivo exportado: {0} +task.import.preview.sourceUnreadable=No se pudo leer el archivo de origen de importación +task.import.preview.unsupportedFormat=La vista previa no es compatible con este formato de archivo +task.import.preview.failed=Error en la vista previa de importación +task.resume.conflict=La tarea ya se está reanudando en otro lugar, actualice la lista de tareas +task.import.sourceNotAllowed=El archivo de origen de importación está fuera de los directorios permitidos diff --git a/chat2db-community-server/chat2db-community-start/src/main/resources/i18n/messages_ja_JP.properties b/chat2db-community-server/chat2db-community-start/src/main/resources/i18n/messages_ja_JP.properties index 69b462c0d7..98bd63a3fc 100644 --- a/chat2db-community-server/chat2db-community-start/src/main/resources/i18n/messages_ja_JP.properties +++ b/chat2db-community-server/chat2db-community-start/src/main/resources/i18n/messages_ja_JP.properties @@ -77,3 +77,8 @@ import.preview.unclosedQuote=CSV の {0} 行目から始まる引用符が閉じ import.csv.invalidValue=CSV の {0} 行目、列「{1}」の値「{2}」を変換できません。日付、時刻、小数形式の設定を確認してください。 task.delete.activeForbidden=保留中または実行中のタスクは削除できません task.delete.artifactFailed=エクスポートファイルを削除できませんでした: {0} +task.import.preview.sourceUnreadable=インポート元ファイルを読み取れません +task.import.preview.unsupportedFormat=このファイル形式はプレビューに対応していません +task.import.preview.failed=インポートプレビューに失敗しました +task.resume.conflict=タスクは他で既に再開中です。タスク一覧を更新してください +task.import.sourceNotAllowed=インポート元ファイルが許可されたディレクトリの外にあります diff --git a/chat2db-community-server/chat2db-community-start/src/main/resources/i18n/messages_ko_KR.properties b/chat2db-community-server/chat2db-community-start/src/main/resources/i18n/messages_ko_KR.properties index 2199090842..8725acb1a9 100644 --- a/chat2db-community-server/chat2db-community-start/src/main/resources/i18n/messages_ko_KR.properties +++ b/chat2db-community-server/chat2db-community-start/src/main/resources/i18n/messages_ko_KR.properties @@ -200,3 +200,8 @@ import.csv.invalidValue=CSV {0}행, "{1}" 열의 값 "{2}"을 변환할 수 없 datasource.identityColor.invalid=식별 색상은 #RRGGBB 형식을 사용해야 합니다 task.delete.activeForbidden=대기 중이거나 실행 중인 작업은 삭제할 수 없습니다 task.delete.artifactFailed=내보낸 파일을 삭제하지 못했습니다: {0} +task.import.preview.sourceUnreadable=원본 파일을 읽을 수 없습니다 +task.import.preview.unsupportedFormat=이 파일 형식은 미리보기를 지원하지 않습니다 +task.import.preview.failed=반어로 미리보기 실패 +task.resume.conflict=작업이 다른 곳에서 이미 재개 중입니다. 작업 목록을 새로고침하세요 +task.import.sourceNotAllowed=가져오기 원본 파일이 허용된 디렉터리 밖에 있습니다 diff --git a/chat2db-community-server/chat2db-community-start/src/main/resources/i18n/messages_zh_CN.properties b/chat2db-community-server/chat2db-community-start/src/main/resources/i18n/messages_zh_CN.properties index 66bdcbf914..f9b17b8514 100644 --- a/chat2db-community-server/chat2db-community-start/src/main/resources/i18n/messages_zh_CN.properties +++ b/chat2db-community-server/chat2db-community-start/src/main/resources/i18n/messages_zh_CN.properties @@ -195,3 +195,8 @@ import.csv.invalidValue=CSV 第 {0} 行的列“{1}”无法转换值“{2}” datasource.identityColor.invalid=身份颜色必须使用 #RRGGBB 格式 task.delete.activeForbidden=等待中或进行中的任务不能删除 task.delete.artifactFailed=删除导出文件失败：{0} +task.import.preview.sourceUnreadable=无法读取导入源文件 +task.import.preview.unsupportedFormat=该文件类型不支持预览 +task.import.preview.failed=导入预览失败 +task.resume.conflict=任务已在别处恢复，请刷新任务列表后重试 +task.import.sourceNotAllowed=导入源文件不在允许的目录内 diff --git a/chat2db-community-server/chat2db-community-storage/pom.xml b/chat2db-community-server/chat2db-community-storage/pom.xml index fec7195a78..6dcf0411e5 100644 --- a/chat2db-community-server/chat2db-community-storage/pom.xml +++ b/chat2db-community-server/chat2db-community-storage/pom.xml @@ -49,6 +49,10 @@ com.alibaba.fastjson2 fastjson2 + + com.h2database + h2 + commons-beanutils commons-beanutils diff --git a/chat2db-community-server/chat2db-community-storage/src/main/java/ai/chat2db/community/storage/TaskLifecyclePolicy.java b/chat2db-community-server/chat2db-community-storage/src/main/java/ai/chat2db/community/storage/TaskLifecyclePolicy.java new file mode 100644 index 0000000000..92064b81d2 --- /dev/null +++ b/chat2db-community-server/chat2db-community-storage/src/main/java/ai/chat2db/community/storage/TaskLifecyclePolicy.java @@ -0,0 +1,98 @@ +package ai.chat2db.community.storage; + +import ai.chat2db.community.domain.api.model.task.Task; +import ai.chat2db.community.domain.api.model.task.TaskConstants; +import ai.chat2db.community.domain.api.model.task.TaskStage; +import ai.chat2db.community.domain.api.model.task.TaskStatus; +import ai.chat2db.community.domain.api.model.task.TaskStatusPatch; + +import java.util.Date; +import java.util.List; + +/** + * Task lifecycle rules shared by every {@code TaskStorage} implementation. The rules live here + * rather than inside a storage class because the file and database implementations must not be able + * to drift apart on status transitions, progress monotonicity or event paging. + */ +public final class TaskLifecyclePolicy { + + /** + * Status written by older releases for a cancellation that had not finished yet; only kept so + * persisted rows remain transitionable. + */ + private static final String LEGACY_CANCELLING_STATUS = "CANCELLING"; + + private TaskLifecyclePolicy() { + } + + public static boolean isLegalTransition(String source, String target, TaskStatusPatch patch) { + if (TaskStatus.PENDING.name().equals(source)) { + return TaskStatus.RUNNING.name().equals(target) || TaskStatus.FAILED.name().equals(target); + } + if (TaskStatus.RUNNING.name().equals(source)) { + return TaskStatus.SUCCESS.name().equals(target) || TaskStatus.FAILED.name().equals(target) + || (TaskStatus.PENDING.name().equals(target) && patch != null + && TaskStage.RESUMING.name().equals(patch.getStage())); + } + if (LEGACY_CANCELLING_STATUS.equals(source)) { + return TaskStatus.FAILED.name().equals(target); + } + return false; + } + + /** + * Applies a status transition to {@code task}, which the caller has already verified is legal. + */ + public static void applyStatusPatch(Task task, String targetStatus, TaskStatusPatch patch) { + TaskStatusPatch effectivePatch = patch == null ? new TaskStatusPatch() : patch; + int previousProgress = task.getProgress() == null ? TaskConstants.PENDING_PROGRESS : task.getProgress(); + task.setStatus(targetStatus); + if (TaskStatus.SUCCESS.name().equals(targetStatus)) { + task.setProgress(TaskConstants.COMPLETED_PROGRESS); + } else if (!TaskStatus.isTerminal(targetStatus) && effectivePatch.getProgress() != null) { + task.setProgress(Math.max(previousProgress, Math.min(TaskConstants.MAX_RUNNING_PROGRESS, + effectivePatch.getProgress()))); + } else { + task.setProgress(previousProgress); + } + if (effectivePatch.getStage() != null) { + task.setStage(effectivePatch.getStage()); + } + task.setProgressMessage(effectivePatch.getProgressMessage()); + task.setErrorCode(TaskStatus.FAILED.name().equals(targetStatus) ? effectivePatch.getErrorCode() : null); + task.setErrorMessage(TaskStatus.FAILED.name().equals(targetStatus) ? effectivePatch.getErrorMessage() : null); + task.setArtifactId(TaskStatus.SUCCESS.name().equals(targetStatus) + ? primaryArtifactId(effectivePatch) : null); + if (effectivePatch.getStartedAt() != null) { + task.setStartedAt(effectivePatch.getStartedAt()); + } + if (effectivePatch.getFinishedAt() != null) { + task.setFinishedAt(effectivePatch.getFinishedAt()); + } + task.setUpdatedAt(effectivePatch.getUpdatedAt() == null ? new Date() : effectivePatch.getUpdatedAt()); + } + + /** + * The legacy single-artifact column always names the primary output, so a multi-artifact + * completion records the first id of the list. + */ + private static String primaryArtifactId(TaskStatusPatch patch) { + List artifactIds = patch.getArtifactIds(); + if (artifactIds != null && !artifactIds.isEmpty()) { + return artifactIds.get(0); + } + return patch.getArtifactId(); + } + + /** + * Progress a running task may report: never below the started value, never into the completed + * range, which only a SUCCESS transition may write. + */ + public static int runningProgress(int requested) { + return Math.max(TaskConstants.STARTED_PROGRESS, Math.min(TaskConstants.MAX_RUNNING_PROGRESS, requested)); + } + + public static int eventLimit(int limit) { + return Math.max(1, Math.min(TaskConstants.MAX_EVENT_LIMIT, limit)); + } +} diff --git a/chat2db-community-server/chat2db-community-storage/src/main/java/ai/chat2db/community/storage/large/FileTaskStorage.java b/chat2db-community-server/chat2db-community-storage/src/main/java/ai/chat2db/community/storage/large/FileTaskStorage.java index 05ac35ac3a..1888bcc45f 100644 --- a/chat2db-community-server/chat2db-community-storage/src/main/java/ai/chat2db/community/storage/large/FileTaskStorage.java +++ b/chat2db-community-server/chat2db-community-storage/src/main/java/ai/chat2db/community/storage/large/FileTaskStorage.java @@ -1,7 +1,9 @@ package ai.chat2db.community.storage.large; import ai.chat2db.community.domain.api.model.PageResponse; +import ai.chat2db.community.domain.api.model.task.ResumeState; import ai.chat2db.community.domain.api.model.task.Task; +import ai.chat2db.community.domain.api.model.task.TaskArtifact; import ai.chat2db.community.domain.api.model.task.TaskConstants; import ai.chat2db.community.domain.api.model.task.TaskEvent; import ai.chat2db.community.domain.api.model.task.TaskProgress; @@ -10,6 +12,7 @@ import ai.chat2db.community.domain.api.model.task.TaskStatusPatch; import ai.chat2db.community.domain.api.service.task.TaskStorage; import ai.chat2db.community.storage.IdUtil; +import ai.chat2db.community.storage.TaskLifecyclePolicy; import cn.hutool.core.io.FileUtil; import com.alibaba.fastjson2.JSON; import lombok.AllArgsConstructor; @@ -17,7 +20,6 @@ import lombok.NoArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; -import org.springframework.stereotype.Component; import java.io.BufferedReader; import java.io.ByteArrayOutputStream; @@ -34,29 +36,28 @@ import java.nio.file.StandardOpenOption; import java.util.ArrayList; import java.util.Collections; +import java.util.Comparator; import java.util.Date; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Consumer; @Slf4j -@Component public class FileTaskStorage implements TaskStorage { - static final String TASK_STORAGE_DIRECTORY = "task-v2"; + public static final String TASK_STORAGE_DIRECTORY = "task-v2"; - static final String TASK_INDEX_NAME = "task"; + public static final String TASK_INDEX_NAME = "task"; - static final String TASK_EVENT_FILE_SUFFIX = "-events.json"; + public static final String TASK_EVENT_FILE_SUFFIX = "-events.json"; static final String TASK_TRANSITION_FILE_SUFFIX = "-transition.json"; static final String DELETING_FILE_SUFFIX = ".deleting"; - private static final String LEGACY_CANCELLING_STATUS = "CANCELLING"; - static final int NO_FILE_LIMIT = 0; private final TaskSnapshotStorage snapshots; @@ -68,7 +69,7 @@ public FileTaskStorage() { this(new TaskSnapshotStorage()); } - FileTaskStorage(String storageBasePath) { + public FileTaskStorage(String storageBasePath) { this(new TaskSnapshotStorage(storageBasePath)); } @@ -137,7 +138,7 @@ public synchronized boolean compareAndSetStatus(Long taskId, String expectedStat TaskStatusPatch patch, TaskEvent lifecycleEvent) { Task current = snapshots.find(taskId); if (current == null || !expectedStatus.equals(current.getStatus()) - || !isLegalTransition(expectedStatus, targetStatus)) { + || !TaskLifecyclePolicy.isLegalTransition(expectedStatus, targetStatus, patch)) { return false; } if (lifecycleEvent == null) { @@ -145,7 +146,7 @@ public synchronized boolean compareAndSetStatus(Long taskId, String expectedStat } Task updated = copy(current); - applyPatch(updated, targetStatus, patch); + TaskLifecyclePolicy.applyStatusPatch(updated, targetStatus, patch); TaskTransition transition = new TaskTransition(updated, prepareEvent(taskId, lifecycleEvent)); writeTransition(transition); commitTransition(transition, false); @@ -159,8 +160,7 @@ public synchronized boolean updateProgressIfRunning(Long taskId, TaskProgress pr || progress == null || progress.getProgress() == null) { return false; } - int requested = Math.max(TaskConstants.STARTED_PROGRESS, - Math.min(TaskConstants.MAX_RUNNING_PROGRESS, progress.getProgress())); + int requested = TaskLifecyclePolicy.runningProgress(progress.getProgress()); int current = currentTask.getProgress() == null ? TaskConstants.PENDING_PROGRESS : currentTask.getProgress(); if (requested < current) { return false; @@ -186,7 +186,7 @@ public synchronized TaskEvent appendEvent(TaskEvent event) { @Override public synchronized List listEvents(Long taskId, long afterSequence, int limit) { - int resultLimit = eventLimit(limit); + int resultLimit = TaskLifecyclePolicy.eventLimit(limit); File file = eventsFile(taskId); repairIncompleteTrailingEvent(file, taskId); if (!file.isFile()) { @@ -244,7 +244,7 @@ private String readNextLine(RandomAccessFile input) throws IOException { @Override public synchronized List listEventsBefore(Long taskId, Long beforeSequence, int limit) { - int resultLimit = eventLimit(limit); + int resultLimit = TaskLifecyclePolicy.eventLimit(limit); File file = eventsFile(taskId); repairIncompleteTrailingEvent(file, taskId); if (!file.isFile()) { @@ -382,6 +382,91 @@ public synchronized boolean deleteTerminalTask(Long taskId, Runnable commitActio } } + @Override + public synchronized List listArtifacts(Long taskId) { + Task task = taskId == null ? null : snapshots.find(taskId); + return task == null || task.getArtifacts() == null ? List.of() : List.copyOf(task.getArtifacts()); + } + + @Override + public synchronized void saveArtifact(Long taskId, TaskArtifact artifact) { + if (taskId == null || artifact == null || artifact.getArtifactId() == null || artifact.getRole() == null) { + throw new IllegalArgumentException("artifact must reference an existing task"); + } + mutateTask(taskId, "artifact", updated -> { + List artifacts = new ArrayList<>( + updated.getArtifacts() == null ? List.of() : updated.getArtifacts()); + artifacts.removeIf(existing -> artifact.getArtifactId().equals(existing.getArtifactId())); + artifacts.add(JSON.parseObject(JSON.toJSONString(artifact), TaskArtifact.class)); + updated.setArtifacts(artifacts); + }); + } + + @Override + public synchronized void deleteArtifact(Long taskId, String artifactId) { + if (taskId == null || artifactId == null || snapshots.find(taskId) == null) { + return; + } + mutateTask(taskId, "artifact", updated -> { + List artifacts = new ArrayList<>( + updated.getArtifacts() == null ? List.of() : updated.getArtifacts()); + artifacts.removeIf(existing -> artifactId.equals(existing.getArtifactId())); + updated.setArtifacts(artifacts); + }); + } + + @Override + public synchronized List listResumableTasks() { + return snapshots.all().stream() + .filter(task -> task.getStatus() != null && !TaskStatus.isTerminal(task.getStatus())) + .filter(task -> task.getResumeStates() != null && !task.getResumeStates().isEmpty()) + .map(this::copy) + .toList(); + } + + @Override + public synchronized void saveResumeState(Long taskId, ResumeState state) { + if (taskId == null || state == null || state.getShardNo() == null || state.getKind() == null) { + throw new IllegalArgumentException("resume state must reference an existing task"); + } + mutateTask(taskId, "resume state", updated -> { + List states = new ArrayList<>( + updated.getResumeStates() == null ? List.of() : updated.getResumeStates()); + states.removeIf(existing -> state.getShardNo().equals(existing.getShardNo())); + states.add(JSON.parseObject(JSON.toJSONString(state), ResumeState.class)); + states.sort(Comparator.comparing(ResumeState::getShardNo)); + updated.setResumeStates(states); + }); + } + + @Override + public synchronized List listResumeStates(Long taskId) { + Task task = taskId == null ? null : snapshots.find(taskId); + return task == null || task.getResumeStates() == null ? List.of() : List.copyOf(task.getResumeStates()); + } + + @Override + public synchronized void clearResumeStates(Long taskId) { + if (taskId == null || snapshots.find(taskId) == null) { + return; + } + mutateTask(taskId, "resume state", updated -> updated.setResumeStates(null)); + } + + /** + * Rewrites the stored snapshot with {@code change} applied to a copy, so artifact and resume + * data share the task snapshot's single-writer semantics and its deletion rollback. + */ + private void mutateTask(Long taskId, String subject, Consumer change) { + Task current = snapshots.find(taskId); + if (current == null) { + throw new IllegalArgumentException(subject + " must reference an existing task"); + } + Task updated = copy(current); + change.accept(updated); + snapshots.replaceStrict(taskId, updated); + } + private void commitTransition(TaskTransition transition, boolean recovery) { TaskEvent event = transition.getEvent(); long previousLength = eventFileLength(event.getTaskId()); @@ -628,51 +713,6 @@ private void truncateEventFile(Long taskId, long length, long lastSequence) { } } - private int eventLimit(int limit) { - return Math.max(1, Math.min(TaskConstants.MAX_EVENT_LIMIT, limit)); - } - - private void applyPatch(Task task, String targetStatus, TaskStatusPatch patch) { - TaskStatusPatch effectivePatch = patch == null ? new TaskStatusPatch() : patch; - int previousProgress = task.getProgress() == null ? TaskConstants.PENDING_PROGRESS : task.getProgress(); - task.setStatus(targetStatus); - if (TaskStatus.SUCCESS.name().equals(targetStatus)) { - task.setProgress(TaskConstants.COMPLETED_PROGRESS); - } else if (!TaskStatus.isTerminal(targetStatus) && effectivePatch.getProgress() != null) { - task.setProgress(Math.max(previousProgress, Math.min(TaskConstants.MAX_RUNNING_PROGRESS, - effectivePatch.getProgress()))); - } else { - task.setProgress(previousProgress); - } - if (effectivePatch.getStage() != null) { - task.setStage(effectivePatch.getStage()); - } - task.setProgressMessage(effectivePatch.getProgressMessage()); - task.setErrorCode(TaskStatus.FAILED.name().equals(targetStatus) ? effectivePatch.getErrorCode() : null); - task.setErrorMessage(TaskStatus.FAILED.name().equals(targetStatus) ? effectivePatch.getErrorMessage() : null); - task.setArtifactId(TaskStatus.SUCCESS.name().equals(targetStatus) ? effectivePatch.getArtifactId() : null); - if (effectivePatch.getStartedAt() != null) { - task.setStartedAt(effectivePatch.getStartedAt()); - } - if (effectivePatch.getFinishedAt() != null) { - task.setFinishedAt(effectivePatch.getFinishedAt()); - } - task.setUpdatedAt(effectivePatch.getUpdatedAt() == null ? new Date() : effectivePatch.getUpdatedAt()); - } - - private boolean isLegalTransition(String source, String target) { - if (TaskStatus.PENDING.name().equals(source)) { - return TaskStatus.RUNNING.name().equals(target) || TaskStatus.FAILED.name().equals(target); - } - if (TaskStatus.RUNNING.name().equals(source)) { - return TaskStatus.SUCCESS.name().equals(target) || TaskStatus.FAILED.name().equals(target); - } - if (LEGACY_CANCELLING_STATUS.equals(source)) { - return TaskStatus.FAILED.name().equals(target); - } - return false; - } - private File eventsFile(Long taskId) { return new File(snapshots.storageDirectory(), taskId + TASK_EVENT_FILE_SUFFIX); } @@ -712,6 +752,7 @@ private void copyInto(Task source, Task target) { target.setStage(copy.getStage()); target.setProgressMessage(copy.getProgressMessage()); target.setTarget(copy.getTarget()); + target.setSpecJson(copy.getSpecJson()); target.setErrorCode(copy.getErrorCode()); target.setErrorMessage(copy.getErrorMessage()); target.setArtifactId(copy.getArtifactId()); diff --git a/chat2db-community-server/chat2db-community-storage/src/main/java/ai/chat2db/community/storage/task/H2TaskStorage.java b/chat2db-community-server/chat2db-community-storage/src/main/java/ai/chat2db/community/storage/task/H2TaskStorage.java new file mode 100644 index 0000000000..2fe6bd770a --- /dev/null +++ b/chat2db-community-server/chat2db-community-storage/src/main/java/ai/chat2db/community/storage/task/H2TaskStorage.java @@ -0,0 +1,587 @@ +package ai.chat2db.community.storage.task; + +import ai.chat2db.community.domain.api.model.PageResponse; +import ai.chat2db.community.domain.api.model.task.ResumeState; +import ai.chat2db.community.domain.api.model.task.Task; +import ai.chat2db.community.domain.api.model.task.TaskArtifact; +import ai.chat2db.community.domain.api.model.task.TaskConstants; +import ai.chat2db.community.domain.api.model.task.TaskEvent; +import ai.chat2db.community.domain.api.model.task.TaskProgress; +import ai.chat2db.community.domain.api.model.task.TaskQuery; +import ai.chat2db.community.domain.api.model.task.TaskStatus; +import ai.chat2db.community.domain.api.model.task.TaskStatusPatch; +import ai.chat2db.community.domain.api.service.task.TaskStorage; +import ai.chat2db.community.storage.IdUtil; +import ai.chat2db.community.storage.TaskLifecyclePolicy; +import com.alibaba.fastjson2.JSON; +import lombok.extern.slf4j.Slf4j; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Types; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Date; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +/** + * Task storage backed by an embedded H2 file database. + * + *

Compared with {@code FileTaskStorage} this removes the single instance-wide monitor and the + * per-event {@code fsync}: task rows are locked one at a time with {@code SELECT ... FOR UPDATE}, + * and durability comes from the transaction commit instead of a forced write per appended event. + */ +@Slf4j +public class H2TaskStorage implements TaskStorage, AutoCloseable { + + private static final String TERMINAL_TASK_FILTER = Arrays.stream(TaskStatus.values()) + .filter(TaskStatus::isTerminal) + .map(status -> "'" + status.name() + "'") + .collect(Collectors.joining(", ", + " WHERE status IS NOT NULL AND status NOT IN (", ")")); + + private final TaskDatabase database; + + public H2TaskStorage() { + this(TaskDatabase.defaultStorageBasePath()); + } + + H2TaskStorage(String storageBasePath) { + this(new TaskDatabase(storageBasePath)); + } + + H2TaskStorage(TaskDatabase database) { + this.database = database; + this.database.initialize(); + } + + @Override + public Task create(Task task, TaskEvent createdEvent) { + if (task == null || createdEvent == null) { + throw new IllegalArgumentException("task and createdEvent are required"); + } + Task stored = copy(task); + Date now = new Date(); + stored.setStatus(TaskStatus.PENDING.name()); + stored.setProgress(TaskConstants.PENDING_PROGRESS); + stored.setCreatedAt(now); + stored.setUpdatedAt(now); + transact(connection -> { + Long taskId = nextTaskId(connection); + stored.setId(taskId); + TaskRows.insertTask(connection, stored, 1L); + TaskRows.insertEvent(connection, prepareEvent(createdEvent, taskId, 1L)); + return null; + }); + copyInto(stored, task); + return copy(stored); + } + + @Override + public Optional get(Long taskId) { + if (taskId == null) { + return Optional.empty(); + } + return transact(connection -> { + Task task = readTask(connection, TaskRows.SELECT_TASK_BY_ID, taskId); + if (task != null) { + task.setArtifacts(readArtifacts(connection, taskId)); + } + return Optional.ofNullable(task); + }); + } + + @Override + public PageResponse list(TaskQuery query) { + TaskQuery effectiveQuery = query == null ? new TaskQuery() : query; + int pageNo = Math.max(1, effectiveQuery.getPageNo() == null ? 1 : effectiveQuery.getPageNo()); + int pageSize = Math.max(1, effectiveQuery.getPageSize() == null + ? TaskConstants.DEFAULT_PAGE_SIZE : effectiveQuery.getPageSize()); + long offset = (long) (pageNo - 1) * pageSize; + return transact(connection -> { + long total = countTasks(connection, effectiveQuery); + List page = total > offset + ? selectTasks(connection, effectiveQuery, offset, pageSize) + : List.of(); + return PageResponse.of(List.copyOf(page), total, pageNo, pageSize); + }); + } + + @Override + public boolean compareAndSetStatus(Long taskId, String expectedStatus, String targetStatus, + TaskStatusPatch patch, TaskEvent lifecycleEvent) { + if (taskId == null) { + return false; + } + return transact(connection -> { + StoredTask current = readStoredTask(connection, taskId, true); + if (current == null || !expectedStatus.equals(current.task().getStatus()) + || !TaskLifecyclePolicy.isLegalTransition(expectedStatus, targetStatus, patch)) { + return false; + } + if (lifecycleEvent == null) { + throw new IllegalArgumentException("A status transition requires a lifecycle event"); + } + Task updated = copy(current.task()); + TaskLifecyclePolicy.applyStatusPatch(updated, targetStatus, patch); + long sequence = current.lastEventSequence() + 1L; + TaskRows.updateTask(connection, updated, sequence); + TaskRows.insertEvent(connection, prepareEvent(lifecycleEvent, taskId, sequence)); + return true; + }); + } + + @Override + public boolean updateProgressIfRunning(Long taskId, TaskProgress progress) { + if (taskId == null || progress == null || progress.getProgress() == null) { + return false; + } + int requested = TaskLifecyclePolicy.runningProgress(progress.getProgress()); + return transact(connection -> { + try (PreparedStatement statement = connection.prepareStatement( + "UPDATE task SET progress = ?, stage = ?, progress_message = ?, updated_at = ?" + + " WHERE id = ? AND status = ? AND progress <= ?")) { + statement.setInt(1, requested); + statement.setString(2, progress.getStage()); + statement.setString(3, progress.getMessage()); + statement.setLong(4, System.currentTimeMillis()); + statement.setLong(5, taskId); + statement.setString(6, TaskStatus.RUNNING.name()); + statement.setInt(7, requested); + return statement.executeUpdate() == 1; + } + }); + } + + @Override + public TaskEvent appendEvent(TaskEvent event) { + if (event == null || event.getTaskId() == null) { + throw new IllegalArgumentException("event must reference an existing task"); + } + return transact(connection -> { + Long sequence = nextEventSequence(connection, event.getTaskId()); + if (sequence == null) { + throw new IllegalArgumentException("event must reference an existing task"); + } + TaskEvent prepared = prepareEvent(event, event.getTaskId(), sequence); + TaskRows.insertEvent(connection, prepared); + return copyEvent(prepared); + }); + } + + @Override + public List listEvents(Long taskId, long afterSequence, int limit) { + if (taskId == null) { + return List.of(); + } + return transact(connection -> selectEvents(connection, + "SELECT " + TaskRows.EVENT_COLUMNS + " FROM task_event" + + " WHERE task_id = ? AND sequence > ? ORDER BY sequence" + + " FETCH FIRST ? ROWS ONLY", + statement -> { + statement.setLong(1, taskId); + statement.setLong(2, afterSequence); + statement.setInt(3, TaskLifecyclePolicy.eventLimit(limit)); + })); + } + + @Override + public List listEventsBefore(Long taskId, Long beforeSequence, int limit) { + if (taskId == null) { + return List.of(); + } + return transact(connection -> { + List newestFirst = selectEvents(connection, + "SELECT " + TaskRows.EVENT_COLUMNS + " FROM task_event" + + " WHERE task_id = ?" + (beforeSequence == null ? "" : " AND sequence < ?") + + " ORDER BY sequence DESC FETCH FIRST ? ROWS ONLY", + statement -> { + statement.setLong(1, taskId); + if (beforeSequence == null) { + statement.setInt(2, TaskLifecyclePolicy.eventLimit(limit)); + } else { + statement.setLong(2, beforeSequence); + statement.setInt(3, TaskLifecyclePolicy.eventLimit(limit)); + } + }); + Collections.reverse(newestFirst); + return newestFirst; + }); + } + + @Override + public List listNonTerminalTasks() { + return transact(connection -> selectTasks(connection, + "SELECT " + TaskRows.TASK_COLUMNS + " FROM task" + TERMINAL_TASK_FILTER + " ORDER BY id")); + } + + @Override + public List listTasksForRecovery() { + return transact(connection -> selectTasks(connection, + "SELECT " + TaskRows.TASK_COLUMNS + " FROM task ORDER BY id")); + } + + @Override + public boolean deleteTerminalTask(Long taskId, Runnable commitAction) { + if (taskId == null) { + return false; + } + // The external commit runs before the database commit, so a failure leaves both the task row + // and its events untouched instead of requiring a compensating restore. + return transact(connection -> { + StoredTask current = readStoredTask(connection, taskId, true); + if (current == null || !TaskStatus.isTerminal(current.task().getStatus())) { + return false; + } + executeUpdate(connection, "DELETE FROM task_event WHERE task_id = ?", + statement -> statement.setLong(1, taskId)); + executeUpdate(connection, "DELETE FROM task_artifact WHERE task_id = ?", + statement -> statement.setLong(1, taskId)); + executeUpdate(connection, "DELETE FROM resume_state WHERE task_id = ?", + statement -> statement.setLong(1, taskId)); + executeUpdate(connection, "DELETE FROM task WHERE id = ?", statement -> statement.setLong(1, taskId)); + if (commitAction != null) { + commitAction.run(); + } + return true; + }); + } + + @Override + public List listArtifacts(Long taskId) { + if (taskId == null) { + return List.of(); + } + return transact(connection -> readArtifacts(connection, taskId)); + } + + @Override + public void saveArtifact(Long taskId, TaskArtifact artifact) { + if (taskId == null || artifact == null || artifact.getArtifactId() == null || artifact.getRole() == null) { + throw new IllegalArgumentException("artifact must reference an existing task"); + } + transact(connection -> { + requireTask(connection, taskId, "artifact"); + TaskRows.upsertArtifact(connection, taskId, artifact); + return null; + }); + } + + @Override + public void deleteArtifact(Long taskId, String artifactId) { + if (taskId == null || artifactId == null) { + return; + } + transact(connection -> executeUpdate(connection, + "DELETE FROM task_artifact WHERE task_id = ? AND artifact_id = ?", statement -> { + statement.setLong(1, taskId); + statement.setString(2, artifactId); + })); + } + + @Override + public List listResumableTasks() { + return transact(connection -> selectTasks(connection, + "SELECT " + TaskRows.TASK_COLUMNS + " FROM task" + TERMINAL_TASK_FILTER + + " AND EXISTS (SELECT 1 FROM resume_state rs WHERE rs.task_id = task.id) ORDER BY id")); + } + + @Override + public void saveResumeState(Long taskId, ResumeState state) { + if (taskId == null || state == null || state.getShardNo() == null || state.getKind() == null) { + throw new IllegalArgumentException("resume state must reference an existing task"); + } + transact(connection -> { + requireTask(connection, taskId, "resume state"); + try (PreparedStatement statement = connection.prepareStatement( + "MERGE INTO resume_state (task_id, shard_no, kind, cursor_json, rows_done, bytes_done, updated_at)" + + " KEY(task_id, shard_no) VALUES (?, ?, ?, ?, ?, ?, ?)")) { + statement.setLong(1, taskId); + statement.setInt(2, state.getShardNo()); + statement.setString(3, state.getKind()); + statement.setString(4, state.getCursorJson()); + setNullableLong(statement, 5, state.getRowsDone()); + setNullableLong(statement, 6, state.getBytesDone()); + statement.setLong(7, (state.getUpdatedAt() == null ? new Date() : state.getUpdatedAt()).getTime()); + statement.executeUpdate(); + } + return null; + }); + } + + @Override + public List listResumeStates(Long taskId) { + if (taskId == null) { + return List.of(); + } + return transact(connection -> { + try (PreparedStatement statement = connection.prepareStatement( + "SELECT shard_no, kind, cursor_json, rows_done, bytes_done, updated_at FROM resume_state" + + " WHERE task_id = ? ORDER BY shard_no")) { + statement.setLong(1, taskId); + try (ResultSet rows = statement.executeQuery()) { + List states = new ArrayList<>(); + while (rows.next()) { + states.add(ResumeState.builder() + .shardNo(rows.getInt("shard_no")) + .kind(rows.getString("kind")) + .cursorJson(rows.getString("cursor_json")) + .rowsDone(TaskRows.getNullableLong(rows, "rows_done")) + .bytesDone(TaskRows.getNullableLong(rows, "bytes_done")) + .updatedAt(TaskRows.getNullableDate(rows, "updated_at")) + .build()); + } + return states; + } + } + }); + } + + @Override + public void clearResumeStates(Long taskId) { + if (taskId == null) { + return; + } + transact(connection -> executeUpdate(connection, "DELETE FROM resume_state WHERE task_id = ?", + statement -> statement.setLong(1, taskId))); + } + + public void close() { + database.close(); + } + + private List readArtifacts(Connection connection, Long taskId) throws SQLException { + try (PreparedStatement statement = connection.prepareStatement(TaskRows.SELECT_ARTIFACTS)) { + statement.setLong(1, taskId); + try (ResultSet rows = statement.executeQuery()) { + return TaskRows.readArtifacts(rows); + } + } + } + + private void requireTask(Connection connection, Long taskId, String subject) throws SQLException { + try (PreparedStatement statement = connection.prepareStatement("SELECT 1 FROM task WHERE id = ?")) { + statement.setLong(1, taskId); + try (ResultSet rows = statement.executeQuery()) { + if (!rows.next()) { + throw new IllegalArgumentException(subject + " must reference an existing task"); + } + } + } + } + + private Long nextTaskId(Connection connection) throws SQLException { + Long taskId; + do { + taskId = IdUtil.generateId(); + } while (readTask(connection, TaskRows.SELECT_TASK_BY_ID, taskId) != null); + return taskId; + } + + /** + * Reserves the next event sequence for a task. The update locks the task row until this + * transaction ends, so concurrent appends cannot be handed the same sequence. + */ + private Long nextEventSequence(Connection connection, Long taskId) throws SQLException { + if (executeUpdate(connection, "UPDATE task SET last_event_sequence = last_event_sequence + 1 WHERE id = ?", + statement -> statement.setLong(1, taskId)) == 0) { + return null; + } + try (PreparedStatement statement = connection.prepareStatement( + "SELECT last_event_sequence FROM task WHERE id = ?")) { + statement.setLong(1, taskId); + try (ResultSet rows = statement.executeQuery()) { + rows.next(); + return rows.getLong(1); + } + } + } + + private StoredTask readStoredTask(Connection connection, Long taskId, boolean forUpdate) throws SQLException { + try (PreparedStatement statement = connection.prepareStatement(forUpdate + ? TaskRows.SELECT_TASK_BY_ID_FOR_UPDATE : TaskRows.SELECT_TASK_BY_ID)) { + statement.setLong(1, taskId); + try (ResultSet rows = statement.executeQuery()) { + return rows.next() ? new StoredTask(TaskRows.readTask(rows), TaskRows.lastEventSequence(rows)) : null; + } + } + } + + private Task readTask(Connection connection, String sql, Long taskId) throws SQLException { + try (PreparedStatement statement = connection.prepareStatement(sql)) { + statement.setLong(1, taskId); + try (ResultSet rows = statement.executeQuery()) { + return rows.next() ? TaskRows.readTask(rows) : null; + } + } + } + + private long countTasks(Connection connection, TaskQuery query) throws SQLException { + try (PreparedStatement statement = connection.prepareStatement( + "SELECT COUNT(*) FROM task" + taskFilterSql(query))) { + bindFilter(statement, query); + try (ResultSet rows = statement.executeQuery()) { + rows.next(); + return rows.getLong(1); + } + } + } + + private List selectTasks(Connection connection, TaskQuery query, long offset, int pageSize) + throws SQLException { + return selectTasks(connection, "SELECT " + TaskRows.TASK_COLUMNS + " FROM task" + taskFilterSql(query) + + " ORDER BY id DESC OFFSET ? ROWS FETCH NEXT ? ROWS ONLY", statement -> { + int index = bindFilter(statement, query); + statement.setLong(index++, offset); + statement.setInt(index, pageSize); + }); + } + + private List selectTasks(Connection connection, String sql) throws SQLException { + return selectTasks(connection, sql, statement -> { + }); + } + + private List selectTasks(Connection connection, String sql, Binder binder) throws SQLException { + try (PreparedStatement statement = connection.prepareStatement(sql)) { + binder.bind(statement); + try (ResultSet rows = statement.executeQuery()) { + List tasks = new ArrayList<>(); + while (rows.next()) { + tasks.add(TaskRows.readTask(rows)); + } + return tasks; + } + } + } + + private List selectEvents(Connection connection, String sql, Binder binder) throws SQLException { + try (PreparedStatement statement = connection.prepareStatement(sql)) { + binder.bind(statement); + try (ResultSet rows = statement.executeQuery()) { + List events = new ArrayList<>(); + while (rows.next()) { + events.add(TaskRows.readEvent(rows)); + } + return events; + } + } + } + + private int bindFilter(PreparedStatement statement, TaskQuery query) throws SQLException { + int index = 0; + if (query.getStatus() != null) { + statement.setString(++index, query.getStatus()); + } + setNullableLong(statement, ++index, query.getUserId()); + setNullableLong(statement, ++index, query.getOrganizationId()); + return index + 1; + } + + /** + * An absent status means "any status", while an absent user or organization means "unscoped", so + * only the latter two compare against null. + */ + private String taskFilterSql(TaskQuery query) { + return query.getStatus() == null + ? " WHERE user_id IS NOT DISTINCT FROM ? AND organization_id IS NOT DISTINCT FROM ?" + : " WHERE status = ? AND user_id IS NOT DISTINCT FROM ?" + + " AND organization_id IS NOT DISTINCT FROM ?"; + } + + /** + * {@code IS NOT DISTINCT FROM} needs the parameter type even when the value is null, otherwise + * H2 cannot infer it from an untyped placeholder. + */ + private void setNullableLong(PreparedStatement statement, int index, Long value) throws SQLException { + if (value == null) { + statement.setNull(index, Types.BIGINT); + } else { + statement.setLong(index, value); + } + } + + private int executeUpdate(Connection connection, String sql, Binder binder) throws SQLException { + try (PreparedStatement statement = connection.prepareStatement(sql)) { + binder.bind(statement); + return statement.executeUpdate(); + } + } + + private TaskEvent prepareEvent(TaskEvent source, Long taskId, long sequence) { + TaskEvent event = copyEvent(source); + event.setTaskId(taskId); + event.setEventId(event.getEventId() == null ? IdUtil.generateId() : event.getEventId()); + event.setSequence(sequence); + event.setCreatedAt(event.getCreatedAt() == null ? new Date() : event.getCreatedAt()); + return event; + } + + private T transact(SqlWork work) { + try (Connection connection = database.open()) { + try { + T result = work.run(connection); + connection.commit(); + return result; + } catch (Exception e) { + try { + connection.rollback(); + } catch (SQLException rollbackFailure) { + e.addSuppressed(rollbackFailure); + } + if (e instanceof RuntimeException runtimeException) { + throw runtimeException; + } + throw new IllegalStateException("Task storage operation failed", e); + } + } catch (SQLException e) { + throw new IllegalStateException("Could not access task storage", e); + } + } + + private Task copy(Task task) { + return JSON.parseObject(JSON.toJSONString(task), Task.class); + } + + private TaskEvent copyEvent(TaskEvent event) { + return JSON.parseObject(JSON.toJSONString(event), TaskEvent.class); + } + + private void copyInto(Task source, Task target) { + Task copy = copy(source); + target.setId(copy.getId()); + target.setType(copy.getType()); + target.setName(copy.getName()); + target.setStatus(copy.getStatus()); + target.setProgress(copy.getProgress()); + target.setStage(copy.getStage()); + target.setProgressMessage(copy.getProgressMessage()); + target.setTarget(copy.getTarget()); + target.setSpecJson(copy.getSpecJson()); + target.setErrorCode(copy.getErrorCode()); + target.setErrorMessage(copy.getErrorMessage()); + target.setArtifactId(copy.getArtifactId()); + target.setUserId(copy.getUserId()); + target.setOrganizationId(copy.getOrganizationId()); + target.setCreatedAt(copy.getCreatedAt()); + target.setStartedAt(copy.getStartedAt()); + target.setFinishedAt(copy.getFinishedAt()); + target.setUpdatedAt(copy.getUpdatedAt()); + } + + private interface Binder { + void bind(PreparedStatement statement) throws SQLException; + } + + private interface SqlWork { + T run(Connection connection) throws Exception; + } + + private record StoredTask(Task task, long lastEventSequence) { + } +} diff --git a/chat2db-community-server/chat2db-community-storage/src/main/java/ai/chat2db/community/storage/task/TaskDatabase.java b/chat2db-community-server/chat2db-community-storage/src/main/java/ai/chat2db/community/storage/task/TaskDatabase.java new file mode 100644 index 0000000000..0c94b3a4d3 --- /dev/null +++ b/chat2db-community-server/chat2db-community-storage/src/main/java/ai/chat2db/community/storage/task/TaskDatabase.java @@ -0,0 +1,195 @@ +package ai.chat2db.community.storage.task; + +import ai.chat2db.community.tools.util.ConfigUtils; +import cn.hutool.core.io.FileUtil; +import lombok.extern.slf4j.Slf4j; + +import java.io.File; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; + +/** + * Owns the embedded H2 file database behind {@link H2TaskStorage}: the JDBC url, connection + * creation and schema bootstrap. + */ +@Slf4j +final class TaskDatabase implements AutoCloseable { + + static final int SCHEMA_VERSION = 3; + + /** + * Sibling of the {@code task-v2} directory written by {@code FileTaskStorage}, so every task + * storage generation lives under the same storage root. + */ + static final String DATABASE_DIRECTORY = "task-h2"; + + private static final String SCHEMA_VERSION_KEY = "schema_version"; + + private static final String[] SCHEMA_SQL = { + "CREATE TABLE IF NOT EXISTS task (" + + "id BIGINT PRIMARY KEY," + + "type VARCHAR(64)," + + "name CLOB," + + "status VARCHAR(64) NOT NULL," + + "progress INT NOT NULL," + + "stage VARCHAR(64)," + + "progress_message CLOB," + + "error_code VARCHAR(64)," + + "error_message CLOB," + + "artifact_id CLOB," + + "target_json CLOB," + + "spec_json CLOB," + + "user_id BIGINT," + + "organization_id BIGINT," + + "created_at BIGINT," + + "started_at BIGINT," + + "finished_at BIGINT," + + "updated_at BIGINT," + + "last_event_sequence BIGINT NOT NULL DEFAULT 0)", + "CREATE INDEX IF NOT EXISTS idx_task_scope ON task(user_id, organization_id, status)", + "CREATE TABLE IF NOT EXISTS task_event (" + + "task_id BIGINT NOT NULL," + + "sequence BIGINT NOT NULL," + + "event_id BIGINT," + + "level VARCHAR(64)," + + "code VARCHAR(64)," + + "stage VARCHAR(64)," + + "message CLOB," + + "details CLOB," + + "created_at BIGINT," + + "PRIMARY KEY (task_id, sequence))", + "CREATE TABLE IF NOT EXISTS task_artifact (" + + "task_id BIGINT NOT NULL," + + "artifact_id VARCHAR(1024) NOT NULL," + + "role VARCHAR(32) NOT NULL," + + "media_type VARCHAR(128)," + + "size_bytes BIGINT," + + "created_at BIGINT," + + "PRIMARY KEY (task_id, artifact_id))", + "CREATE TABLE IF NOT EXISTS resume_state (" + + "task_id BIGINT NOT NULL," + + "shard_no INT NOT NULL," + + "kind VARCHAR(32) NOT NULL," + + "cursor_json CLOB," + + "rows_done BIGINT," + + "bytes_done BIGINT," + + "updated_at BIGINT," + + "PRIMARY KEY (task_id, shard_no))", + "CREATE TABLE IF NOT EXISTS schema_meta (" + + "meta_key VARCHAR(64) PRIMARY KEY," + + "meta_value CLOB)", + }; + + /** + * Statements that add columns to databases created by an older schema version; idempotent so + * they are safe on a freshly created database as well. + */ + private static final String[] UPGRADE_SQL = { + "ALTER TABLE task ADD COLUMN IF NOT EXISTS spec_json CLOB", + }; + + private final String jdbcUrl; + + private boolean initialized; + + TaskDatabase(String storageBasePath) { + File directory = new File(storageBasePath, DATABASE_DIRECTORY); + FileUtil.mkdir(directory); + String databaseFile = new File(directory, "task").getAbsolutePath().replace(File.separatorChar, '/'); + // DB_CLOSE_DELAY keeps the store open between operations, so a connection per operation does + // not pay for re-opening the MVStore; LOCK_TIMEOUT covers serialized event-sequence allocation. + this.jdbcUrl = "jdbc:h2:" + databaseFile + ";DB_CLOSE_DELAY=-1;LOCK_TIMEOUT=10000"; + } + + static String defaultStorageBasePath() { + return ConfigUtils.getEnvBasePath() + File.separator + "storage"; + } + + /** + * Opens a connection with manual commit, so every write path has to decide explicitly where its + * transaction boundary is. + */ + Connection open() throws SQLException { + Connection connection = DriverManager.getConnection(jdbcUrl, null, null); + connection.setAutoCommit(false); + return connection; + } + + synchronized void initialize() { + if (initialized) { + return; + } + loadDriver(); + try (Connection connection = open()) { + try (Statement statement = connection.createStatement()) { + for (String sql : SCHEMA_SQL) { + statement.execute(sql); + } + } + int stored = readSchemaVersion(connection); + if (stored > SCHEMA_VERSION) { + throw new IllegalStateException("Task storage schema version " + stored + + " is newer than this build supports (" + SCHEMA_VERSION + ")"); + } + if (stored > 0 && stored < SCHEMA_VERSION) { + try (Statement statement = connection.createStatement()) { + for (String sql : UPGRADE_SQL) { + statement.execute(sql); + } + } + } + if (stored != SCHEMA_VERSION) { + writeSchemaVersion(connection); + } + connection.commit(); + } catch (SQLException e) { + throw new IllegalStateException("Could not initialize task storage", e); + } + initialized = true; + } + + private int readSchemaVersion(Connection connection) throws SQLException { + try (PreparedStatement statement = connection.prepareStatement( + "SELECT meta_value FROM schema_meta WHERE meta_key = ?")) { + statement.setString(1, SCHEMA_VERSION_KEY); + try (ResultSet rows = statement.executeQuery()) { + return rows.next() ? Integer.parseInt(rows.getString(1).trim()) : 0; + } + } + } + + private void writeSchemaVersion(Connection connection) throws SQLException { + try (PreparedStatement statement = connection.prepareStatement( + "MERGE INTO schema_meta (meta_key, meta_value) KEY(meta_key) VALUES (?, ?)")) { + statement.setString(1, SCHEMA_VERSION_KEY); + statement.setString(2, String.valueOf(SCHEMA_VERSION)); + statement.executeUpdate(); + } + } + + private void loadDriver() { + try { + Class.forName("org.h2.Driver"); + } catch (ClassNotFoundException e) { + throw new IllegalStateException("H2 driver is not available", e); + } + } + + @Override + public synchronized void close() { + if (!initialized) { + return; + } + try (Connection connection = DriverManager.getConnection(jdbcUrl, null, null); + Statement statement = connection.createStatement()) { + statement.execute("SHUTDOWN"); + } catch (SQLException e) { + log.warn("Could not close task storage cleanly", e); + } + initialized = false; + } +} diff --git a/chat2db-community-server/chat2db-community-storage/src/main/java/ai/chat2db/community/storage/task/TaskRows.java b/chat2db-community-server/chat2db-community-storage/src/main/java/ai/chat2db/community/storage/task/TaskRows.java new file mode 100644 index 0000000000..0672e79233 --- /dev/null +++ b/chat2db-community-server/chat2db-community-storage/src/main/java/ai/chat2db/community/storage/task/TaskRows.java @@ -0,0 +1,232 @@ +package ai.chat2db.community.storage.task; + +import ai.chat2db.community.domain.api.model.task.Task; +import ai.chat2db.community.domain.api.model.task.TaskArtifact; +import ai.chat2db.community.domain.api.model.task.TaskEvent; +import ai.chat2db.community.domain.api.model.task.TaskTargetSnapshot; +import com.alibaba.fastjson2.JSON; +import org.apache.commons.lang3.StringUtils; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Types; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; + +/** + * Row mapping shared by {@link H2TaskStorage} and {@link TaskStorageMigrator} so both write the + * same representation of a task. + */ +final class TaskRows { + + static final String TASK_COLUMNS = "id, type, name, status, progress, stage, progress_message," + + " error_code, error_message, artifact_id, target_json, spec_json, user_id, organization_id," + + " created_at, started_at, finished_at, updated_at, last_event_sequence"; + + static final String EVENT_COLUMNS = "task_id, sequence, event_id, level, code, stage, message," + + " details, created_at"; + + static final String INSERT_TASK = "INSERT INTO task (" + TASK_COLUMNS + ") VALUES (" + + placeholders(19) + ")"; + + static final String UPDATE_TASK = "UPDATE task SET type = ?, name = ?, status = ?, progress = ?," + + " stage = ?, progress_message = ?, error_code = ?, error_message = ?, artifact_id = ?," + + " target_json = ?, spec_json = ?, user_id = ?, organization_id = ?, created_at = ?," + + " started_at = ?, finished_at = ?, updated_at = ?, last_event_sequence = ? WHERE id = ?"; + + static final String INSERT_EVENT = "INSERT INTO task_event (" + EVENT_COLUMNS + ") VALUES (" + + placeholders(9) + ")"; + + static final String SELECT_TASK_BY_ID = "SELECT " + TASK_COLUMNS + " FROM task WHERE id = ?"; + + static final String SELECT_TASK_BY_ID_FOR_UPDATE = SELECT_TASK_BY_ID + " FOR UPDATE"; + + static final String UPSERT_ARTIFACT = "MERGE INTO task_artifact" + + " (task_id, artifact_id, role, media_type, size_bytes, created_at)" + + " KEY(task_id, artifact_id) VALUES (?, ?, ?, ?, ?, ?)"; + + static final String SELECT_ARTIFACTS = "SELECT artifact_id, role, media_type, size_bytes, created_at" + + " FROM task_artifact WHERE task_id = ? ORDER BY created_at, artifact_id"; + + private TaskRows() { + } + + static void upsertArtifact(Connection connection, Long taskId, TaskArtifact artifact) throws SQLException { + try (PreparedStatement statement = connection.prepareStatement(UPSERT_ARTIFACT)) { + statement.setLong(1, taskId); + statement.setString(2, artifact.getArtifactId()); + statement.setString(3, artifact.getRole()); + setString(statement, 4, artifact.getMediaType()); + setLong(statement, 5, artifact.getSizeBytes()); + setDate(statement, 6, artifact.getCreatedAt() == null ? new Date() : artifact.getCreatedAt()); + statement.executeUpdate(); + } + } + + static void insertTask(Connection connection, Task task, long lastEventSequence) throws SQLException { + try (PreparedStatement statement = connection.prepareStatement(INSERT_TASK)) { + bindTask(statement, task, lastEventSequence, false); + statement.executeUpdate(); + } + } + + static void updateTask(Connection connection, Task task, long lastEventSequence) throws SQLException { + try (PreparedStatement statement = connection.prepareStatement(UPDATE_TASK)) { + bindTask(statement, task, lastEventSequence, true); + statement.executeUpdate(); + } + } + + static void insertEvent(Connection connection, TaskEvent event) throws SQLException { + try (PreparedStatement statement = connection.prepareStatement(INSERT_EVENT)) { + bindEvent(statement, event); + statement.executeUpdate(); + } + } + + static Task readTask(ResultSet rows) throws SQLException { + String targetJson = rows.getString("target_json"); + return Task.builder() + .id(rows.getLong("id")) + .type(rows.getString("type")) + .name(rows.getString("name")) + .status(rows.getString("status")) + .progress(rows.getInt("progress")) + .stage(rows.getString("stage")) + .progressMessage(rows.getString("progress_message")) + .target(StringUtils.isBlank(targetJson) ? null + : JSON.parseObject(targetJson, TaskTargetSnapshot.class)) + .specJson(rows.getString("spec_json")) + .errorCode(rows.getString("error_code")) + .errorMessage(rows.getString("error_message")) + .artifactId(rows.getString("artifact_id")) + .userId(getNullableLong(rows, "user_id")) + .organizationId(getNullableLong(rows, "organization_id")) + .createdAt(getNullableDate(rows, "created_at")) + .startedAt(getNullableDate(rows, "started_at")) + .finishedAt(getNullableDate(rows, "finished_at")) + .updatedAt(getNullableDate(rows, "updated_at")) + .build(); + } + + static TaskEvent readEvent(ResultSet rows) throws SQLException { + String detailsJson = rows.getString("details"); + return TaskEvent.builder() + .eventId(getNullableLong(rows, "event_id")) + .taskId(getNullableLong(rows, "task_id")) + .sequence(rows.getLong("sequence")) + .level(rows.getString("level")) + .code(rows.getString("code")) + .stage(rows.getString("stage")) + .message(rows.getString("message")) + .details(StringUtils.isBlank(detailsJson) ? null : JSON.parseObject(detailsJson)) + .createdAt(getNullableDate(rows, "created_at")) + .build(); + } + + static long lastEventSequence(ResultSet rows) throws SQLException { + return rows.getLong("last_event_sequence"); + } + + static List readArtifacts(ResultSet rows) throws SQLException { + List artifacts = new ArrayList<>(); + while (rows.next()) { + artifacts.add(TaskArtifact.builder() + .artifactId(rows.getString("artifact_id")) + .role(rows.getString("role")) + .mediaType(rows.getString("media_type")) + .sizeBytes(getNullableLong(rows, "size_bytes")) + .createdAt(getNullableDate(rows, "created_at")) + .build()); + } + return artifacts; + } + + private static void bindTask(PreparedStatement statement, Task task, long lastEventSequence, + boolean forUpdate) throws SQLException { + int index = 1; + if (!forUpdate) { + statement.setLong(index++, task.getId()); + } + setString(statement, index++, task.getType()); + setString(statement, index++, task.getName()); + statement.setString(index++, task.getStatus()); + statement.setInt(index++, task.getProgress() == null ? 0 : task.getProgress()); + setString(statement, index++, task.getStage()); + setString(statement, index++, task.getProgressMessage()); + setString(statement, index++, task.getErrorCode()); + setString(statement, index++, task.getErrorMessage()); + setString(statement, index++, task.getArtifactId()); + setString(statement, index++, task.getTarget() == null ? null : JSON.toJSONString(task.getTarget())); + setString(statement, index++, task.getSpecJson()); + setLong(statement, index++, task.getUserId()); + setLong(statement, index++, task.getOrganizationId()); + index = setDate(statement, index, task.getCreatedAt()); + index = setDate(statement, index, task.getStartedAt()); + index = setDate(statement, index, task.getFinishedAt()); + index = setDate(statement, index, task.getUpdatedAt()); + statement.setLong(index++, lastEventSequence); + if (forUpdate) { + statement.setLong(index, task.getId()); + } + } + + private static void bindEvent(PreparedStatement statement, TaskEvent event) throws SQLException { + int index = 1; + statement.setLong(index++, event.getTaskId()); + statement.setLong(index++, event.getSequence()); + setLong(statement, index++, event.getEventId()); + setString(statement, index++, event.getLevel()); + setString(statement, index++, event.getCode()); + setString(statement, index++, event.getStage()); + setString(statement, index++, event.getMessage()); + setString(statement, index++, event.getDetails() == null ? null : JSON.toJSONString(event.getDetails())); + setDate(statement, index, event.getCreatedAt()); + } + + private static void setString(PreparedStatement statement, int index, String value) throws SQLException { + if (value == null) { + statement.setNull(index, Types.VARCHAR); + } else { + statement.setString(index, value); + } + } + + private static void setLong(PreparedStatement statement, int index, Long value) throws SQLException { + if (value == null) { + statement.setNull(index, Types.BIGINT); + } else { + statement.setLong(index, value); + } + } + + private static int setDate(PreparedStatement statement, int index, Date value) throws SQLException { + if (value == null) { + statement.setNull(index, Types.BIGINT); + } else { + statement.setLong(index, value.getTime()); + } + return index + 1; + } + + static Long getNullableLong(ResultSet rows, String column) throws SQLException { + long value = rows.getLong(column); + return rows.wasNull() ? null : value; + } + + static Date getNullableDate(ResultSet rows, String column) throws SQLException { + Long value = getNullableLong(rows, column); + return value == null ? null : new Date(value); + } + + private static String placeholders(int count) { + StringBuilder builder = new StringBuilder(); + for (int index = 0; index < count; index++) { + builder.append(index == 0 ? "?" : ", ?"); + } + return builder.toString(); + } +} diff --git a/chat2db-community-server/chat2db-community-storage/src/main/java/ai/chat2db/community/storage/task/TaskStorageConfiguration.java b/chat2db-community-server/chat2db-community-storage/src/main/java/ai/chat2db/community/storage/task/TaskStorageConfiguration.java new file mode 100644 index 0000000000..4e09f81bbc --- /dev/null +++ b/chat2db-community-server/chat2db-community-storage/src/main/java/ai/chat2db/community/storage/task/TaskStorageConfiguration.java @@ -0,0 +1,40 @@ +package ai.chat2db.community.storage.task; + +import ai.chat2db.community.domain.api.service.task.TaskStorage; +import ai.chat2db.community.storage.large.FileTaskStorage; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +/** + * Selects the task storage implementation. {@code chat2db.task.storage=h2} is the default; + * {@code -Dchat2db.task.storage=file} is the rollback switch that keeps the previous layout usable + * for one release after the H2 migration has run. + */ +@Configuration +public class TaskStorageConfiguration { + + static final String STORAGE_PROPERTY = "chat2db.task.storage"; + + /** + * SpEL instead of {@code @ConditionalOnProperty} so that only an exact opt-out selects the file + * layout: an unknown or mistyped value must not leave the application without any task storage. + */ + private static final String FILE_CONDITION = + "'${" + STORAGE_PROPERTY + ":h2}'.equalsIgnoreCase('file')"; + + @Bean + @ConditionalOnExpression(FILE_CONDITION) + public TaskStorage fileTaskStorage() { + return new FileTaskStorage(); + } + + @Bean(destroyMethod = "close") + @ConditionalOnExpression("!" + FILE_CONDITION) + public TaskStorage h2TaskStorage() { + String storageBasePath = TaskDatabase.defaultStorageBasePath(); + TaskDatabase database = new TaskDatabase(storageBasePath); + new TaskStorageMigrator(database, storageBasePath).migrateIfRequired(); + return new H2TaskStorage(database); + } +} diff --git a/chat2db-community-server/chat2db-community-storage/src/main/java/ai/chat2db/community/storage/task/TaskStorageMigrator.java b/chat2db-community-server/chat2db-community-storage/src/main/java/ai/chat2db/community/storage/task/TaskStorageMigrator.java new file mode 100644 index 0000000000..a1b71480d1 --- /dev/null +++ b/chat2db-community-server/chat2db-community-storage/src/main/java/ai/chat2db/community/storage/task/TaskStorageMigrator.java @@ -0,0 +1,240 @@ +package ai.chat2db.community.storage.task; + +import ai.chat2db.community.domain.api.model.task.Task; +import ai.chat2db.community.domain.api.model.task.TaskArtifact; +import ai.chat2db.community.domain.api.model.task.TaskArtifactRole; +import ai.chat2db.community.domain.api.model.task.TaskEvent; +import ai.chat2db.community.storage.large.FileTaskStorage; +import cn.hutool.core.io.FileUtil; +import com.alibaba.fastjson2.JSON; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +/** + * Imports the {@code task-v2} snapshot layout written by {@code FileTaskStorage} into the H2 schema. + * + *

The import and the completion marker are written by one transaction, so a task is either fully + * migrated or untouched, and a crash before the commit re-runs the whole import. The legacy + * directory is renamed rather than deleted so a bad migration stays recoverable. + */ +@Slf4j +public class TaskStorageMigrator { + + static final String MIGRATION_MARKER_KEY = "task_v2_migrated"; + + static final String MIGRATED_DIRECTORY_SUFFIX = ".migrated"; + + private static final String TASK_FILE_SUFFIX = ".json"; + + private final TaskDatabase database; + + private final File legacyDirectory; + + public TaskStorageMigrator(String storageBasePath) { + this(new TaskDatabase(storageBasePath), storageBasePath); + } + + TaskStorageMigrator(TaskDatabase database, String storageBasePath) { + this.database = database; + this.legacyDirectory = new File(storageBasePath, FileTaskStorage.TASK_STORAGE_DIRECTORY); + } + + /** + * @return the number of tasks imported; zero when there was nothing to migrate + */ + public int migrateIfRequired() { + database.initialize(); + if (isMigrated()) { + warnAboutFilesWrittenAfterMigration(); + return 0; + } + Map imported = readLegacyTasks(); + int count = importTasks(imported); + renameLegacyDirectory(imported.size()); + return count; + } + + private boolean isMigrated() { + try (Connection connection = database.open()) { + try (PreparedStatement statement = connection.prepareStatement( + "SELECT meta_value FROM schema_meta WHERE meta_key = ?")) { + statement.setString(1, MIGRATION_MARKER_KEY); + try (ResultSet rows = statement.executeQuery()) { + return rows.next(); + } + } finally { + connection.rollback(); + } + } catch (SQLException e) { + throw new IllegalStateException("Could not inspect task storage migration state", e); + } + } + + private Map readLegacyTasks() { + Map tasks = new LinkedHashMap<>(); + for (Long taskId : readLegacyIndex()) { + File snapshot = new File(legacyDirectory, taskId + TASK_FILE_SUFFIX); + if (!snapshot.isFile()) { + throw new IllegalStateException( + "Task " + taskId + " is listed in the legacy index but has no snapshot"); + } + Task task = parseTask(snapshot); + if (task == null || !taskId.equals(task.getId())) { + throw new IllegalStateException("Legacy task snapshot does not match its index: " + snapshot); + } + List events = readLegacyEvents(taskId); + long lastSequence = events.stream().mapToLong(TaskEvent::getSequence).max().orElse(0L); + tasks.put(taskId, new ImportedTask(task, events, lastSequence)); + } + return tasks; + } + + private List readLegacyIndex() { + File index = new File(legacyDirectory, + FileTaskStorage.TASK_INDEX_NAME + TASK_FILE_SUFFIX); + if (!index.isFile()) { + return List.of(); + } + return FileUtil.readLines(index, "UTF-8").stream() + .filter(StringUtils::isNotBlank) + .map(line -> { + try { + return Long.valueOf(line.trim()); + } catch (NumberFormatException e) { + throw new IllegalStateException("Unreadable legacy task index entry: " + line, e); + } + }) + .toList(); + } + + private Task parseTask(File file) { + try { + return JSON.parseObject(FileUtil.readUtf8String(file), Task.class); + } catch (RuntimeException e) { + throw new IllegalStateException("Could not parse legacy task snapshot " + file, e); + } + } + + private List readLegacyEvents(Long taskId) { + File file = new File(legacyDirectory, taskId + FileTaskStorage.TASK_EVENT_FILE_SUFFIX); + if (!file.isFile()) { + return List.of(); + } + return FileUtil.readLines(file, "UTF-8").stream() + .filter(StringUtils::isNotBlank) + .map(line -> parseEvent(line, taskId)) + .filter(Objects::nonNull) + .toList(); + } + + private TaskEvent parseEvent(String line, Long taskId) { + try { + TaskEvent event = JSON.parseObject(line, TaskEvent.class); + if (event == null || event.getSequence() == null || !taskId.equals(event.getTaskId())) { + log.warn("Skipping invalid legacy task event for task {}", taskId); + return null; + } + return event; + } catch (RuntimeException e) { + // A truncated trailing line is expected when the previous process was killed mid-append. + log.warn("Skipping unreadable legacy task event for task {}", taskId); + return null; + } + } + + private int importTasks(Map imported) { + try (Connection connection = database.open()) { + try { + for (ImportedTask entry : imported.values()) { + TaskRows.insertTask(connection, entry.task(), entry.lastSequence()); + for (TaskEvent event : entry.events()) { + TaskRows.insertEvent(connection, event); + } + for (TaskArtifact artifact : migratedArtifacts(entry.task())) { + TaskRows.upsertArtifact(connection, entry.task().getId(), artifact); + } + } + markMigrated(connection, imported.size()); + connection.commit(); + } catch (SQLException | RuntimeException e) { + connection.rollback(); + throw e; + } + return imported.size(); + } catch (SQLException e) { + throw new IllegalStateException("Could not import legacy task storage", e); + } + } + + /** + * Tasks created before artifacts were tracked recorded only the legacy single {@code artifactId}; + * write it as the primary output so download and delete see the same file set as for new tasks. + */ + private static List migratedArtifacts(Task task) { + List artifacts = new ArrayList<>( + task.getArtifacts() == null ? List.of() : task.getArtifacts()); + if (StringUtils.isNotBlank(task.getArtifactId()) && artifacts.stream() + .noneMatch(artifact -> task.getArtifactId().equals(artifact.getArtifactId()))) { + artifacts.add(TaskArtifact.builder() + .artifactId(task.getArtifactId()) + .role(TaskArtifactRole.OUTPUT) + .createdAt(task.getFinishedAt() == null ? task.getUpdatedAt() : task.getFinishedAt()) + .build()); + } + return artifacts; + } + + private void markMigrated(Connection connection, int taskCount) throws SQLException { + try (PreparedStatement statement = connection.prepareStatement( + "MERGE INTO schema_meta (meta_key, meta_value) KEY(meta_key) VALUES (?, ?)")) { + statement.setString(1, MIGRATION_MARKER_KEY); + statement.setString(2, "imported " + taskCount + " tasks at " + System.currentTimeMillis()); + statement.executeUpdate(); + } + } + + private void renameLegacyDirectory(int taskCount) { + if (!legacyDirectory.isDirectory()) { + return; + } + File migrated = new File(legacyDirectory.getParentFile(), + legacyDirectory.getName() + MIGRATED_DIRECTORY_SUFFIX); + if (migrated.exists()) { + migrated = new File(legacyDirectory.getParentFile(), + legacyDirectory.getName() + MIGRATED_DIRECTORY_SUFFIX + "-" + System.currentTimeMillis()); + } + try { + Files.move(legacyDirectory.toPath(), migrated.toPath()); + log.info("Imported {} tasks into H2 task storage and moved the legacy directory to {}", + taskCount, migrated); + } catch (IOException e) { + // The transaction already committed, so the next start must not import a second time; the + // directory is left in place and reported instead. + log.error("Imported {} tasks into H2 task storage but could not move {}; the legacy directory" + + " is now unused and must be removed manually", taskCount, legacyDirectory, e); + } + } + + private void warnAboutFilesWrittenAfterMigration() { + if (legacyDirectory.isDirectory()) { + log.warn("Task storage was already migrated to H2, but {} still exists. Tasks written while" + + " the file storage was selected again are not visible.", legacyDirectory); + } + } + + private record ImportedTask(Task task, List events, long lastSequence) { + } +} diff --git a/chat2db-community-server/chat2db-community-storage/src/test/java/ai/chat2db/community/storage/AbstractTaskStorageContractTest.java b/chat2db-community-server/chat2db-community-storage/src/test/java/ai/chat2db/community/storage/AbstractTaskStorageContractTest.java new file mode 100644 index 0000000000..ffa8fd4380 --- /dev/null +++ b/chat2db-community-server/chat2db-community-storage/src/test/java/ai/chat2db/community/storage/AbstractTaskStorageContractTest.java @@ -0,0 +1,623 @@ +package ai.chat2db.community.storage; + +import ai.chat2db.community.domain.api.model.PageResponse; +import ai.chat2db.community.domain.api.model.task.ResumeState; +import ai.chat2db.community.domain.api.model.task.Task; +import ai.chat2db.community.domain.api.model.task.TaskArtifact; +import ai.chat2db.community.domain.api.model.task.TaskArtifactRole; +import ai.chat2db.community.domain.api.model.task.TaskConstants; +import ai.chat2db.community.domain.api.model.task.TaskEvent; +import ai.chat2db.community.domain.api.model.task.TaskEventCode; +import ai.chat2db.community.domain.api.model.task.TaskEventLevel; +import ai.chat2db.community.domain.api.model.task.TaskProgress; +import ai.chat2db.community.domain.api.model.task.TaskQuery; +import ai.chat2db.community.domain.api.model.task.TaskStatus; +import ai.chat2db.community.domain.api.model.task.TaskStatusPatch; +import ai.chat2db.community.domain.api.model.task.TaskStage; +import ai.chat2db.community.domain.api.model.task.TaskTargetSnapshot; +import ai.chat2db.community.domain.api.service.task.TaskStorage; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.File; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.stream.Collectors; +import java.util.stream.LongStream; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Behaviour every {@link TaskStorage} implementation owes its callers, independent of where the + * rows live. Each implementation subclass supplies a fresh storage over {@link #baseDir}; calling + * {@link #storage()} more than once in one test therefore means "restart the application". + */ +public abstract class AbstractTaskStorageContractTest { + + @TempDir + protected File baseDir; + + private final List createdStorages = new ArrayList<>(); + + protected abstract TaskStorage createStorage(); + + protected TaskStorage storage() { + TaskStorage storage = createStorage(); + if (storage instanceof AutoCloseable closeable) { + createdStorages.add(closeable); + } + return storage; + } + + @AfterEach + void closeStorages() { + for (AutoCloseable storage : createdStorages) { + try { + storage.close(); + } catch (Exception e) { + // Storage shutdown is best effort; a locked file must not mask the real assertions. + } + } + createdStorages.clear(); + } + + @Test + void taskAndEventResultsAreDefensiveCopies() { + TaskStorage storage = storage(); + Task input = task("original"); + Task created = storage.create(input, event(TaskEventCode.TASK_CREATED.name())); + Long taskId = created.getId(); + + assertNotSame(input.getTarget(), created.getTarget()); + input.setName("mutated-input"); + input.getTarget().setTableName("mutated-input-table"); + created.setName("mutated-return"); + created.getTarget().setTableName("mutated-return-table"); + + Task fetched = storage.get(taskId).orElseThrow(); + assertEquals("original", fetched.getName()); + assertEquals("source_table", fetched.getTarget().getTableName()); + + fetched.setName("mutated-get"); + storage.list(query(1, 20)).getData().get(0).setName("mutated-list"); + storage.listNonTerminalTasks().get(0).setName("mutated-non-terminal"); + assertEquals("original", storage.get(taskId).orElseThrow().getName()); + + Map details = new HashMap<>(); + details.put("rows", 10); + TaskEvent event = event(TaskEventCode.QUERY_STARTED.name()); + event.setTaskId(taskId); + event.setDetails(details); + TaskEvent appended = storage.appendEvent(event); + details.put("rows", 99); + appended.getDetails().put("rows", 88); + + TaskEvent persisted = storage.listEvents(taskId, 1, 10).get(0); + assertEquals(10, persisted.getDetails().get("rows")); + persisted.getDetails().put("rows", 77); + assertEquals(10, storage.listEvents(taskId, 1, 10).get(0).getDetails().get("rows")); + } + + @Test + void compareAndSetEnforcesLegalTransitionsAndTerminalImmutability() { + TaskStorage storage = storage(); + Task created = create(storage, "task"); + Long taskId = created.getId(); + + assertTrue(start(storage, taskId)); + assertFalse(storage.compareAndSetStatus(taskId, TaskStatus.PENDING.name(), TaskStatus.FAILED.name(), + TaskStatusPatch.builder().errorCode("LATE").build(), event(TaskEventCode.TASK_FAILED.name()))); + + TaskStatusPatch success = TaskStatusPatch.builder() + .progress(40) + .artifactId("artifact-1") + .finishedAt(new Date()) + .build(); + assertTrue(storage.compareAndSetStatus(taskId, TaskStatus.RUNNING.name(), TaskStatus.SUCCESS.name(), + success, event(TaskEventCode.TASK_SUCCEEDED.name()))); + + Task finished = storage.get(taskId).orElseThrow(); + assertEquals(TaskStatus.SUCCESS.name(), finished.getStatus()); + assertEquals(TaskConstants.COMPLETED_PROGRESS, finished.getProgress()); + assertEquals("artifact-1", finished.getArtifactId()); + assertFalse(storage.compareAndSetStatus(taskId, TaskStatus.SUCCESS.name(), TaskStatus.FAILED.name(), + TaskStatusPatch.builder().errorCode("TOO_LATE").build(), event(TaskEventCode.TASK_FAILED.name()))); + assertEquals(TaskStatus.SUCCESS.name(), storage.get(taskId).orElseThrow().getStatus()); + assertEquals(List.of(1L, 2L, 3L), sequences(storage.listEvents(taskId, 0, 20))); + } + + @Test + void progressIsRunningOnlyMonotonicAndPreservedByFailure() { + TaskStorage storage = storage(); + Task created = create(storage, "task"); + Long taskId = created.getId(); + + assertFalse(storage.updateProgressIfRunning(taskId, progress(30, "query"))); + assertTrue(start(storage, taskId)); + assertTrue(storage.updateProgressIfRunning(taskId, progress(50, "query"))); + assertFalse(storage.updateProgressIfRunning(taskId, progress(40, "write"))); + assertEquals(50, storage.get(taskId).orElseThrow().getProgress()); + + assertTrue(storage.updateProgressIfRunning(taskId, progress(120, "write"))); + Task running = storage.get(taskId).orElseThrow(); + assertEquals(TaskConstants.MAX_RUNNING_PROGRESS, running.getProgress()); + assertEquals("write", running.getStage()); + + TaskStatusPatch failure = TaskStatusPatch.builder() + .progress(TaskConstants.COMPLETED_PROGRESS) + .errorCode("EXPORT_FAILED") + .errorMessage("failed") + .finishedAt(new Date()) + .build(); + assertTrue(storage.compareAndSetStatus(taskId, TaskStatus.RUNNING.name(), TaskStatus.FAILED.name(), + failure, event(TaskEventCode.TASK_FAILED.name()))); + Task failed = storage.get(taskId).orElseThrow(); + assertEquals(TaskConstants.MAX_RUNNING_PROGRESS, failed.getProgress()); + assertEquals("EXPORT_FAILED", failed.getErrorCode()); + assertFalse(storage.updateProgressIfRunning(taskId, progress(50, "late"))); + assertEquals(TaskConstants.MAX_RUNNING_PROGRESS, storage.get(taskId).orElseThrow().getProgress()); + } + + @Test + void terminalStatusRaceHasExactlyOneWinnerAndOneTerminalEvent() throws Exception { + TaskStorage storage = storage(); + Task created = create(storage, "task"); + Long taskId = created.getId(); + assertTrue(start(storage, taskId)); + ExecutorService executor = Executors.newFixedThreadPool(2); + CountDownLatch start = new CountDownLatch(1); + + try { + Future success = executor.submit(() -> { + start.await(); + return storage.compareAndSetStatus(taskId, TaskStatus.RUNNING.name(), TaskStatus.SUCCESS.name(), + TaskStatusPatch.builder().artifactId("artifact").build(), + event(TaskEventCode.TASK_SUCCEEDED.name())); + }); + Future failure = executor.submit(() -> { + start.await(); + return storage.compareAndSetStatus(taskId, TaskStatus.RUNNING.name(), TaskStatus.FAILED.name(), + TaskStatusPatch.builder().errorCode("FAILED").errorMessage("failed").build(), + event(TaskEventCode.TASK_FAILED.name())); + }); + + start.countDown(); + assertTrue(success.get() ^ failure.get(), "exactly one terminal transition must win"); + } finally { + executor.shutdownNow(); + } + + assertTrue(TaskStatus.isTerminal(storage.get(taskId).orElseThrow().getStatus())); + assertEquals(3, storage.listEvents(taskId, 0, 20).size()); + } + + @Test + void paginationIsStableNewestFirstWithIdTieBreaker() { + TaskStorage storage = storage(); + List created = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + created.add(create(storage, "task-" + i)); + } + Comparator newestFirst = Comparator + .comparing(Task::getCreatedAt, Comparator.nullsLast(Comparator.reverseOrder())) + .thenComparing(Task::getId, Comparator.nullsLast(Comparator.reverseOrder())); + List expected = created.stream().sorted(newestFirst).map(Task::getId).toList(); + + List firstPage = ids(storage.list(query(1, 2)).getData()); + List secondPage = ids(storage.list(query(2, 2)).getData()); + assertEquals(expected.subList(0, 2), firstPage); + assertEquals(expected.subList(2, 4), secondPage); + assertEquals(firstPage, ids(storage.list(query(1, 2)).getData())); + + TaskQuery hugePage = query(Integer.MAX_VALUE, 2); + PageResponse empty = storage.list(hugePage); + assertTrue(empty.getData().isEmpty()); + assertEquals(5L, empty.getTotal()); + } + + @Test + @SuppressWarnings("StringOperationCanBeSimplified") + void statusFilterUsesStringValueEquality() { + TaskStorage storage = storage(); + Task pending = create(storage, "pending"); + Task running = create(storage, "running"); + assertTrue(start(storage, running.getId())); + TaskQuery query = query(1, 20); + query.setStatus(new String(TaskStatus.RUNNING.name())); + + PageResponse page = storage.list(query); + + assertEquals(1L, page.getTotal()); + assertEquals(List.of(running.getId()), ids(page.getData())); + assertFalse(page.getData().stream().anyMatch(task -> pending.getId().equals(task.getId()))); + } + + @Test + void concurrentEventsReceiveUniqueOrderedSequencesAndSupportIncrementalReads() throws Exception { + TaskStorage storage = storage(); + Task created = create(storage, "task"); + Long taskId = created.getId(); + ExecutorService executor = Executors.newFixedThreadPool(6); + CountDownLatch start = new CountDownLatch(1); + List> futures = new ArrayList<>(); + + try { + for (int i = 0; i < 20; i++) { + futures.add(executor.submit(() -> { + start.await(); + TaskEvent event = event(TaskEventCode.QUERY_STARTED.name()); + event.setTaskId(taskId); + event.setSequence(999L); + return storage.appendEvent(event); + })); + } + start.countDown(); + + List returnedSequences = new ArrayList<>(); + for (Future future : futures) { + returnedSequences.add(future.get().getSequence()); + } + returnedSequences.sort(Long::compareTo); + assertEquals(LongStream.rangeClosed(2, 21).boxed().toList(), returnedSequences); + } finally { + executor.shutdownNow(); + } + + assertEquals(LongStream.rangeClosed(1, 21).boxed().toList(), + sequences(storage.listEvents(taskId, 0, 100))); + assertEquals(List.of(11L, 12L, 13L, 14L, 15L), + sequences(storage.listEvents(taskId, 10, 5))); + assertEquals(List.of(17L, 18L, 19L, 20L, 21L), + sequences(storage.listEventsBefore(taskId, null, 5))); + assertEquals(List.of(6L, 7L, 8L, 9L, 10L), + sequences(storage.listEventsBefore(taskId, 11L, 5))); + assertEquals(List.of(1L, 2L, 3L, 4L, 5L), + sequences(storage.listEventsBefore(taskId, 6L, 10))); + assertTrue(storage.listEventsBefore(taskId, 1L, 5).isEmpty()); + } + + @Test + void reversePaginationPreservesUtf8EventMessages() { + TaskStorage storage = storage(); + Task created = create(storage, "task"); + TaskEvent second = event(TaskEventCode.QUERY_STARTED.name()); + second.setTaskId(created.getId()); + second.setMessage("读取表数据"); + storage.appendEvent(second); + + List events = storage.listEventsBefore(created.getId(), null, 1); + + assertEquals(1, events.size()); + assertEquals("读取表数据", events.get(0).getMessage()); + } + + @Test + void incrementalReadStartsAfterGivenSequenceAndPreservesUtf8() { + TaskStorage storage = storage(); + Task created = create(storage, "task"); + TaskEvent second = event(TaskEventCode.QUERY_STARTED.name()); + second.setTaskId(created.getId()); + second.setMessage("已导出 1000 行"); + storage.appendEvent(second); + assertEquals(List.of(2L), sequences(storage.listEventsBefore(created.getId(), null, 1))); + TaskEvent third = event(TaskEventCode.QUERY_COMPLETED.name()); + third.setTaskId(created.getId()); + third.setMessage("导出完成"); + storage.appendEvent(third); + + List events = storage.listEvents(created.getId(), 2L, 10); + + assertEquals(List.of(3L), sequences(events)); + assertEquals("导出完成", events.get(0).getMessage()); + } + + @Test + void terminalTaskIsDeletedButActiveTaskIsRejected() { + TaskStorage storage = storage(); + Task task = create(storage, "task"); + Long taskId = task.getId(); + + assertFalse(storage.deleteTerminalTask(taskId, () -> {})); + assertTrue(storage.get(taskId).isPresent()); + + assertTrue(start(storage, taskId)); + assertTrue(storage.compareAndSetStatus(taskId, TaskStatus.RUNNING.name(), TaskStatus.SUCCESS.name(), + TaskStatusPatch.builder().artifactId("artifact").finishedAt(new Date()).build(), + event(TaskEventCode.TASK_SUCCEEDED.name()))); + assertTrue(storage.deleteTerminalTask(taskId, () -> {})); + + assertTrue(storage.get(taskId).isEmpty()); + assertTrue(storage.listEvents(taskId, 0, 10).isEmpty()); + assertTrue(storage.list(query(1, 20)).getData().isEmpty()); + + TaskStorage reloaded = storage(); + assertTrue(reloaded.get(taskId).isEmpty()); + assertTrue(reloaded.listEvents(taskId, 0, 10).isEmpty()); + } + + @Test + void failedTerminalTaskDeletionKeepsSnapshotAndEvents() { + TaskStorage storage = storage(); + Task task = create(storage, "task"); + Long taskId = task.getId(); + assertTrue(start(storage, taskId)); + assertTrue(storage.compareAndSetStatus(taskId, TaskStatus.RUNNING.name(), TaskStatus.SUCCESS.name(), + TaskStatusPatch.builder().artifactId("artifact").finishedAt(new Date()).build(), + event(TaskEventCode.TASK_SUCCEEDED.name()))); + + assertThrows(IllegalStateException.class, + () -> storage.deleteTerminalTask(taskId, () -> { + throw new IllegalStateException("artifact commit failed"); + })); + + assertEquals(TaskStatus.SUCCESS.name(), storage.get(taskId).orElseThrow().getStatus()); + assertEquals(List.of(1L, 2L, 3L), sequences(storage.listEvents(taskId, 0, 10))); + } + + @Test + void scopeFilterMatchesNullColumnsOnlyAgainstNullQueryValues() { + TaskStorage storage = storage(); + Task own = task("own"); + own.setUserId(1L); + own.setOrganizationId(2L); + Long ownId = storage.create(own, event(TaskEventCode.TASK_CREATED.name())).getId(); + Long unscopedId = create(storage, "unscoped").getId(); + + TaskQuery scoped = query(1, 20); + scoped.setUserId(1L); + scoped.setOrganizationId(2L); + assertEquals(List.of(ownId), ids(storage.list(scoped).getData())); + assertEquals(List.of(unscopedId), ids(storage.list(query(1, 20)).getData())); + + TaskQuery otherUser = query(1, 20); + otherUser.setUserId(99L); + otherUser.setOrganizationId(2L); + assertEquals(0L, storage.list(otherUser).getTotal()); + } + + @Test + void appendEventRejectsUnknownTask() { + TaskStorage storage = storage(); + create(storage, "task"); + TaskEvent event = event(TaskEventCode.QUERY_STARTED.name()); + event.setTaskId(-1L); + + assertThrows(IllegalArgumentException.class, () -> storage.appendEvent(event)); + } + + @Test + void statusTransitionWithoutLifecycleEventLeavesTaskUnchanged() { + TaskStorage storage = storage(); + Long taskId = create(storage, "task").getId(); + + assertThrows(IllegalArgumentException.class, () -> storage.compareAndSetStatus(taskId, + TaskStatus.PENDING.name(), TaskStatus.RUNNING.name(), + TaskStatusPatch.builder().progress(TaskConstants.STARTED_PROGRESS).build(), null)); + + assertEquals(TaskStatus.PENDING.name(), storage.get(taskId).orElseThrow().getStatus()); + assertEquals(List.of(1L), sequences(storage.listEvents(taskId, 0, 10))); + } + + @Test + void artifactsAreRecordedPerTaskReplacedByIdAndWipedWithTheTask() { + TaskStorage storage = storage(); + Long taskId = create(storage, "task").getId(); + Long otherTaskId = create(storage, "other").getId(); + storage.saveArtifact(taskId, artifact("artifact-1", TaskArtifactRole.OUTPUT, "text/csv", 10L)); + storage.saveArtifact(taskId, artifact("artifact-2", "REJECT", "application/x-ndjson", 20L)); + storage.saveArtifact(otherTaskId, artifact("artifact-1", TaskArtifactRole.OUTPUT, "text/csv", 5L)); + + assertEquals(Set.of("artifact-1", "artifact-2"), + storage.listArtifacts(taskId).stream().map(TaskArtifact::getArtifactId) + .collect(Collectors.toSet())); + assertEquals(List.of("artifact-1"), + storage.listArtifacts(otherTaskId).stream().map(TaskArtifact::getArtifactId).toList()); + + storage.saveArtifact(taskId, artifact("artifact-1", TaskArtifactRole.OUTPUT, "text/plain", 99L)); + List replaced = storage.listArtifacts(taskId); + assertEquals(2, replaced.size()); + TaskArtifact updated = replaced.stream() + .filter(candidate -> "artifact-1".equals(candidate.getArtifactId())) + .findFirst().orElseThrow(); + assertEquals("text/plain", updated.getMediaType()); + assertEquals(99L, updated.getSizeBytes()); + + storage.deleteArtifact(taskId, "artifact-2"); + storage.deleteArtifact(taskId, "missing-artifact"); + assertEquals(List.of("artifact-1"), storage.listArtifacts(taskId).stream() + .map(TaskArtifact::getArtifactId).toList()); + assertEquals(List.of("artifact-1"), storage.get(taskId).orElseThrow().getArtifacts().stream() + .map(TaskArtifact::getArtifactId).toList()); + } + + @Test + void artifactAndResumeStateRejectUnknownTasks() { + TaskStorage storage = storage(); + create(storage, "task"); + + assertThrows(IllegalArgumentException.class, + () -> storage.saveArtifact(-1L, artifact("artifact-1", TaskArtifactRole.OUTPUT, "text/csv", 1L))); + assertThrows(IllegalArgumentException.class, + () -> storage.saveResumeState(-1L, resumeState(0, 10L))); + } + + @Test + void terminalTaskDeletionRemovesArtifactsAndResumeStates() { + TaskStorage storage = storage(); + Long taskId = create(storage, "task").getId(); + assertTrue(start(storage, taskId)); + storage.saveArtifact(taskId, artifact("artifact-1", TaskArtifactRole.OUTPUT, "text/csv", 10L)); + storage.saveResumeState(taskId, resumeState(0, 100L)); + assertTrue(storage.compareAndSetStatus(taskId, TaskStatus.RUNNING.name(), TaskStatus.SUCCESS.name(), + TaskStatusPatch.builder().artifactIds(List.of("artifact-1")).finishedAt(new Date()).build(), + event(TaskEventCode.TASK_SUCCEEDED.name()))); + assertEquals("artifact-1", storage.get(taskId).orElseThrow().getArtifactId()); + + assertTrue(storage.deleteTerminalTask(taskId, () -> {})); + + TaskStorage reloaded = storage(); + assertTrue(reloaded.listArtifacts(taskId).isEmpty()); + assertTrue(reloaded.listResumeStates(taskId).isEmpty()); + } + + @Test + void nonTerminalTasksWithResumeStatesAreListedAsResumable() { + TaskStorage storage = storage(); + Long runningId = create(storage, "running").getId(); + Long finishedId = create(storage, "finished").getId(); + Long plainId = create(storage, "plain").getId(); + assertTrue(start(storage, runningId)); + assertTrue(start(storage, finishedId)); + assertTrue(storage.compareAndSetStatus(finishedId, TaskStatus.RUNNING.name(), TaskStatus.SUCCESS.name(), + TaskStatusPatch.builder().finishedAt(new Date()).build(), + event(TaskEventCode.TASK_SUCCEEDED.name()))); + + storage.saveResumeState(runningId, resumeState(0, 100L)); + storage.saveResumeState(finishedId, resumeState(0, 100L)); + storage.saveResumeState(finishedId, resumeState(1, 200L)); + + assertEquals(List.of(runningId), ids(storage.listResumableTasks())); + + storage.clearResumeStates(runningId); + assertTrue(storage.listResumeStates(runningId).isEmpty()); + assertTrue(storage.listResumableTasks().isEmpty()); + assertEquals(2, storage.listResumeStates(finishedId).size()); + } + + @Test + void resumeStatesKeepTheirFieldsAndAreSortedByShard() { + TaskStorage storage = storage(); + Long taskId = create(storage, "task").getId(); + storage.saveResumeState(taskId, resumeState(2, 300L)); + storage.saveResumeState(taskId, resumeState(0, 100L)); + storage.saveResumeState(taskId, resumeState(0, 250L)); + + List states = storage.listResumeStates(taskId); + + assertEquals(List.of(0, 2), states.stream().map(ResumeState::getShardNo).toList()); + assertEquals(250L, states.get(0).getRowsDone()); + assertEquals("KEYSET", states.get(1).getKind()); + assertEquals("{\"lastKey\":300}", states.get(1).getCursorJson()); + assertEquals(3000L, states.get(1).getBytesDone()); + } + + @Test + void runningTaskCanBeRequeuedToPendingForResume() { + TaskStorage storage = storage(); + Long taskId = create(storage, "task").getId(); + assertTrue(start(storage, taskId)); + + assertTrue(storage.compareAndSetStatus(taskId, TaskStatus.RUNNING.name(), TaskStatus.PENDING.name(), + TaskStatusPatch.builder().stage(TaskStage.RESUMING.name()).build(), + event(TaskEventCode.RESUME_AVAILABLE.name()))); + + Task requeued = storage.get(taskId).orElseThrow(); + assertEquals(TaskStatus.PENDING.name(), requeued.getStatus()); + assertEquals(TaskStage.RESUMING.name(), requeued.getStage()); + assertEquals(List.of(1L, 2L, 3L), sequences(storage.listEvents(taskId, 0, 20))); + } + + @Test + void runningTaskCannotBeRequeuedWithoutTheResumingStage() { + TaskStorage storage = storage(); + Long taskId = create(storage, "task").getId(); + assertTrue(start(storage, taskId)); + + assertFalse(storage.compareAndSetStatus(taskId, TaskStatus.RUNNING.name(), TaskStatus.PENDING.name(), + TaskStatusPatch.builder().stage(TaskStage.PENDING.name()).build(), + event(TaskEventCode.RESUME_AVAILABLE.name()))); + + Task unchanged = storage.get(taskId).orElseThrow(); + assertEquals(TaskStatus.RUNNING.name(), unchanged.getStatus()); + assertEquals(List.of(1L, 2L), sequences(storage.listEvents(taskId, 0, 20))); + } + + protected Task create(TaskStorage storage, String name) { + return storage.create(task(name), event(TaskEventCode.TASK_CREATED.name())); + } + + protected Task task(String name) { + return Task.builder() + .type("QUERY_RESULT_EXPORT") + .name(name) + .target(TaskTargetSnapshot.builder() + .dataSourceId(1L) + .databaseName("database") + .schemaName("schema") + .tableName("source_table") + .build()) + .build(); + } + + protected TaskEvent event(String code) { + return TaskEvent.builder() + .level(TaskEventLevel.INFO.name()) + .code(code) + .message(code) + .build(); + } + + protected boolean start(TaskStorage storage, Long taskId) { + return storage.compareAndSetStatus(taskId, TaskStatus.PENDING.name(), TaskStatus.RUNNING.name(), + TaskStatusPatch.builder() + .progress(TaskConstants.STARTED_PROGRESS) + .stage("started") + .startedAt(new Date()) + .build(), + event(TaskEventCode.TASK_STARTED.name())); + } + + protected TaskProgress progress(int value, String stage) { + return TaskProgress.builder().progress(value).stage(stage).message(stage).build(); + } + + protected TaskQuery query(int pageNo, int pageSize) { + TaskQuery query = new TaskQuery(); + query.setPageNo(pageNo); + query.setPageSize(pageSize); + return query; + } + + protected List sequences(List events) { + return events.stream().map(TaskEvent::getSequence).toList(); + } + + protected List ids(List tasks) { + return tasks.stream().map(Task::getId).toList(); + } + + protected TaskArtifact artifact(String artifactId, String role, String mediaType, Long sizeBytes) { + return TaskArtifact.builder() + .artifactId(artifactId) + .role(role) + .mediaType(mediaType) + .sizeBytes(sizeBytes) + .createdAt(new Date()) + .build(); + } + + protected ResumeState resumeState(int shardNo, long rowsDone) { + return ResumeState.builder() + .shardNo(shardNo) + .kind("KEYSET") + .cursorJson("{\"lastKey\":" + rowsDone + "}") + .rowsDone(rowsDone) + .bytesDone(rowsDone * 10L) + .updatedAt(new Date()) + .build(); + } +} diff --git a/chat2db-community-server/chat2db-community-storage/src/test/java/ai/chat2db/community/storage/large/FileTaskStorageTest.java b/chat2db-community-server/chat2db-community-storage/src/test/java/ai/chat2db/community/storage/large/FileTaskStorageTest.java index 79f4776613..7f9238bd18 100644 --- a/chat2db-community-server/chat2db-community-storage/src/test/java/ai/chat2db/community/storage/large/FileTaskStorageTest.java +++ b/chat2db-community-server/chat2db-community-storage/src/test/java/ai/chat2db/community/storage/large/FileTaskStorageTest.java @@ -1,47 +1,44 @@ package ai.chat2db.community.storage.large; -import ai.chat2db.community.domain.api.model.PageResponse; import ai.chat2db.community.domain.api.model.task.Task; import ai.chat2db.community.domain.api.model.task.TaskConstants; import ai.chat2db.community.domain.api.model.task.TaskEvent; import ai.chat2db.community.domain.api.model.task.TaskEventCode; -import ai.chat2db.community.domain.api.model.task.TaskEventLevel; -import ai.chat2db.community.domain.api.model.task.TaskProgress; -import ai.chat2db.community.domain.api.model.task.TaskQuery; import ai.chat2db.community.domain.api.model.task.TaskStatus; import ai.chat2db.community.domain.api.model.task.TaskStatusPatch; -import ai.chat2db.community.domain.api.model.task.TaskTargetSnapshot; +import ai.chat2db.community.domain.api.service.task.TaskStorage; +import ai.chat2db.community.storage.AbstractTaskStorageContractTest; import cn.hutool.core.io.FileUtil; import com.alibaba.fastjson2.JSON; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; import java.io.File; -import java.util.ArrayList; -import java.util.Comparator; import java.util.Date; -import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.stream.LongStream; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertNotSame; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; -class FileTaskStorageTest { +/** + * File-layout behaviour of {@link FileTaskStorage}. The storage-independent contract lives in + * {@link AbstractTaskStorageContractTest}. + */ +class FileTaskStorageTest extends AbstractTaskStorageContractTest { - @TempDir - File baseDir; + @Override + protected TaskStorage createStorage() { + return new FileTaskStorage(baseDir.getAbsolutePath()); + } + + private FileTaskStorage fileStorage() { + return (FileTaskStorage) storage(); + } @Test void usesTaskV2LayoutAndIgnoresLegacyTaskDirectory() { @@ -51,7 +48,7 @@ void usesTaskV2LayoutAndIgnoresLegacyTaskDirectory() { FileUtil.writeUtf8String(JSON.toJSONString(Task.builder().id(42L).name("legacy").build()), new File(legacyDirectory, "42.json")); - FileTaskStorage storage = storage(); + FileTaskStorage storage = fileStorage(); assertEquals(0L, storage.list(query(1, 20)).getTotal()); Task created = create(storage, "new-task"); @@ -71,7 +68,7 @@ void usesTaskV2LayoutAndIgnoresLegacyTaskDirectory() { @Test void keepsMoreThanLegacyLimitAndReloadsEveryTaskAndEvent() { - FileTaskStorage storage = storage(); + FileTaskStorage storage = fileStorage(); for (int i = 0; i < 25; i++) { create(storage, "task-" + i); @@ -83,268 +80,17 @@ void keepsMoreThanLegacyLimitAndReloadsEveryTaskAndEvent() { assertNotNull(files); assertEquals(51, files.length, "one index plus one snapshot and one event file per task"); - FileTaskStorage reloaded = storage(); - PageResponse page = reloaded.list(query(1, 100)); - assertEquals(25L, page.getTotal()); - assertEquals(25, page.getData().size()); - for (Task task : page.getData()) { + FileTaskStorage reloaded = fileStorage(); + List reloadedTasks = reloaded.list(query(1, 100)).getData(); + assertEquals(25, reloadedTasks.size()); + for (Task task : reloadedTasks) { assertEquals(List.of(1L), sequences(reloaded.listEvents(task.getId(), 0, 10))); } } - @Test - void taskAndEventResultsAreDefensiveCopies() { - FileTaskStorage storage = storage(); - Task input = task("original"); - Task created = storage.create(input, event(TaskEventCode.TASK_CREATED.name())); - Long taskId = created.getId(); - - assertNotSame(input.getTarget(), created.getTarget()); - input.setName("mutated-input"); - input.getTarget().setTableName("mutated-input-table"); - created.setName("mutated-return"); - created.getTarget().setTableName("mutated-return-table"); - - Task fetched = storage.get(taskId).orElseThrow(); - assertEquals("original", fetched.getName()); - assertEquals("source_table", fetched.getTarget().getTableName()); - - fetched.setName("mutated-get"); - storage.list(query(1, 20)).getData().get(0).setName("mutated-list"); - storage.listNonTerminalTasks().get(0).setName("mutated-non-terminal"); - assertEquals("original", storage.get(taskId).orElseThrow().getName()); - - Map details = new HashMap<>(); - details.put("rows", 10); - TaskEvent event = event(TaskEventCode.QUERY_STARTED.name()); - event.setTaskId(taskId); - event.setDetails(details); - TaskEvent appended = storage.appendEvent(event); - details.put("rows", 99); - appended.getDetails().put("rows", 88); - - TaskEvent persisted = storage.listEvents(taskId, 1, 10).get(0); - assertEquals(10, persisted.getDetails().get("rows")); - persisted.getDetails().put("rows", 77); - assertEquals(10, storage.listEvents(taskId, 1, 10).get(0).getDetails().get("rows")); - } - - @Test - void compareAndSetEnforcesLegalTransitionsAndTerminalImmutability() { - FileTaskStorage storage = storage(); - Task created = create(storage, "task"); - Long taskId = created.getId(); - - assertTrue(start(storage, taskId)); - assertFalse(storage.compareAndSetStatus(taskId, TaskStatus.PENDING.name(), TaskStatus.FAILED.name(), - TaskStatusPatch.builder().errorCode("LATE").build(), event(TaskEventCode.TASK_FAILED.name()))); - - TaskStatusPatch success = TaskStatusPatch.builder() - .progress(40) - .artifactId("artifact-1") - .finishedAt(new Date()) - .build(); - assertTrue(storage.compareAndSetStatus(taskId, TaskStatus.RUNNING.name(), TaskStatus.SUCCESS.name(), - success, event(TaskEventCode.TASK_SUCCEEDED.name()))); - - Task finished = storage.get(taskId).orElseThrow(); - assertEquals(TaskStatus.SUCCESS.name(), finished.getStatus()); - assertEquals(TaskConstants.COMPLETED_PROGRESS, finished.getProgress()); - assertEquals("artifact-1", finished.getArtifactId()); - assertFalse(storage.compareAndSetStatus(taskId, TaskStatus.SUCCESS.name(), TaskStatus.FAILED.name(), - TaskStatusPatch.builder().errorCode("TOO_LATE").build(), event(TaskEventCode.TASK_FAILED.name()))); - assertEquals(TaskStatus.SUCCESS.name(), storage.get(taskId).orElseThrow().getStatus()); - assertEquals(List.of(1L, 2L, 3L), sequences(storage.listEvents(taskId, 0, 20))); - } - - @Test - void progressIsRunningOnlyMonotonicAndPreservedByFailure() { - FileTaskStorage storage = storage(); - Task created = create(storage, "task"); - Long taskId = created.getId(); - - assertFalse(storage.updateProgressIfRunning(taskId, progress(30, "query"))); - assertTrue(start(storage, taskId)); - assertTrue(storage.updateProgressIfRunning(taskId, progress(50, "query"))); - assertFalse(storage.updateProgressIfRunning(taskId, progress(40, "write"))); - assertEquals(50, storage.get(taskId).orElseThrow().getProgress()); - - assertTrue(storage.updateProgressIfRunning(taskId, progress(120, "write"))); - Task running = storage.get(taskId).orElseThrow(); - assertEquals(TaskConstants.MAX_RUNNING_PROGRESS, running.getProgress()); - assertEquals("write", running.getStage()); - - TaskStatusPatch failure = TaskStatusPatch.builder() - .progress(TaskConstants.COMPLETED_PROGRESS) - .errorCode("EXPORT_FAILED") - .errorMessage("failed") - .finishedAt(new Date()) - .build(); - assertTrue(storage.compareAndSetStatus(taskId, TaskStatus.RUNNING.name(), TaskStatus.FAILED.name(), - failure, event(TaskEventCode.TASK_FAILED.name()))); - Task failed = storage.get(taskId).orElseThrow(); - assertEquals(TaskConstants.MAX_RUNNING_PROGRESS, failed.getProgress()); - assertEquals("EXPORT_FAILED", failed.getErrorCode()); - assertFalse(storage.updateProgressIfRunning(taskId, progress(50, "late"))); - assertEquals(TaskConstants.MAX_RUNNING_PROGRESS, storage.get(taskId).orElseThrow().getProgress()); - } - - @Test - void terminalStatusRaceHasExactlyOneWinnerAndOneTerminalEvent() throws Exception { - FileTaskStorage storage = storage(); - Task created = create(storage, "task"); - Long taskId = created.getId(); - assertTrue(start(storage, taskId)); - ExecutorService executor = Executors.newFixedThreadPool(2); - CountDownLatch start = new CountDownLatch(1); - - try { - Future success = executor.submit(() -> { - start.await(); - return storage.compareAndSetStatus(taskId, TaskStatus.RUNNING.name(), TaskStatus.SUCCESS.name(), - TaskStatusPatch.builder().artifactId("artifact").build(), - event(TaskEventCode.TASK_SUCCEEDED.name())); - }); - Future failure = executor.submit(() -> { - start.await(); - return storage.compareAndSetStatus(taskId, TaskStatus.RUNNING.name(), TaskStatus.FAILED.name(), - TaskStatusPatch.builder().errorCode("FAILED").errorMessage("failed").build(), - event(TaskEventCode.TASK_FAILED.name())); - }); - - start.countDown(); - assertTrue(success.get() ^ failure.get(), "exactly one terminal transition must win"); - } finally { - executor.shutdownNow(); - } - - assertTrue(TaskStatus.isTerminal(storage.get(taskId).orElseThrow().getStatus())); - assertEquals(3, storage.listEvents(taskId, 0, 20).size()); - } - - @Test - void paginationIsStableNewestFirstWithIdTieBreaker() { - FileTaskStorage storage = storage(); - List created = new ArrayList<>(); - for (int i = 0; i < 5; i++) { - created.add(create(storage, "task-" + i)); - } - Comparator newestFirst = Comparator - .comparing(Task::getCreatedAt, Comparator.nullsLast(Comparator.reverseOrder())) - .thenComparing(Task::getId, Comparator.nullsLast(Comparator.reverseOrder())); - List expected = created.stream().sorted(newestFirst).map(Task::getId).toList(); - - List firstPage = ids(storage.list(query(1, 2)).getData()); - List secondPage = ids(storage.list(query(2, 2)).getData()); - assertEquals(expected.subList(0, 2), firstPage); - assertEquals(expected.subList(2, 4), secondPage); - assertEquals(firstPage, ids(storage.list(query(1, 2)).getData())); - - TaskQuery hugePage = query(Integer.MAX_VALUE, 2); - PageResponse empty = storage.list(hugePage); - assertTrue(empty.getData().isEmpty()); - assertEquals(5L, empty.getTotal()); - } - - @Test - @SuppressWarnings("StringOperationCanBeSimplified") - void statusFilterUsesStringValueEquality() { - FileTaskStorage storage = storage(); - Task pending = create(storage, "pending"); - Task running = create(storage, "running"); - assertTrue(start(storage, running.getId())); - TaskQuery query = query(1, 20); - query.setStatus(new String(TaskStatus.RUNNING.name())); - - PageResponse page = storage.list(query); - - assertEquals(1L, page.getTotal()); - assertEquals(List.of(running.getId()), ids(page.getData())); - assertFalse(page.getData().stream().anyMatch(task -> pending.getId().equals(task.getId()))); - } - - @Test - void concurrentEventsReceiveUniqueOrderedSequencesAndSupportIncrementalReads() throws Exception { - FileTaskStorage storage = storage(); - Task created = create(storage, "task"); - Long taskId = created.getId(); - ExecutorService executor = Executors.newFixedThreadPool(6); - CountDownLatch start = new CountDownLatch(1); - List> futures = new ArrayList<>(); - - try { - for (int i = 0; i < 20; i++) { - futures.add(executor.submit(() -> { - start.await(); - TaskEvent event = event(TaskEventCode.QUERY_STARTED.name()); - event.setTaskId(taskId); - event.setSequence(999L); - return storage.appendEvent(event); - })); - } - start.countDown(); - - List returnedSequences = new ArrayList<>(); - for (Future future : futures) { - returnedSequences.add(future.get().getSequence()); - } - returnedSequences.sort(Long::compareTo); - assertEquals(LongStream.rangeClosed(2, 21).boxed().toList(), returnedSequences); - } finally { - executor.shutdownNow(); - } - - assertEquals(LongStream.rangeClosed(1, 21).boxed().toList(), - sequences(storage.listEvents(taskId, 0, 100))); - assertEquals(List.of(11L, 12L, 13L, 14L, 15L), - sequences(storage.listEvents(taskId, 10, 5))); - assertEquals(List.of(17L, 18L, 19L, 20L, 21L), - sequences(storage.listEventsBefore(taskId, null, 5))); - assertEquals(List.of(6L, 7L, 8L, 9L, 10L), - sequences(storage.listEventsBefore(taskId, 11L, 5))); - assertEquals(List.of(1L, 2L, 3L, 4L, 5L), - sequences(storage.listEventsBefore(taskId, 6L, 10))); - assertTrue(storage.listEventsBefore(taskId, 1L, 5).isEmpty()); - } - - @Test - void reversePaginationPreservesUtf8EventMessages() { - FileTaskStorage storage = storage(); - Task created = create(storage, "task"); - TaskEvent second = event(TaskEventCode.QUERY_STARTED.name()); - second.setTaskId(created.getId()); - second.setMessage("\u8bfb\u53d6\u8868\u6570\u636e"); - storage.appendEvent(second); - - List events = storage.listEventsBefore(created.getId(), null, 1); - - assertEquals(1, events.size()); - assertEquals("\u8bfb\u53d6\u8868\u6570\u636e", events.get(0).getMessage()); - } - - @Test - void incrementalReadStartsAfterCachedTailAndPreservesUtf8() { - FileTaskStorage storage = storage(); - Task created = create(storage, "task"); - TaskEvent second = event(TaskEventCode.QUERY_STARTED.name()); - second.setTaskId(created.getId()); - second.setMessage("\u5df2\u5bfc\u51fa 1000 \u884c"); - storage.appendEvent(second); - assertEquals(List.of(2L), sequences(storage.listEventsBefore(created.getId(), null, 1))); - TaskEvent third = event(TaskEventCode.QUERY_COMPLETED.name()); - third.setTaskId(created.getId()); - third.setMessage("\u5bfc\u51fa\u5b8c\u6210"); - storage.appendEvent(third); - - List events = storage.listEvents(created.getId(), 2L, 10); - - assertEquals(List.of(3L), sequences(events)); - assertEquals("\u5bfc\u51fa\u5b8c\u6210", events.get(0).getMessage()); - } - @Test void incompleteTrailingEventIsDiscardedWithoutLosingNextAppendedEvent() { - FileTaskStorage storage = storage(); + FileTaskStorage storage = fileStorage(); Task created = create(storage, "task"); Long taskId = created.getId(); TaskEvent second = event(TaskEventCode.QUERY_STARTED.name()); @@ -352,19 +98,19 @@ void incompleteTrailingEventIsDiscardedWithoutLosingNextAppendedEvent() { storage.appendEvent(second); FileUtil.appendUtf8String("{\"sequence\":", eventsFile(taskId)); - FileTaskStorage firstReload = storage(); + FileTaskStorage firstReload = fileStorage(); assertEquals(List.of(1L, 2L), sequences(firstReload.listEvents(taskId, 0, 20))); TaskEvent third = event(TaskEventCode.QUERY_COMPLETED.name()); third.setTaskId(taskId); assertEquals(3L, firstReload.appendEvent(third).getSequence()); - FileTaskStorage secondReload = storage(); + FileTaskStorage secondReload = fileStorage(); assertEquals(List.of(1L, 2L, 3L), sequences(secondReload.listEvents(taskId, 0, 20))); } @Test void unreadableMiddleEventDoesNotHideLaterValidEvents() { - FileTaskStorage storage = storage(); + FileTaskStorage storage = fileStorage(); Task created = create(storage, "task"); Long taskId = created.getId(); TaskEvent second = event(TaskEventCode.QUERY_STARTED.name()); @@ -375,7 +121,7 @@ void unreadableMiddleEventDoesNotHideLaterValidEvents() { third.setTaskId(taskId); assertEquals(3L, storage.appendEvent(third).getSequence()); - FileTaskStorage reloaded = storage(); + FileTaskStorage reloaded = fileStorage(); assertEquals(List.of(1L, 2L, 3L), sequences(reloaded.listEvents(taskId, 0, 20))); assertEquals(List.of(2L, 3L), sequences(reloaded.listEventsBefore(taskId, null, 2))); @@ -383,7 +129,7 @@ void unreadableMiddleEventDoesNotHideLaterValidEvents() { @Test void residualTransitionJournalRestoresLifecycleEventAndSnapshot() { - FileTaskStorage storage = storage(); + FileTaskStorage storage = fileStorage(); Task created = create(storage, "task"); Long taskId = created.getId(); assertTrue(start(storage, taskId)); @@ -402,7 +148,7 @@ void residualTransitionJournalRestoresLifecycleEventAndSnapshot() { taskId + FileTaskStorage.TASK_TRANSITION_FILE_SUFFIX); FileUtil.writeUtf8String(JSON.toJSONString(transition), transitionFile); - FileTaskStorage reloaded = storage(); + FileTaskStorage reloaded = fileStorage(); Task recovered = reloaded.get(taskId).orElseThrow(); assertEquals(TaskStatus.SUCCESS.name(), recovered.getStatus()); @@ -413,7 +159,7 @@ void residualTransitionJournalRestoresLifecycleEventAndSnapshot() { @Test void deletesTerminalTaskSnapshotIndexAndEventsButRejectsActiveTask() { - FileTaskStorage storage = storage(); + FileTaskStorage storage = fileStorage(); Task task = create(storage, "task"); Long taskId = task.getId(); @@ -427,19 +173,14 @@ void deletesTerminalTaskSnapshotIndexAndEventsButRejectsActiveTask() { event(TaskEventCode.TASK_SUCCEEDED.name()))); assertTrue(storage.deleteTerminalTask(taskId, () -> {})); - assertTrue(storage.get(taskId).isEmpty()); assertFalse(detailFile(taskId).exists()); assertFalse(eventsFile(taskId).exists()); assertFalse(FileUtil.readLines(indexFile(), "UTF-8").contains(String.valueOf(taskId))); - - FileTaskStorage reloaded = storage(); - assertTrue(reloaded.get(taskId).isEmpty()); - assertTrue(reloaded.listEvents(taskId, 0, 10).isEmpty()); } @Test - void terminalTaskDeletionRollsBackSnapshotAndEventsWhenCommitFails() { - FileTaskStorage storage = storage(); + void failedTerminalTaskDeletionKeepsSnapshotFileAndIndexEntry() { + FileTaskStorage storage = fileStorage(); Task task = create(storage, "task"); Long taskId = task.getId(); assertTrue(start(storage, taskId)); @@ -452,16 +193,14 @@ void terminalTaskDeletionRollsBackSnapshotAndEventsWhenCommitFails() { throw new IllegalStateException("artifact commit failed"); })); - assertEquals(TaskStatus.SUCCESS.name(), storage.get(taskId).orElseThrow().getStatus()); assertTrue(detailFile(taskId).isFile()); assertTrue(eventsFile(taskId).isFile()); - assertEquals(List.of(1L, 2L, 3L), sequences(storage.listEvents(taskId, 0, 10))); assertTrue(FileUtil.readLines(indexFile(), "UTF-8").contains(String.valueOf(taskId))); } @Test void snapshotWriteFailureDoesNotChangeInMemoryProgress() { - FileTaskStorage storage = storage(); + FileTaskStorage storage = fileStorage(); Task created = create(storage, "task"); Long taskId = created.getId(); assertTrue(start(storage, taskId)); @@ -477,7 +216,7 @@ void snapshotWriteFailureDoesNotChangeInMemoryProgress() { @Test void eventWriteFailureRollsBackStatusInMemoryAndSnapshot() { - FileTaskStorage storage = storage(); + FileTaskStorage storage = fileStorage(); Task created = create(storage, "task"); Long taskId = created.getId(); File eventFile = eventsFile(taskId); @@ -498,56 +237,6 @@ void eventWriteFailureRollsBackStatusInMemoryAndSnapshot() { assertEquals(List.of(1L), sequences(storage.listEvents(taskId, 0, 20))); } - private FileTaskStorage storage() { - return new FileTaskStorage(baseDir.getAbsolutePath()); - } - - private Task create(FileTaskStorage storage, String name) { - return storage.create(task(name), event(TaskEventCode.TASK_CREATED.name())); - } - - private Task task(String name) { - return Task.builder() - .type("QUERY_RESULT_EXPORT") - .name(name) - .target(TaskTargetSnapshot.builder() - .dataSourceId(1L) - .databaseName("database") - .schemaName("schema") - .tableName("source_table") - .build()) - .build(); - } - - private TaskEvent event(String code) { - return TaskEvent.builder() - .level(TaskEventLevel.INFO.name()) - .code(code) - .message(code) - .build(); - } - - private boolean start(FileTaskStorage storage, Long taskId) { - return storage.compareAndSetStatus(taskId, TaskStatus.PENDING.name(), TaskStatus.RUNNING.name(), - TaskStatusPatch.builder() - .progress(TaskConstants.STARTED_PROGRESS) - .stage("started") - .startedAt(new Date()) - .build(), - event(TaskEventCode.TASK_STARTED.name())); - } - - private TaskProgress progress(int value, String stage) { - return TaskProgress.builder().progress(value).stage(stage).message(stage).build(); - } - - private TaskQuery query(int pageNo, int pageSize) { - TaskQuery query = new TaskQuery(); - query.setPageNo(pageNo); - query.setPageSize(pageSize); - return query; - } - private File taskDirectory() { return new File(baseDir, FileTaskStorage.TASK_STORAGE_DIRECTORY); } @@ -563,12 +252,4 @@ private File detailFile(Long taskId) { private File eventsFile(Long taskId) { return new File(taskDirectory(), taskId + FileTaskStorage.TASK_EVENT_FILE_SUFFIX); } - - private List sequences(List events) { - return events.stream().map(TaskEvent::getSequence).toList(); - } - - private List ids(List tasks) { - return tasks.stream().map(Task::getId).toList(); - } } diff --git a/chat2db-community-server/chat2db-community-storage/src/test/java/ai/chat2db/community/storage/task/H2TaskStorageTest.java b/chat2db-community-server/chat2db-community-storage/src/test/java/ai/chat2db/community/storage/task/H2TaskStorageTest.java new file mode 100644 index 0000000000..ce1b9d5fcb --- /dev/null +++ b/chat2db-community-server/chat2db-community-storage/src/test/java/ai/chat2db/community/storage/task/H2TaskStorageTest.java @@ -0,0 +1,113 @@ +package ai.chat2db.community.storage.task; + +import ai.chat2db.community.domain.api.model.task.Task; +import ai.chat2db.community.domain.api.model.task.TaskConstants; +import ai.chat2db.community.domain.api.model.task.TaskEvent; +import ai.chat2db.community.domain.api.model.task.TaskEventCode; +import ai.chat2db.community.domain.api.model.task.TaskStatus; +import ai.chat2db.community.domain.api.model.task.TaskStatusPatch; +import ai.chat2db.community.domain.api.service.task.TaskStorage; +import ai.chat2db.community.storage.AbstractTaskStorageContractTest; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.stream.LongStream; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * H2-specific behaviour: the JDBC row mapping and the task row lock that replaces + * {@code FileTaskStorage}'s instance-wide monitor. + */ +class H2TaskStorageTest extends AbstractTaskStorageContractTest { + + @Override + protected TaskStorage createStorage() { + return new H2TaskStorage(baseDir.getAbsolutePath()); + } + + @Test + void reopensDatabaseAndRoundTripsEveryTaskField() { + H2TaskStorage storage = (H2TaskStorage) storage(); + Task input = task("full"); + input.setUserId(7L); + input.setOrganizationId(9L); + Task created = storage.create(input, event(TaskEventCode.TASK_CREATED.name())); + Long taskId = created.getId(); + Date startedAt = new Date(1_700_000_000_123L); + Map details = new HashMap<>(); + details.put("rows", 10); + details.put("table", "t_order"); + TaskEvent event = event(TaskEventCode.QUERY_STARTED.name()); + event.setTaskId(taskId); + event.setStage("query"); + event.setDetails(details); + storage.appendEvent(event); + assertTrue(storage.compareAndSetStatus(taskId, TaskStatus.PENDING.name(), TaskStatus.RUNNING.name(), + TaskStatusPatch.builder().progress(1).stage("started").startedAt(startedAt).build(), + event(TaskEventCode.TASK_STARTED.name()))); + storage.close(); + + H2TaskStorage reopened = (H2TaskStorage) storage(); + Task stored = reopened.get(taskId).orElseThrow(); + assertEquals("full", stored.getName()); + assertEquals(7L, stored.getUserId()); + assertEquals(9L, stored.getOrganizationId()); + assertEquals("database", stored.getTarget().getDatabaseName()); + assertEquals("source_table", stored.getTarget().getTableName()); + assertEquals(startedAt, stored.getStartedAt()); + assertNull(stored.getFinishedAt()); + assertEquals(TaskConstants.STARTED_PROGRESS, stored.getProgress()); + + List events = reopened.listEvents(taskId, 0, 10); + assertEquals(List.of(1L, 2L, 3L), sequences(events)); + TaskEvent storedEvent = events.get(1); + assertEquals("query", storedEvent.getStage()); + assertEquals(10, storedEvent.getDetails().get("rows")); + assertEquals("t_order", storedEvent.getDetails().get("table")); + } + + @Test + void separateInstancesSerializeEventSequencesByRowLock() throws Exception { + Task created = create(storage(), "shared"); + Long taskId = created.getId(); + TaskStorage first = storage(); + TaskStorage second = storage(); + + int writers = 6; + ExecutorService executor = Executors.newFixedThreadPool(writers); + CountDownLatch start = new CountDownLatch(1); + List> futures = new ArrayList<>(); + try { + for (int writer = 0; writer < writers; writer++) { + TaskStorage storage = writer % 2 == 0 ? first : second; + futures.add(executor.submit(() -> { + start.await(); + TaskEvent event = event(TaskEventCode.QUERY_STARTED.name()); + event.setTaskId(taskId); + return storage.appendEvent(event).getSequence(); + })); + } + start.countDown(); + + List returned = new ArrayList<>(); + for (Future future : futures) { + returned.add(future.get()); + } + returned.sort(Long::compareTo); + assertEquals(LongStream.rangeClosed(2, 1 + writers).boxed().toList(), returned); + } finally { + executor.shutdownNow(); + } + } +} diff --git a/chat2db-community-server/chat2db-community-storage/src/test/java/ai/chat2db/community/storage/task/TaskStorageConfigurationTest.java b/chat2db-community-server/chat2db-community-storage/src/test/java/ai/chat2db/community/storage/task/TaskStorageConfigurationTest.java new file mode 100644 index 0000000000..81872dab32 --- /dev/null +++ b/chat2db-community-server/chat2db-community-storage/src/test/java/ai/chat2db/community/storage/task/TaskStorageConfigurationTest.java @@ -0,0 +1,119 @@ +package ai.chat2db.community.storage.task; + +import ai.chat2db.community.domain.api.model.task.Task; +import ai.chat2db.community.domain.api.model.task.TaskEvent; +import ai.chat2db.community.domain.api.model.task.TaskEventCode; +import ai.chat2db.community.domain.api.model.task.TaskEventLevel; +import ai.chat2db.community.domain.api.service.task.TaskStorage; +import ai.chat2db.community.storage.TestHome; +import ai.chat2db.community.storage.large.FileTaskStorage; +import org.junit.jupiter.api.Test; +import org.springframework.context.annotation.AnnotationConfigApplicationContext; +import org.springframework.core.env.MapPropertySource; + +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; + +/** + * Bean selection for {@link TaskStorageConfiguration}: the rollback switch must replace the storage + * implementation, never run two of them side by side. + */ +class TaskStorageConfigurationTest { + + static { + // The H2 bean resolves its file from user.home, so a fresh home keeps consecutive runs from + // reopening the previous run's database file. + TestHome.init(); + } + + @Test + void h2IsSelectedByDefaultAndExplicitly() { + assertH2Selected(context(Map.of())); + assertH2Selected(context(Map.of(TaskStorageConfiguration.STORAGE_PROPERTY, "h2"))); + } + + @Test + void filePropertySelectsTheLegacyStorage() { + try (AnnotationConfigApplicationContext context = + context(Map.of(TaskStorageConfiguration.STORAGE_PROPERTY, "file"))) { + assertEquals(1, context.getBeansOfType(TaskStorage.class).size()); + assertInstanceOf(FileTaskStorage.class, context.getBean(TaskStorage.class)); + } + } + + @Test + void unknownStorageValueKeepsH2BecauseOnlyFileOptsOut() { + assertH2Selected(context(Map.of(TaskStorageConfiguration.STORAGE_PROPERTY, "postgres"))); + } + + @Test + void fileSwitchIsCaseInsensitive() { + try (AnnotationConfigApplicationContext context = + context(Map.of(TaskStorageConfiguration.STORAGE_PROPERTY, "FILE"))) { + assertEquals(1, context.getBeansOfType(TaskStorage.class).size()); + assertInstanceOf(FileTaskStorage.class, context.getBean(TaskStorage.class)); + } + } + + @Test + void closingTheContextShutsTheH2StorageAndReopeningReadsTheTaskBack() { + Long taskId; + try (AnnotationConfigApplicationContext context = context(Map.of())) { + taskId = storage(context).create(task("wired"), event()).getId(); + } + + try (AnnotationConfigApplicationContext context = context(Map.of())) { + TaskStorage storage = storage(context); + assertEquals("wired", storage.get(taskId).orElseThrow().getName()); + assertEquals(List.of(TaskEventCode.TASK_CREATED.name()), + storage.listEvents(taskId, 0L, 10).stream().map(TaskEvent::getCode).toList()); + } + } + + @Test + void h2BeanDeclaresAnExplicitDestroyMethodSoTheDatabaseFileIsReleased() { + try (AnnotationConfigApplicationContext context = context(Map.of())) { + assertEquals("close", context.getBeanDefinition("h2TaskStorage").getDestroyMethodName()); + } + } + + private static void assertH2Selected(AnnotationConfigApplicationContext context) { + try (context) { + assertEquals(1, context.getBeansOfType(TaskStorage.class).size()); + assertInstanceOf(H2TaskStorage.class, context.getBean(TaskStorage.class)); + } + } + + private static TaskStorage storage(AnnotationConfigApplicationContext context) { + return context.getBean(TaskStorage.class); + } + + private static AnnotationConfigApplicationContext context(Map properties) { + AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(); + if (!properties.isEmpty()) { + context.getEnvironment().getPropertySources() + .addFirst(new MapPropertySource("test-task-storage", properties)); + } + context.register(TaskStorageConfiguration.class); + context.refresh(); + return context; + } + + private static TaskEvent event() { + return TaskEvent.builder() + .level(TaskEventLevel.INFO.name()) + .code(TaskEventCode.TASK_CREATED.name()) + .message(TaskEventCode.TASK_CREATED.name()) + .build(); + } + + private static Task task(String name) { + return Task.builder() + .type("QUERY_RESULT_EXPORT") + .name(name) + .build(); + } +} diff --git a/chat2db-community-server/chat2db-community-storage/src/test/java/ai/chat2db/community/storage/task/TaskStorageMigrationTest.java b/chat2db-community-server/chat2db-community-storage/src/test/java/ai/chat2db/community/storage/task/TaskStorageMigrationTest.java new file mode 100644 index 0000000000..4ccb1eb090 --- /dev/null +++ b/chat2db-community-server/chat2db-community-storage/src/test/java/ai/chat2db/community/storage/task/TaskStorageMigrationTest.java @@ -0,0 +1,178 @@ +package ai.chat2db.community.storage.task; + +import ai.chat2db.community.domain.api.model.task.Task; +import ai.chat2db.community.domain.api.model.task.TaskArtifact; +import ai.chat2db.community.domain.api.model.task.TaskArtifactRole; +import ai.chat2db.community.domain.api.model.task.TaskConstants; +import ai.chat2db.community.domain.api.model.task.TaskEvent; +import ai.chat2db.community.domain.api.model.task.TaskEventCode; +import ai.chat2db.community.domain.api.model.task.TaskStatus; +import ai.chat2db.community.domain.api.model.task.TaskStatusPatch; +import ai.chat2db.community.storage.large.FileTaskStorage; +import cn.hutool.core.io.FileUtil; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.File; +import java.util.Comparator; +import java.util.Date; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class TaskStorageMigrationTest { + + @TempDir + File baseDir; + + private TaskDatabase database; + + @AfterEach + void closeDatabase() { + if (database != null) { + database.close(); + } + } + + private TaskDatabase database() { + if (database == null) { + database = new TaskDatabase(baseDir.getAbsolutePath()); + } + return database; + } + + @Test + void importsEveryTaskAndEventAndRetiresTheLegacyDirectory() { + FileTaskStorage file = new FileTaskStorage(baseDir.getAbsolutePath()); + Long pendingId = file.create(task("pending"), event(TaskEventCode.TASK_CREATED.name())).getId(); + Long finishedId = file.create(task("finished"), event(TaskEventCode.TASK_CREATED.name())).getId(); + assertTrue(file.compareAndSetStatus(finishedId, TaskStatus.PENDING.name(), TaskStatus.RUNNING.name(), + TaskStatusPatch.builder().progress(TaskConstants.STARTED_PROGRESS).stage("started").build(), + event(TaskEventCode.TASK_STARTED.name()))); + TaskEvent exported = event(TaskEventCode.QUERY_COMPLETED.name()); + exported.setTaskId(finishedId); + exported.setMessage("已导出 1200 行"); + file.appendEvent(exported); + assertTrue(file.compareAndSetStatus(finishedId, TaskStatus.RUNNING.name(), TaskStatus.SUCCESS.name(), + TaskStatusPatch.builder().artifactId("artifact.csv").finishedAt(new Date()).build(), + event(TaskEventCode.TASK_SUCCEEDED.name()))); + // A killed process can leave a half-written trailing line; it must not block the import. + FileUtil.appendUtf8String("{\"sequence\":", eventsFile(finishedId)); + + assertEquals(2, new TaskStorageMigrator(database(), baseDir.getAbsolutePath()).migrateIfRequired()); + + H2TaskStorage migrated = new H2TaskStorage(database()); + assertEquals(List.of(pendingId, finishedId), ids(migrated.listTasksForRecovery())); + assertEquals(TaskStatus.PENDING.name(), migrated.get(pendingId).orElseThrow().getStatus()); + + Task finished = migrated.get(finishedId).orElseThrow(); + assertEquals(TaskStatus.SUCCESS.name(), finished.getStatus()); + assertEquals(TaskConstants.COMPLETED_PROGRESS, finished.getProgress()); + assertEquals("artifact.csv", finished.getArtifactId()); + assertEquals(3L, finished.getUserId()); + assertEquals(4L, finished.getOrganizationId()); + + List migratedArtifacts = migrated.listArtifacts(finishedId); + assertEquals(List.of("artifact.csv"), + migratedArtifacts.stream().map(TaskArtifact::getArtifactId).toList()); + assertEquals(TaskArtifactRole.OUTPUT, migratedArtifacts.get(0).getRole()); + assertTrue(migrated.listArtifacts(pendingId).isEmpty()); + + List events = migrated.listEvents(finishedId, 0, 20); + assertEquals(List.of(1L, 2L, 3L, 4L), sequences(events)); + assertEquals("已导出 1200 行", events.get(2).getMessage()); + assertEquals(List.of(1L), sequences(migrated.listEvents(pendingId, 0, 20))); + + TaskEvent next = event(TaskEventCode.QUERY_STARTED.name()); + next.setTaskId(finishedId); + assertEquals(5L, migrated.appendEvent(next).getSequence()); + + assertFalse(legacyDirectory().isDirectory()); + File migratedDirectory = new File(baseDir, FileTaskStorage.TASK_STORAGE_DIRECTORY + + TaskStorageMigrator.MIGRATED_DIRECTORY_SUFFIX); + assertTrue(migratedDirectory.isDirectory()); + assertTrue(new File(migratedDirectory, finishedId + FileTaskStorage.TASK_EVENT_FILE_SUFFIX).isFile()); + } + + @Test + void runningTheMigrationTwiceImportsNothingAgain() { + FileTaskStorage file = new FileTaskStorage(baseDir.getAbsolutePath()); + Long taskId = file.create(task("once"), event(TaskEventCode.TASK_CREATED.name())).getId(); + + TaskStorageMigrator migrator = new TaskStorageMigrator(database(), baseDir.getAbsolutePath()); + assertEquals(1, migrator.migrateIfRequired()); + assertEquals(0, migrator.migrateIfRequired()); + + H2TaskStorage migrated = new H2TaskStorage(database()); + assertEquals(List.of(taskId), ids(migrated.listTasksForRecovery())); + assertEquals(List.of(1L), sequences(migrated.listEvents(taskId, 0, 20))); + } + + @Test + void migratesAnAbsentLegacyDirectoryWithoutWritingTasks() { + String storageBasePath = new File(baseDir, "unused").getAbsolutePath(); + TaskDatabase fresh = new TaskDatabase(storageBasePath); + try { + assertEquals(0, new TaskStorageMigrator(fresh, storageBasePath).migrateIfRequired()); + assertTrue(new H2TaskStorage(fresh).listTasksForRecovery().isEmpty()); + } finally { + fresh.close(); + } + } + + @Test + void missingIndexedSnapshotAbortsWithoutMarkingOrRenamingLegacyStorage() { + assertTrue(legacyDirectory().mkdirs()); + FileUtil.writeUtf8String("42\n", indexFile()); + TaskStorageMigrator migrator = new TaskStorageMigrator(database(), baseDir.getAbsolutePath()); + + assertThrows(IllegalStateException.class, migrator::migrateIfRequired); + + assertTrue(legacyDirectory().isDirectory()); + assertTrue(indexFile().isFile()); + assertTrue(new H2TaskStorage(database()).listTasksForRecovery().isEmpty()); + assertThrows(IllegalStateException.class, migrator::migrateIfRequired, + "a failed migration must not write its completion marker"); + } + + private Task task(String name) { + return Task.builder() + .type("TABLE_DATA_EXPORT") + .name(name) + .userId(3L) + .organizationId(4L) + .build(); + } + + private TaskEvent event(String code) { + return TaskEvent.builder() + .level("INFO") + .code(code) + .message(code) + .build(); + } + + private File eventsFile(Long taskId) { + return new File(legacyDirectory(), taskId + FileTaskStorage.TASK_EVENT_FILE_SUFFIX); + } + + private File indexFile() { + return new File(legacyDirectory(), FileTaskStorage.TASK_INDEX_NAME + ".json"); + } + + private File legacyDirectory() { + return new File(baseDir, FileTaskStorage.TASK_STORAGE_DIRECTORY); + } + + private List sequences(List events) { + return events.stream().map(TaskEvent::getSequence).toList(); + } + + private List ids(List tasks) { + return tasks.stream().map(Task::getId).sorted(Comparator.naturalOrder()).toList(); + } +} diff --git a/chat2db-community-server/chat2db-community-web/src/main/java/ai/chat2db/community/web/api/controller/TaskController.java b/chat2db-community-server/chat2db-community-web/src/main/java/ai/chat2db/community/web/api/controller/TaskController.java index a5da0a9c7d..befdba6f85 100644 --- a/chat2db-community-server/chat2db-community-web/src/main/java/ai/chat2db/community/web/api/controller/TaskController.java +++ b/chat2db-community-server/chat2db-community-web/src/main/java/ai/chat2db/community/web/api/controller/TaskController.java @@ -1,7 +1,9 @@ package ai.chat2db.community.web.api.controller; import ai.chat2db.community.domain.api.model.PageResponse; +import ai.chat2db.community.domain.api.model.task.ImportPreview; import ai.chat2db.community.domain.api.model.task.Task; +import ai.chat2db.community.domain.api.model.task.TaskArtifact; import ai.chat2db.community.domain.api.model.task.TaskEvent; import ai.chat2db.community.domain.api.model.task.TaskQuery; import ai.chat2db.community.domain.api.service.task.IImportTaskSubmissionService; @@ -18,6 +20,7 @@ import ai.chat2db.community.web.api.model.request.task.TaskImportRequest; import ai.chat2db.community.web.api.model.response.task.TaskSubmitResponse; import jakarta.validation.Valid; +import org.apache.commons.lang3.StringUtils; import org.springframework.core.io.Resource; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.DeleteMapping; @@ -64,6 +67,19 @@ public DataResult submitImport(@Valid @RequestBody TaskImpor return DataResult.of(new TaskSubmitResponse(taskId)); } + @PostMapping("/import/preview") + public DataResult previewImport(@Valid @RequestBody TaskImportRequest request) { + if (StringUtils.isBlank(request.getFileId())) { + throw new ai.chat2db.community.tools.exception.ParamBusinessException("fileId"); + } + return DataResult.of(taskService.previewImport(taskWebConverter.importRequest2spec(request))); + } + + @PostMapping("/resume") + public DataResult resume(@RequestBody @Valid TaskIdRequest request) { + return DataResult.of(new TaskSubmitResponse(taskService.resume(request.getTaskId()))); + } + @GetMapping("/list") public WebPageResult list(TaskQuery query) { PageResponse page = taskService.list(query); @@ -94,9 +110,16 @@ public ActionResult delete(@Valid TaskIdRequest request) { return ActionResult.isSuccess(); } + @GetMapping("/artifacts") + public DataResult> artifacts(@Valid TaskIdRequest request) { + return DataResult.of(taskService.listArtifacts(request.getTaskId())); + } + @GetMapping("/artifact") public ResponseEntity artifact(@Valid TaskIdRequest request) { - return taskDownloadWebConverter.toResponse(taskService.resolveArtifact(request.getTaskId())); + return taskDownloadWebConverter.toResponse(StringUtils.isBlank(request.getArtifactId()) + ? taskService.resolveArtifact(request.getTaskId()) + : taskService.resolveArtifact(request.getTaskId(), request.getArtifactId())); } @GetMapping("/active-count") diff --git a/chat2db-community-server/chat2db-community-web/src/main/java/ai/chat2db/community/web/api/converter/task/TaskWebConverter.java b/chat2db-community-server/chat2db-community-web/src/main/java/ai/chat2db/community/web/api/converter/task/TaskWebConverter.java index 4669cb3597..2acf9decdd 100644 --- a/chat2db-community-server/chat2db-community-web/src/main/java/ai/chat2db/community/web/api/converter/task/TaskWebConverter.java +++ b/chat2db-community-server/chat2db-community-web/src/main/java/ai/chat2db/community/web/api/converter/task/TaskWebConverter.java @@ -59,10 +59,14 @@ public ImportTaskSpec importRequest2spec(TaskImportRequest request) { .target(target(request.getDataSourceId(), request.getDatabaseName(), request.getSchemaName(), request.getTableName())) .sourceFile(sourceFile) + .importFileId(request.getFileId()) .displayFileName(StringUtils.defaultIfBlank(request.getDisplayFileName(), fileName(sourceFile))) .format(format) .dataTimeFormat(request.getDataTimeFormat()) .csvOptions(csvOptions(format, request.getCsvOptions())) + .options(request.getOptions()) + .unmappedTarget(request.getUnmappedTarget()) + .mode(normalize(request.getMode())) .build(); } diff --git a/chat2db-community-server/chat2db-community-web/src/main/java/ai/chat2db/community/web/api/model/request/task/TaskIdRequest.java b/chat2db-community-server/chat2db-community-web/src/main/java/ai/chat2db/community/web/api/model/request/task/TaskIdRequest.java index 4d6f0a3245..c540f7bd11 100644 --- a/chat2db-community-server/chat2db-community-web/src/main/java/ai/chat2db/community/web/api/model/request/task/TaskIdRequest.java +++ b/chat2db-community-server/chat2db-community-web/src/main/java/ai/chat2db/community/web/api/model/request/task/TaskIdRequest.java @@ -8,4 +8,9 @@ public class TaskIdRequest { @NotNull private Long taskId; + + /** + * Optional on the download path; blank selects the primary artifact. + */ + private String artifactId; } diff --git a/chat2db-community-server/chat2db-community-web/src/main/java/ai/chat2db/community/web/api/model/request/task/TaskImportRequest.java b/chat2db-community-server/chat2db-community-web/src/main/java/ai/chat2db/community/web/api/model/request/task/TaskImportRequest.java index 6672f792f7..76209e6f93 100644 --- a/chat2db-community-server/chat2db-community-web/src/main/java/ai/chat2db/community/web/api/model/request/task/TaskImportRequest.java +++ b/chat2db-community-server/chat2db-community-web/src/main/java/ai/chat2db/community/web/api/model/request/task/TaskImportRequest.java @@ -1,7 +1,9 @@ package ai.chat2db.community.web.api.model.request.task; -import ai.chat2db.community.web.api.model.request.data.source.DataSourceBaseRequest; +import ai.chat2db.community.domain.api.model.task.ImportOptions; import ai.chat2db.community.domain.api.model.task.CsvOptions; +import ai.chat2db.community.domain.api.model.task.ImportOptions; +import ai.chat2db.community.web.api.model.request.data.source.DataSourceBaseRequest; import lombok.Data; @Data @@ -24,4 +26,11 @@ public class TaskImportRequest extends DataSourceBaseRequest { private String dataTimeFormat; private CsvOptions csvOptions; + + private ImportOptions options; + + private ai.chat2db.community.domain.api.model.task.UnmappedTargetStrategy unmappedTarget; + + /** Execution mode: ULTRA_FAST or STANDARD (default when absent). */ + private String mode; } diff --git a/chat2db-community-server/chat2db-community-web/src/test/java/ai/chat2db/community/web/api/controller/TaskControllerDesktopContractTest.java b/chat2db-community-server/chat2db-community-web/src/test/java/ai/chat2db/community/web/api/controller/TaskControllerDesktopContractTest.java index b0cc81d2ac..05a9e730d4 100644 --- a/chat2db-community-server/chat2db-community-web/src/test/java/ai/chat2db/community/web/api/controller/TaskControllerDesktopContractTest.java +++ b/chat2db-community-server/chat2db-community-web/src/test/java/ai/chat2db/community/web/api/controller/TaskControllerDesktopContractTest.java @@ -20,6 +20,16 @@ class TaskControllerDesktopContractTest { + @Test + void previewRejectsRawServerPathsBeforeReadingFiles() { + TaskController controller = new TaskController(null, null, null, null); + var request = new ai.chat2db.community.web.api.model.request.task.TaskImportRequest(); + request.setSourceFile("C:/private/data.csv"); + org.junit.jupiter.api.Assertions.assertThrows( + ai.chat2db.community.tools.exception.ParamBusinessException.class, + () -> controller.previewImport(request)); + } + @Test void taskEndpointsUseStaticPathsAndAtMostOneRequestObject() { Set paths = Arrays.stream(TaskController.class.getDeclaredMethods()) @@ -28,8 +38,8 @@ void taskEndpointsUseStaticPathsAndAtMostOneRequestObject() { .flatMap(mapping -> Arrays.stream(mapping.path())) .collect(Collectors.toSet()); - assertEquals(Set.of("/export", "/import", "/list", "/get", "/events", "/delete", - "/artifact", "/active-count", "/prepare-user-exit", "/abort-user-exit"), paths); + assertEquals(Set.of("/export", "/import", "/import/preview", "/resume", "/list", "/get", "/events", "/delete", + "/artifact", "/artifacts", "/active-count", "/prepare-user-exit", "/abort-user-exit"), paths); Arrays.stream(TaskController.class.getDeclaredMethods()) .filter(method -> requestMapping(method) != null) diff --git a/chat2db-community-server/chat2db-community-web/src/test/java/ai/chat2db/community/web/api/converter/task/TaskWebConverterTest.java b/chat2db-community-server/chat2db-community-web/src/test/java/ai/chat2db/community/web/api/converter/task/TaskWebConverterTest.java index 2de000e477..9d7eba0b1b 100644 --- a/chat2db-community-server/chat2db-community-web/src/test/java/ai/chat2db/community/web/api/converter/task/TaskWebConverterTest.java +++ b/chat2db-community-server/chat2db-community-web/src/test/java/ai/chat2db/community/web/api/converter/task/TaskWebConverterTest.java @@ -19,6 +19,17 @@ class TaskWebConverterTest { + @Test + void importPreviewPreservesStagedSourceAndNullStrategy() { + var request = new ai.chat2db.community.web.api.model.request.task.TaskImportRequest(); + request.setFileId("staged-source"); + request.setFormat("CSV"); + request.setUnmappedTarget(ai.chat2db.community.domain.api.model.task.UnmappedTargetStrategy.NULL); + var result = new TaskWebConverter().importRequest2spec(request); + assertEquals("staged-source", result.getImportFileId()); + assertEquals(ai.chat2db.community.domain.api.model.task.UnmappedTargetStrategy.NULL, result.getUnmappedTarget()); + } + private final TaskWebConverter converter = new TaskWebConverter(); @Test From 04696d5084609e18e21eb7523f94471a8257a06f Mon Sep 17 00:00:00 2001 From: dreamWork Date: Tue, 8 Sep 2026 22:51:39 +0800 Subject: [PATCH 04/38] feat(import): add resumable parallel pipelines with guarded wizard controls Execute row-oriented imports through adaptive batching, bounded concurrency, durable resume journals, deterministic column mapping, and explicit error policies. Retain staged SQL sources until successful completion so interrupted ordered imports remain recoverable. Run a mandatory read-only admission scan before creating workers. Reject unreadable, compressed, multiline, structurally ambiguous, or unsupported parallel inputs; downgrade small CSV files; and allow operators to acknowledge only strong relationship and ordering risks while preserving hard blockers and audit events. Expose import file selection, preview, mapping, mode switching, inline risk confirmation, and explainable admission verdicts in the shared wizard. Keep advanced export formats hidden and the legacy SQL export dialog mounted until the export batch supplies its backend contracts. Cover batching, concurrency, resume, admission, staging, mapping, request conversion, and wizard parameters; verify lint and the complete Community Web build at this commit. --- chat2db-community-client/package.json | 3 +- .../scripts/i18n-source-hashes.json | 4 +- .../components/ImportExportFile/index.tsx | 489 +++++++-- .../mappingValidation.test.ts | 42 + .../ImportExportFile/mappingValidation.ts | 40 + .../components/ImportExportFile/style.ts | 110 +- .../ImportExportFile/taskParams.test.ts | 149 +++ .../components/ImportExportFile/taskParams.ts | 126 +++ .../components/ImportFileModal/index.tsx | 120 +-- .../ImportFileModal/submission.test.ts | 2 +- .../src/blocks/ImportAndExport/index.tsx | 2 +- .../NewTree/hooks/useCreateRightClickMenu.tsx | 20 +- .../src/constants/importExport.ts | 12 + .../src/i18n/en-US/workspace.ts | 4 +- .../src/i18n/es-ES/workspace.ts | 3 +- .../src/i18n/ja-JP/workspace.ts | 2 +- .../src/i18n/ko-KR/workspace.ts | 3 +- .../src/i18n/zh-CN/workspace.ts | 3 +- .../model/task/ImportAdmissionFinding.java | 25 + .../api/model/task/ImportAdmissionReport.java | 37 + .../domain/api/model/task/ImportPreview.java | 3 + .../domain/api/model/task/ImportTaskSpec.java | 7 + .../domain/api/model/task/TaskErrorCode.java | 1 + .../api/model/task/TaskExecutionMode.java | 24 + .../core/impl/task/AdaptiveBatchSizer.java | 60 ++ .../impl/task/AdaptiveConcurrencyGate.java | 183 ++++ .../core/impl/task/TaskResumeJournal.java | 334 +++++++ .../core/impl/task/TaskServiceImpl.java | 4 +- .../executor/DataFileImportTaskExecutor.java | 5 +- .../executor/SqlFileImportTaskExecutor.java | 7 +- .../core/impl/task/imports/BaseImporter.java | 23 +- .../task/imports/ImportColumnResolver.java | 61 +- .../task/imports/ImportParallelAdmission.java | 270 +++++ .../impl/task/imports/ImportRowBatcher.java | 936 ++++++++++++++++++ .../task/imports/excel/BaseExcelImporter.java | 276 ++---- .../impl/task/imports/excel/CSVImporter.java | 93 +- .../excel/CsvImportValueNormalizer.java | 4 +- .../impl/task/AdaptiveBatchSizerTest.java | 64 ++ .../task/AdaptiveConcurrencyGateTest.java | 120 +++ .../core/impl/task/CsvImportPipelineTest.java | 261 +++++ .../impl/task/ImportResumeRoundTripTest.java | 339 +++++++ .../task/ImportRowBatcherParallelTest.java | 351 +++++++ .../core/impl/task/TaskResumeJournalTest.java | 113 +++ .../SqlFileImportTaskExecutorTest.java | 6 +- .../imports/ImportParallelAdmissionTest.java | 103 ++ .../excel/CSVImporterColumnMappingTest.java | 83 +- .../json/JSONImporterExtensionTest.java | 6 + .../api/converter/task/TaskWebConverter.java | 4 + .../model/request/task/TaskImportRequest.java | 2 + .../converter/task/TaskWebConverterTest.java | 2 + 50 files changed, 4523 insertions(+), 418 deletions(-) create mode 100644 chat2db-community-client/src/blocks/ImportAndExport/components/ImportExportFile/mappingValidation.test.ts create mode 100644 chat2db-community-client/src/blocks/ImportAndExport/components/ImportExportFile/mappingValidation.ts create mode 100644 chat2db-community-client/src/blocks/ImportAndExport/components/ImportExportFile/taskParams.test.ts create mode 100644 chat2db-community-client/src/blocks/ImportAndExport/components/ImportExportFile/taskParams.ts create mode 100644 chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/task/ImportAdmissionFinding.java create mode 100644 chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/task/ImportAdmissionReport.java create mode 100644 chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/task/TaskExecutionMode.java create mode 100644 chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/AdaptiveBatchSizer.java create mode 100644 chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/AdaptiveConcurrencyGate.java create mode 100644 chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/TaskResumeJournal.java create mode 100644 chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/ImportParallelAdmission.java create mode 100644 chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/ImportRowBatcher.java create mode 100644 chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/AdaptiveBatchSizerTest.java create mode 100644 chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/AdaptiveConcurrencyGateTest.java create mode 100644 chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/CsvImportPipelineTest.java create mode 100644 chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/ImportResumeRoundTripTest.java create mode 100644 chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/ImportRowBatcherParallelTest.java create mode 100644 chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/TaskResumeJournalTest.java create mode 100644 chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/imports/ImportParallelAdmissionTest.java diff --git a/chat2db-community-client/package.json b/chat2db-community-client/package.json index 1115744aa9..46c7ba489a 100644 --- a/chat2db-community-client/package.json +++ b/chat2db-community-client/package.json @@ -12,7 +12,7 @@ "build": "yarn run build:web:community", "build:prod": "yarn run build:web:community", "build:web": "umi build", - "prebuild:web:community": "yarn test:community-boundary && yarn test:database-capabilities && yarn test:table-editor && yarn test:retired-ai && yarn test:tree-title-highlight && yarn test:tree-search-lifecycle && yarn test:driver-upload && yarn test:tree-loading && yarn test:tree-node-lookup && yarn test:tree-node-path && yarn test:data-source-authorization && yarn test:active-transactions && yarn test:data-source-mutation-refresh && yarn test:ai-model-config && yarn test:ai-model-select && yarn test:export-connections && yarn test:main-page-navigation && yarn test:shortcut && yarn test:console-tab-name && yarn test:workspace-tab-scroll && yarn test:workspace-split-lifecycle && yarn test:sql-completion-context && yarn test:file-manager-label && yarn test:local-file-encoding && yarn test:local-file-save && yarn test:local-file-tree-path && yarn test:saved-console-dirty && yarn test:editor-close && yarn test:result-set-ui && yarn test:result-pagination && yarn test:sql-execution-stream && yarn test:result-table-layout && yarn test:result-status && yarn test:data-source-identity && yarn test:data-source-execution-snapshot && yarn test:data-source-watermark && yarn test:hot-update && yarn test:task-center && yarn test:import-preview && yarn test:application-exit && yarn test:monaco-lifecycle && yarn test:result-set-editor && yarn test:account-grants-request && yarn test:connection-close-request && yarn test:operation-log-filters && yarn test:dashboard-store", + "prebuild:web:community": "yarn test:community-boundary && yarn test:database-capabilities && yarn test:table-editor && yarn test:retired-ai && yarn test:tree-title-highlight && yarn test:tree-search-lifecycle && yarn test:driver-upload && yarn test:tree-loading && yarn test:tree-node-lookup && yarn test:tree-node-path && yarn test:data-source-authorization && yarn test:active-transactions && yarn test:data-source-mutation-refresh && yarn test:ai-model-config && yarn test:ai-model-select && yarn test:export-connections && yarn test:main-page-navigation && yarn test:shortcut && yarn test:console-tab-name && yarn test:workspace-tab-scroll && yarn test:workspace-split-lifecycle && yarn test:sql-completion-context && yarn test:file-manager-label && yarn test:local-file-encoding && yarn test:local-file-save && yarn test:local-file-tree-path && yarn test:saved-console-dirty && yarn test:editor-close && yarn test:result-set-ui && yarn test:result-pagination && yarn test:sql-execution-stream && yarn test:result-table-layout && yarn test:result-status && yarn test:data-source-identity && yarn test:data-source-execution-snapshot && yarn test:data-source-watermark && yarn test:hot-update && yarn test:task-center && yarn test:import-export-wizard && yarn test:import-preview && yarn test:application-exit && yarn test:monaco-lifecycle && yarn test:result-set-editor && yarn test:account-grants-request && yarn test:connection-close-request && yarn test:operation-log-filters && yarn test:dashboard-store", "postbuild:web:community": "node ./scripts/verify-production-bundles.cjs", "build:web:2java": "yarn run build:web:community && rm -rf ../chat2db-community-server/chat2db-community-start/src/main/resources/thymeleaf/* && cp -r dist/index.html ../chat2db-community-server/chat2db-community-start/src/main/resources/thymeleaf/", "build:web:community": "cross-env UMI_ENV=community cross-env APP_NAME=chat2db-community cross-env APP_VERSION=${npm_config_app_version} cross-env PRINT_LOGS=${npm_config_print_logs} cross-env APP_PORT=${npm_config_app_port} umi build", @@ -24,6 +24,7 @@ "lint:eslint": "eslint \"src/**/*.{js,jsx,ts,tsx}\" --max-warnings=0", "lint:style": "stylelint \"src/**/*.{css,less}\" --max-warnings=0", "test:i18n": "node ./scripts/validate-i18n.cjs", + "test:import-export-wizard": "tsx src/blocks/ImportAndExport/components/ImportExportFile/taskParams.test.ts && tsx src/blocks/ImportAndExport/components/ImportExportFile/mappingValidation.test.ts", "test:community-boundary": "node ./scripts/verify-community-boundary.cjs", "test:database-capabilities": "tsx src/utils/databaseJudgments.test.ts", "test:table-editor": "tsx src/blocks/DatabaseTableEditor/ColumnList/normalizeColumn.test.ts", diff --git a/chat2db-community-client/scripts/i18n-source-hashes.json b/chat2db-community-client/scripts/i18n-source-hashes.json index 8b48bd2e58..c63213d6a7 100644 --- a/chat2db-community-client/scripts/i18n-source-hashes.json +++ b/chat2db-community-client/scripts/i18n-source-hashes.json @@ -21,7 +21,7 @@ "spaceSetting.ts": "defdf3138e5e1782f0282760bcef4ce1b8550cc80b03052fea468d97165cb980", "sqlEditor.ts": "5c41e5f24d5ba78909795bcc681cabb8460cca075233b665dc4940afe82ce0c9", "stream.ts": "2e2fca0b1ba1ecd5a1501b767be8857cf0b72d800d69a4acf9cd62a61d5ceaf5", - "workspace.ts": "c72e4530738cc8a592dd12c65e7f4cf29bac924e3c2ef79e60470dc1e6e617d0" + "workspace.ts": "47a329763fbbbf54db9a97ca98918b9f92a9a1e32fabf3ba505bc52b6abe2a3e" }, "ko-KR": { "ai.ts": "e7a2fcbfb3ed647bd14f63aca4b6c35c8fb402c49a6a1f36bb04f76f4e8415af", @@ -42,7 +42,7 @@ "spaceSetting.ts": "defdf3138e5e1782f0282760bcef4ce1b8550cc80b03052fea468d97165cb980", "sqlEditor.ts": "5c41e5f24d5ba78909795bcc681cabb8460cca075233b665dc4940afe82ce0c9", "stream.ts": "2e2fca0b1ba1ecd5a1501b767be8857cf0b72d800d69a4acf9cd62a61d5ceaf5", - "workspace.ts": "c72e4530738cc8a592dd12c65e7f4cf29bac924e3c2ef79e60470dc1e6e617d0" + "workspace.ts": "47a329763fbbbf54db9a97ca98918b9f92a9a1e32fabf3ba505bc52b6abe2a3e" } } } diff --git a/chat2db-community-client/src/blocks/ImportAndExport/components/ImportExportFile/index.tsx b/chat2db-community-client/src/blocks/ImportAndExport/components/ImportExportFile/index.tsx index b7bbbe8862..9cfe3a30bb 100644 --- a/chat2db-community-client/src/blocks/ImportAndExport/components/ImportExportFile/index.tsx +++ b/chat2db-community-client/src/blocks/ImportAndExport/components/ImportExportFile/index.tsx @@ -1,32 +1,31 @@ import { memo, useMemo, useState, forwardRef, ForwardedRef, useImperativeHandle, useEffect } from 'react'; import { useStyles } from './style'; import UploadLocalFile, { type FileUrl } from '@/components/UploadLocalFile'; -import { Form, Input, Select } from 'antd'; +import { Alert, Checkbox, Form, Input, Select, InputNumber, Switch, Table, Tooltip } from 'antd'; import i18n from '@/i18n'; import { useImportExportStore } from '@/store/importExport'; import { IconButton } from '@chat2db/ui'; import { ImportExportType, ImportExportFileType, ImportExportTaskType } from '@/constants/importExport'; -import { ExportTaskParams, ImportTaskParams } from '@/service/importExport'; -import { isDesktop, isDevelopment } from '@/utils/env'; +import importExportServices, { ExportTaskParams, ImportTaskParams } from '@/service/importExport'; +import { IImportPreview, ImportExecutionMode } from '@/typings/importExport'; +import { isDesktop } from '@/utils/env'; import jcefApi from '@/jcef'; -import { hasSelectedImportFile } from './selection'; +import { CircleHelp } from 'lucide-react'; +import { buildTaskParams, initialFileType, type ImportExportFormValue } from './taskParams'; +import sqlService from '@/service/sql'; +import { stageSelectedImportFile } from '../ImportMappingContent/fileStaging'; +import { getImportPreviewErrorMessage } from '../ImportMappingContent/mapping'; +import { getImportMappingIssues, mergeImportColumnMappings } from './mappingValidation'; interface IProps { className?: string; setIsReady?: (p: boolean) => void; - onImportFileChange?: (file?: FileUrl) => void; } export interface ImportExportFileRef { getValues: () => ExportTaskParams | ImportTaskParams | null; } -interface ImportExportFormValue { - exportType: ImportExportFileType; - containsHeader: boolean; - fileUrl?: string; -} - const exportTypeOptions = [ { label: 'CSV', value: ImportExportFileType.CSV, accept: '.csv' }, { label: 'XLSX', value: ImportExportFileType.XLSX, accept: '.xlsx' }, @@ -35,91 +34,260 @@ const exportTypeOptions = [ { label: 'SQL', value: ImportExportFileType.SQL, accept: '.sql' }, ]; +// The import backend parses these formats; NDJSON/Markdown are export-only. +const importTypeOptions = exportTypeOptions.filter( + (option) => option.value !== ImportExportFileType.NDJSON && option.value !== ImportExportFileType.MARKDOWN, +); + +// Formats that can be checkpointed for resumable export. +const checkpointableFormats = [ + ImportExportFileType.CSV, + ImportExportFileType.NDJSON, + ImportExportFileType.MARKDOWN, + ImportExportFileType.SQL, +]; + const ImportExportFile = forwardRef((props: IProps, ref: ForwardedRef) => { - const { setIsReady, onImportFileChange } = props; + const { setIsReady } = props; const { styles } = useStyles(); const [form] = Form.useForm(); - const [selectedFilePaths, setSelectedFilePaths] = useState([]); - const [exportLocation, setExportLocation] = useState(''); - const [formValue, setFormValue] = useState({ - exportType: ImportExportFileType.CSV, - containsHeader: true, - }); - const { importExportDataBoundInfo } = useImportExportStore((state) => { return { importExportDataBoundInfo: state.importExportDataBoundInfo, }; }); + const defaultFileType = importExportDataBoundInfo + ? initialFileType(importExportDataBoundInfo) + : ImportExportFileType.CSV; + const [selectedFile, setSelectedFile] = useState(); + const [stagedFile, setStagedFile] = useState<{ selection: FileUrl; id: string }>(); + const [previewLoading, setPreviewLoading] = useState(false); + const [importError, setImportError] = useState(); + const [exportLocation, setExportLocation] = useState(''); + const [importPreview, setImportPreview] = useState(null); + const [columnMappings, setColumnMappings] = useState>({}); + const [formValue, setFormValue] = useState({ + exportType: defaultFileType, + containsHeader: true, + }); + const [mode, setMode] = useState('STANDARD'); + const [confirmedNoStrongRelations, setConfirmedNoStrongRelations] = useState(false); const isImport = importExportDataBoundInfo?.type === ImportExportType.IMPORT; const isExport = importExportDataBoundInfo?.type === ImportExportType.EXPORT; + const isTableTarget = importExportDataBoundInfo?.targetScope === 'TABLE'; + const fileTypeOptions = importExportDataBoundInfo?.fileType + ? exportTypeOptions.filter((option) => option.value === importExportDataBoundInfo.fileType) + : isImport + ? importTypeOptions + : exportTypeOptions; useEffect(() => { if (importExportDataBoundInfo) { const { dataSourceName, databaseName, schemaName, tableName } = importExportDataBoundInfo; const tableNameDisplay = [dataSourceName, databaseName, schemaName, tableName].filter(Boolean).join('/'); + const exportType = initialFileType(importExportDataBoundInfo); + const initialValues: ImportExportFormValue = { + exportType, + containsHeader: true, + }; + setFormValue(initialValues); + setSelectedFile(undefined); + setStagedFile(undefined); + setImportError(undefined); + setExportLocation(''); + setImportPreview(null); + setColumnMappings({}); + setMode('STANDARD'); + setConfirmedNoStrongRelations(false); + form.resetFields(); form.setFieldsValue({ tableNameDisplay: tableNameDisplay, + ...initialValues, }); } - }, [importExportDataBoundInfo]); + }, [form, importExportDataBoundInfo]); // Gets the corresponding file type based on the export type const uploadLocalFileAccept = useMemo(() => { return formValue.exportType ? exportTypeOptions.find((item) => item.value === formValue.exportType)?.accept : ''; }, [formValue.exportType]); + // Both browser and desktop selections use the server's opaque staged-file contract. useEffect(() => { - if (isExport) { - setIsReady?.(!isDesktop || !!exportLocation || !!formValue.fileUrl); + setStagedFile(undefined); + setImportPreview(null); + setImportError(undefined); + if (!selectedFile) return; + let cancelled = false; + stageSelectedImportFile(selectedFile, sqlService.uploadImportFile, sqlService.stageDesktopImportFile) + .then((id) => { + if (!cancelled) setStagedFile({ selection: selectedFile, id }); + }) + .catch((error) => { + if (!cancelled) setImportError(getImportPreviewErrorMessage(error, i18n('common.text.failure'))); + }); + return () => { + cancelled = true; + }; + }, [selectedFile]); + + // Previews the selected import file once both the file and the format are known, so the + // column mapping panel below reflects what the backend will actually import. + const fileId = stagedFile?.selection === selectedFile ? stagedFile?.id : undefined; + const previewableFormat = + isImport && + isTableTarget && + [ImportExportFileType.CSV, ImportExportFileType.XLS, ImportExportFileType.XLSX].includes(formValue.exportType); + useEffect(() => { + setImportPreview(null); + setPreviewLoading(false); + if (!previewableFormat || !fileId || !importExportDataBoundInfo) { + setImportPreview(null); + return () => {}; } - }, [exportLocation, formValue]); + let cancelled = false; + setPreviewLoading(true); + setImportError(undefined); + const { dataSourceId, databaseName, schemaName, tableName } = importExportDataBoundInfo; + importExportServices + .previewImport({ + dataSourceId, + databaseName, + schemaName, + taskType: ImportExportTaskType.DATA_FILE_IMPORT, + format: formValue.exportType, + tableName, + fileId, + mode, + confirmedNoStrongRelations, + options: + formValue.exportType === ImportExportFileType.CSV + ? { + charset: formValue.charset || undefined, + delimiter: formValue.delimiter || undefined, + quoteChar: formValue.quoteChar || undefined, + } + : undefined, + }) + .then((preview) => { + if (cancelled) return; + setImportPreview(preview); + setColumnMappings((current) => mergeImportColumnMappings(preview, current)); + }) + .catch((error) => { + if (!cancelled) setImportError(getImportPreviewErrorMessage(error, i18n('common.text.failure'))); + }) + .finally(() => { + if (!cancelled) setPreviewLoading(false); + }); + return () => { + cancelled = true; + }; + }, [ + previewableFormat, + fileId, + formValue.exportType, + formValue.charset, + formValue.delimiter, + formValue.quoteChar, + mode, + confirmedNoStrongRelations, + importExportDataBoundInfo, + ]); + + const targetColumnOptions = useMemo(() => { + if (!importPreview) return []; + const names = new Set(); + importPreview.columnMatches.forEach((match) => { + if (match.tableColumn) names.add(match.tableColumn); + }); + importPreview.missingTableColumns.forEach((name) => names.add(name)); + return Array.from(names) + .sort() + .map((name) => ({ label: name, value: name })); + }, [importPreview]); + + const mappingIssues = getImportMappingIssues(importPreview, columnMappings, formValue.unmappedTarget); + const admission = importPreview?.parallelAdmission; + const unsupportedParallelFormat = + isImport && mode === 'ULTRA_FAST' && formValue.exportType !== ImportExportFileType.CSV; + const parallelForbidden = + isImport && + mode === 'ULTRA_FAST' && + (unsupportedParallelFormat || !confirmedNoStrongRelations || admission?.verdict === 'PARALLEL_FORBIDDEN'); + const importReady = + !!fileId && + !importError && + !previewLoading && + !parallelForbidden && + (!previewableFormat || + (!!importPreview && !mappingIssues.duplicate && !mappingIssues.empty && !mappingIssues.required.length)); + useEffect(() => { + if (isImport) setIsReady?.(importReady); + }, [isImport, importReady, setIsReady]); - const handleSelectedFilesChange = (files: FileUrl[]) => { - setSelectedFilePaths(files.map((item) => item.filePath).filter((path): path is string => !!path)); - if (isImport) { - setIsReady?.(hasSelectedImportFile(files)); - onImportFileChange?.(files[0]); + useEffect(() => { + if (isExport) { + setIsReady?.(!isDesktop || !!exportLocation); } + }, [exportLocation, isExport, setIsReady, importExportDataBoundInfo]); + + const handleFileUrlListChange = (files: FileUrl[]) => { + setIsReady?.(false); + setColumnMappings({}); + setSelectedFile(files[0]); }; useImperativeHandle(ref, () => ({ getValues: () => { if (!importExportDataBoundInfo) return null; - const { dataSourceId, databaseName, schemaName, tableName } = importExportDataBoundInfo; - const commonValues = { - dataSourceId, - databaseName, - schemaName, - format: formValue.exportType, - }; - if (isExport) { - return { - ...commonValues, - taskType: ImportExportTaskType.TABLE_DATA_EXPORT, - tableNames: [tableName], - containsHeader: formValue.containsHeader, - exportPath: exportLocation || formValue.fileUrl, - }; - } - return { - ...commonValues, - taskType: - formValue.exportType === ImportExportFileType.SQL - ? ImportExportTaskType.SQL_FILE_IMPORT - : ImportExportTaskType.DATA_FILE_IMPORT, - tableName, - sourceFile: selectedFilePaths[0] || '', - }; + if (isImport && !importReady) return null; + const params = buildTaskParams({ + boundInfo: importExportDataBoundInfo, + formValue, + mode, + sourceFile: '', + exportLocation, + desktop: isDesktop, + importPreview, + columnMappings, + checkpointableFormats, + confirmedNoStrongRelations, + }); + return isImport + ? ({ + ...params, + sourceFile: undefined, + fileId, + displayFileName: selectedFile?.fileName || selectedFile?.file?.name, + } as ImportTaskParams) + : params; }, })); const handleFormChange = (changedValues, allValues) => { - setFormValue({ + if (changedValues.exportType) { + setSelectedFile(undefined); + setColumnMappings({}); + setIsReady?.(false); + } + if (['charset', 'delimiter', 'quoteChar'].some((key) => key in changedValues)) { + setImportPreview(null); + setIsReady?.(false); + } + const nextValue: ImportExportFormValue = { ...formValue, ...allValues, - }); + }; + if ( + changedValues.compression || + (changedValues.exportType && !checkpointableFormats.includes(changedValues.exportType)) + ) { + form.setFieldValue('checkpointRows', undefined); + nextValue.checkpointRows = undefined; + } + setFormValue(nextValue); }; const handleSelectExportLocation = async () => { @@ -128,6 +296,26 @@ const ImportExportFile = forwardRef((props: IProps, ref: ForwardedRef { + setMode(checked ? 'ULTRA_FAST' : 'STANDARD'); + if (!checked) setConfirmedNoStrongRelations(false); + }; + + const ultraModeTooltip = ( +

+
{i18n('workspace.importExport.ultraModeHint')}
+
{i18n('workspace.importExport.ultraModeConfirmIntro')}
+
    +
  • {i18n('workspace.importExport.ultraModeBenefit1')}
    • +
  • {i18n('workspace.importExport.ultraModeBenefit2')}
    • +
  • {i18n('workspace.importExport.ultraModeBenefit3')}
    • +
  • {i18n('workspace.importExport.ultraModeRisk1')}
    • +
  • {i18n('workspace.importExport.ultraModeRisk2')}
    • +
  • {i18n('workspace.importExport.ultraModeRisk3')}
    • +
+
+ ); + return (
- - - - {isExport && isDesktop && (
- +
)} {isImport && ( - - + + )} - {isDevelopment && isExport && ( - - - + {isImport && formValue.exportType === ImportExportFileType.CSV && ( + <> + + + + + + + + + + + + + + + + + setFormValue((previous) => ({ ...previous, unmappedTarget }))} + /> + {(importPreview.detectedCharset || importPreview.detectedDelimiter) && ( +
+ {importPreview.detectedCharset && ( + + {i18n('workspace.importExport.charset')}: {importPreview.detectedCharset} + + )} + {importPreview.detectedDelimiter && ( + + {i18n('workspace.importExport.delimiter')}: {importPreview.detectedDelimiter} + + )} +
+ )} + {importPreview.columnMatches.map((match) => ( +
+ + {match.fileColumn} + + + + +
+ + + + +
+ + {isImport && mode === 'ULTRA_FAST' && ( +
+ `[${finding.code}]`) + .join(', ')}. ${i18n('workspace.importExport.parallelAdmissionEffectiveMode')}: ${ + admission.effectiveMode + }` + : undefined + } + /> + setConfirmedNoStrongRelations(event.target.checked)} + > + {i18n('workspace.importExport.ultraModeAcknowledge')} + +
)} +
+ {mode === 'ULTRA_FAST' + ? i18n('workspace.importExport.modeBadgeUltra') + : i18n('workspace.importExport.modeBadgeStandard')} +
{/* {i18n('workspace.importExport.containsHeader')} */} diff --git a/chat2db-community-client/src/blocks/ImportAndExport/components/ImportExportFile/mappingValidation.test.ts b/chat2db-community-client/src/blocks/ImportAndExport/components/ImportExportFile/mappingValidation.test.ts new file mode 100644 index 0000000000..0ac4ae22b0 --- /dev/null +++ b/chat2db-community-client/src/blocks/ImportAndExport/components/ImportExportFile/mappingValidation.test.ts @@ -0,0 +1,42 @@ +import assert from 'node:assert/strict'; +import { getImportMappingIssues, mergeImportColumnMappings } from './mappingValidation'; +import type { IImportPreview } from '@/typings/importExport'; + +const preview: IImportPreview = { + fileColumns: ['Full Name', 'status'], + columnMatches: [], + sampleRows: [], + missingTableColumns: [], + targetColumns: [ + { name: 'id', dataType: 'INT', nullable: false, autoIncrement: true, comment: null, defaultValue: null }, + { name: 'name', dataType: 'VARCHAR', nullable: false, autoIncrement: false, comment: null, defaultValue: null }, + { name: 'status', dataType: 'VARCHAR', nullable: false, autoIncrement: false, comment: null, defaultValue: "'NEW'" }, + ], +}; +assert.deepEqual(getImportMappingIssues(preview, { 'Full Name': 'name' }).required, []); +assert.deepEqual( + getImportMappingIssues(preview, { 'Full Name': 'name' }, 'NULL').required.map((column) => column.name), + ['status'], +); +assert.equal(getImportMappingIssues(preview, { 'Full Name': 'name', status: 'NAME' }).duplicate, true); +assert.equal(getImportMappingIssues(preview, { 'Full Name': undefined }).empty, true); +assert.deepEqual( + getImportMappingIssues(preview, { status: 'status' }).required.map((column) => column.name), + ['name'], +); +const refreshedPreview: IImportPreview = { + ...preview, + columnMatches: [ + { fileColumn: 'Full Name', matched: false }, + { fileColumn: 'status', tableColumn: 'status', matched: true }, + ], +}; +assert.deepEqual(mergeImportColumnMappings(refreshedPreview, { 'Full Name': 'name', status: undefined }), { + 'Full Name': 'name', + status: undefined, +}); +assert.deepEqual(mergeImportColumnMappings(refreshedPreview, { 'Full Name': 'removed_column' }), { + 'Full Name': undefined, + status: 'status', +}); +console.log('Import mapping validation and refresh-preservation checks passed'); diff --git a/chat2db-community-client/src/blocks/ImportAndExport/components/ImportExportFile/mappingValidation.ts b/chat2db-community-client/src/blocks/ImportAndExport/components/ImportExportFile/mappingValidation.ts new file mode 100644 index 0000000000..400ff35722 --- /dev/null +++ b/chat2db-community-client/src/blocks/ImportAndExport/components/ImportExportFile/mappingValidation.ts @@ -0,0 +1,40 @@ +import type { IImportPreview } from '@/typings/importExport'; + +export const mergeImportColumnMappings = ( + preview: IImportPreview, + previous: Record, +) => { + const validTargets = new Set(preview.targetColumns.map((column) => column.name)); + return Object.fromEntries( + preview.columnMatches.map((match) => { + if (Object.prototype.hasOwnProperty.call(previous, match.fileColumn)) { + const previousTarget = previous[match.fileColumn]; + if (!previousTarget || validTargets.has(previousTarget)) { + return [match.fileColumn, previousTarget]; + } + } + return [match.fileColumn, match.matched ? match.tableColumn : undefined]; + }), + ); +}; + +export const getImportMappingIssues = ( + preview: IImportPreview | null, + mapping: Record, + strategy: 'DEFAULT' | 'NULL' = 'DEFAULT', +) => { + const targets = Object.values(mapping).filter((target): target is string => !!target); + const normalizedTargets = targets.map((target) => target.trim().toLowerCase()); + return { + duplicate: new Set(normalizedTargets).size !== targets.length, + empty: targets.length === 0, + required: (preview?.targetColumns || []).filter( + (column) => + !column.nullable && + !column.autoIncrement && + !targets.includes(column.name) && + (strategy === 'NULL' || column.defaultValue == null), + ), + unmapped: (preview?.targetColumns || []).filter((column) => !targets.includes(column.name)), + }; +}; diff --git a/chat2db-community-client/src/blocks/ImportAndExport/components/ImportExportFile/style.ts b/chat2db-community-client/src/blocks/ImportAndExport/components/ImportExportFile/style.ts index d6c5cd8a46..5b6787a5d9 100644 --- a/chat2db-community-client/src/blocks/ImportAndExport/components/ImportExportFile/style.ts +++ b/chat2db-community-client/src/blocks/ImportAndExport/components/ImportExportFile/style.ts @@ -1,6 +1,6 @@ import { createStyles } from 'antd-style'; -export const useStyles = createStyles(({ css }) => { +export const useStyles = createStyles(({ css, token }) => { return { checkboxBody: css` .ant-form-item { @@ -20,7 +20,115 @@ export const useStyles = createStyles(({ css }) => { border-radius: 6px !important; `, form: css` + display: grid; + grid-template-columns: repeat(2, minmax(0, 1fr)); + column-gap: 16px; padding-top: 20px; + + @media (max-width: 640px) { + grid-template-columns: minmax(0, 1fr); + } + `, + fullWidth: css` + grid-column: 1 / -1; + `, + previewPanel: css` + grid-column: 1 / -1; + min-width: 0; + display: flex; + flex-direction: column; + gap: 8px; + padding: 12px; + border: 1px solid rgba(128, 128, 128, 0.25); + border-radius: 8px; + margin-bottom: 16px; + `, + previewTitle: css` + font-weight: 600; + `, + previewMeta: css` + display: flex; + gap: 16px; + color: rgba(128, 128, 128, 1); + font-size: 12px; + `, + previewRow: css` + display: flex; + align-items: center; + gap: 8px; + .ant-select { + flex: 1; + } + `, + previewFileColumn: css` + width: 120px; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + `, + previewWarning: css` + color: rgba(230, 162, 60, 1); + font-size: 12px; + `, + modeControl: css` + display: flex; + align-items: center; + gap: 8px; + min-height: 22px; + `, + modeHelpButton: css` + display: inline-flex; + align-items: center; + justify-content: center; + width: 20px; + height: 20px; + padding: 0; + color: ${token.colorTextTertiary}; + cursor: help; + background: transparent; + border: 0; + + &:hover { + color: ${token.colorPrimary}; + } + + &:focus-visible { + outline: 2px solid currentColor; + outline-offset: 2px; + } + `, + modeTooltip: css` + width: min(400px, calc(100vw - 48px)); + + ul { + padding-left: 18px; + margin: 6px 0 0; + } + + li + li { + margin-top: 4px; + } + `, + modeTooltipTitle: css` + margin-top: 8px; + font-weight: 600; + `, + modeIndicator: css` + grid-column: 1 / -1; + margin: -8px 0 12px; + color: rgba(128, 128, 128, 0.75); + font-size: 12px; + `, + admissionPanel: css` + grid-column: 1 / -1; + display: flex; + flex-direction: column; + gap: 10px; + margin: -6px 0 12px; + + .ant-alert-description { + white-space: pre-line; + } `, }; }); diff --git a/chat2db-community-client/src/blocks/ImportAndExport/components/ImportExportFile/taskParams.test.ts b/chat2db-community-client/src/blocks/ImportAndExport/components/ImportExportFile/taskParams.test.ts new file mode 100644 index 0000000000..002f486d71 --- /dev/null +++ b/chat2db-community-client/src/blocks/ImportAndExport/components/ImportExportFile/taskParams.test.ts @@ -0,0 +1,149 @@ +import assert from 'node:assert/strict'; +import { ImportExportFileType, ImportExportTaskType, ImportExportType } from '@/constants/importExport'; +import type { ImportExportDataBoundInfo } from '@/typings/importExport'; +import { buildTaskParams, initialFileType, type ImportExportFormValue } from './taskParams'; + +const checkpointableFormats = [ + ImportExportFileType.CSV, + ImportExportFileType.NDJSON, + ImportExportFileType.MARKDOWN, + ImportExportFileType.SQL, +]; +const baseForm: ImportExportFormValue = { + exportType: ImportExportFileType.CSV, + containsHeader: true, +}; + +function build(boundInfo: ImportExportDataBoundInfo, overrides: Partial = {}) { + return buildTaskParams({ + boundInfo, + formValue: { ...baseForm, ...overrides }, + mode: 'ULTRA_FAST', + sourceFile: 'C:\\imports\\dump.sql', + exportLocation: 'C:\\exports', + desktop: true, + importPreview: null, + columnMappings: {}, + checkpointableFormats, + }); +} + +const tableExport: ImportExportDataBoundInfo = { + dataSourceId: 1, + databaseName: 'app', + schemaName: 'public', + tableName: 'orders', + targetScope: 'TABLE', + type: ImportExportType.EXPORT, +}; +assert.equal(initialFileType(tableExport), ImportExportFileType.CSV); +assert.deepEqual(build(tableExport, { checkpointRows: 10000 }), { + dataSourceId: 1, + databaseName: 'app', + schemaName: 'public', + format: ImportExportFileType.CSV, + mode: 'ULTRA_FAST', + taskType: ImportExportTaskType.TABLE_DATA_EXPORT, + tableNames: ['orders'], + containsHeader: true, + exportPath: 'C:\\exports', + compression: undefined, + checkpointRows: 10000, +}); + +const schemaExport: ImportExportDataBoundInfo = { + dataSourceId: 1, + databaseName: 'app', + schemaName: 'reporting', + targetScope: 'SCHEMA', + type: ImportExportType.EXPORT, + fileType: ImportExportFileType.SQL, + sqlExportScope: 'ALL', +}; +assert.equal(initialFileType(schemaExport), ImportExportFileType.SQL); +assert.deepEqual(build(schemaExport, { exportType: ImportExportFileType.SQL, compression: 'GZIP' }), { + dataSourceId: 1, + databaseName: 'app', + schemaName: 'reporting', + format: ImportExportFileType.SQL, + mode: 'ULTRA_FAST', + taskType: ImportExportTaskType.SQL_EXPORT, + tableNames: undefined, + scope: 'ALL', + containData: true, + containsHeader: true, + exportPath: 'C:\\exports', +}); + +const databaseImport: ImportExportDataBoundInfo = { + dataSourceId: 1, + databaseName: 'app', + targetScope: 'DATABASE', + type: ImportExportType.IMPORT, + fileType: ImportExportFileType.SQL, +}; +assert.deepEqual(build(databaseImport, { exportType: ImportExportFileType.SQL }), { + dataSourceId: 1, + databaseName: 'app', + schemaName: undefined, + format: ImportExportFileType.SQL, + mode: 'ULTRA_FAST', + taskType: ImportExportTaskType.SQL_FILE_IMPORT, + tableName: undefined, + sourceFile: 'C:\\imports\\dump.sql', + options: undefined, +}); + +const tableImport: ImportExportDataBoundInfo = { + ...tableExport, + type: ImportExportType.IMPORT, +}; +const mappedImport = buildTaskParams({ + boundInfo: tableImport, + formValue: { ...baseForm, delimiter: ';', onError: 'SKIP', maxErrors: 5 }, + mode: 'STANDARD', + sourceFile: 'C:\\imports\\orders.csv', + exportLocation: '', + desktop: true, + importPreview: { + fileColumns: ['order_id'], + columnMatches: [], + missingTableColumns: [], + sampleRows: [], + }, + columnMappings: { order_id: 'id' }, + checkpointableFormats, +}); +assert.equal(mappedImport.taskType, ImportExportTaskType.DATA_FILE_IMPORT); +assert.equal('tableName' in mappedImport ? mappedImport.tableName : undefined, 'orders'); +assert.deepEqual('options' in mappedImport ? mappedImport.options : undefined, { + charset: undefined, + delimiter: ';', + quoteChar: undefined, + skipRows: undefined, + nullString: undefined, + onError: 'SKIP', + maxErrors: 5, + columnMappings: [{ sourceColumn: 'order_id', targetColumn: 'id' }], +}); + +const confirmedParallelImport = buildTaskParams({ + boundInfo: tableImport, + formValue: baseForm, + mode: 'ULTRA_FAST', + sourceFile: 'C:\\imports\\orders.csv', + exportLocation: '', + desktop: true, + importPreview: null, + columnMappings: {}, + checkpointableFormats, + confirmedNoStrongRelations: true, +}); +assert.equal( + 'confirmedNoStrongRelations' in confirmedParallelImport + ? confirmedParallelImport.confirmedNoStrongRelations + : undefined, + true, +); + +console.log('Import/export wizard task parameter tests passed'); diff --git a/chat2db-community-client/src/blocks/ImportAndExport/components/ImportExportFile/taskParams.ts b/chat2db-community-client/src/blocks/ImportAndExport/components/ImportExportFile/taskParams.ts new file mode 100644 index 0000000000..e619d9520b --- /dev/null +++ b/chat2db-community-client/src/blocks/ImportAndExport/components/ImportExportFile/taskParams.ts @@ -0,0 +1,126 @@ +import { ImportExportFileType, ImportExportTaskType, ImportExportType } from '@/constants/importExport'; +import type { ExportTaskParams, ImportTaskParams } from '@/service/importExport'; +import type { + IImportColumnMapping, + IImportOptions, + IImportPreview, + ImportExecutionMode, + ImportExportDataBoundInfo, +} from '@/typings/importExport'; + +export interface ImportExportFormValue { + exportType: ImportExportFileType; + containsHeader: boolean; + compression?: string; + checkpointRows?: number; + charset?: string; + delimiter?: string; + quoteChar?: string; + skipRows?: number; + nullString?: string; + onError?: 'ABORT' | 'SKIP'; + maxErrors?: number; + unmappedTarget?: 'DEFAULT' | 'NULL'; +} + +interface BuildTaskParamsInput { + boundInfo: ImportExportDataBoundInfo; + formValue: ImportExportFormValue; + mode: ImportExecutionMode; + sourceFile: string; + exportLocation: string; + desktop: boolean; + importPreview: IImportPreview | null; + columnMappings: Record; + checkpointableFormats: ImportExportFileType[]; + confirmedNoStrongRelations?: boolean; +} + +export function initialFileType(boundInfo: ImportExportDataBoundInfo): ImportExportFileType { + if (boundInfo.fileType) return boundInfo.fileType; + return boundInfo.targetScope === 'TABLE' ? ImportExportFileType.CSV : ImportExportFileType.SQL; +} + +export function buildTaskParams({ + boundInfo, + formValue, + mode, + sourceFile, + exportLocation, + desktop, + importPreview, + columnMappings, + checkpointableFormats, + confirmedNoStrongRelations = false, +}: BuildTaskParamsInput): ExportTaskParams | ImportTaskParams { + const { dataSourceId, databaseName, schemaName, tableName } = boundInfo; + const commonValues = { + dataSourceId, + databaseName, + schemaName, + format: formValue.exportType, + mode, + }; + + if (boundInfo.type === ImportExportType.EXPORT) { + if (boundInfo.sqlExportScope) { + return { + ...commonValues, + taskType: ImportExportTaskType.SQL_EXPORT, + format: ImportExportFileType.SQL, + tableNames: tableName ? [tableName] : undefined, + scope: boundInfo.sqlExportScope, + containData: boundInfo.sqlExportScope === 'ALL', + containsHeader: formValue.containsHeader, + exportPath: desktop ? exportLocation : undefined, + }; + } + return { + ...commonValues, + taskType: ImportExportTaskType.TABLE_DATA_EXPORT, + tableNames: tableName ? [tableName] : undefined, + containsHeader: formValue.containsHeader, + exportPath: desktop ? exportLocation : undefined, + compression: formValue.compression || undefined, + checkpointRows: + !formValue.compression && checkpointableFormats.includes(formValue.exportType) + ? formValue.checkpointRows || undefined + : undefined, + }; + } + + const mappingList: IImportColumnMapping[] | undefined = importPreview + ? Object.entries(columnMappings) + .filter((entry) => !!entry[1]) + .map(([sourceColumn, targetColumn]) => ({ sourceColumn, targetColumn: targetColumn as string })) + : undefined; + const options: IImportOptions | undefined = + formValue.exportType === ImportExportFileType.SQL + ? undefined + : { + ...(formValue.exportType === ImportExportFileType.CSV + ? { + charset: formValue.charset || undefined, + delimiter: formValue.delimiter || undefined, + quoteChar: formValue.quoteChar || undefined, + skipRows: formValue.skipRows || undefined, + nullString: formValue.nullString || undefined, + onError: formValue.onError || undefined, + maxErrors: formValue.onError === 'SKIP' ? formValue.maxErrors || undefined : undefined, + } + : {}), + ...(mappingList ? { columnMappings: mappingList } : {}), + }; + return { + ...commonValues, + taskType: + formValue.exportType === ImportExportFileType.SQL + ? ImportExportTaskType.SQL_FILE_IMPORT + : ImportExportTaskType.DATA_FILE_IMPORT, + tableName: boundInfo.targetScope === 'TABLE' ? tableName : undefined, + sourceFile, + ...(mode === 'ULTRA_FAST' && confirmedNoStrongRelations ? { confirmedNoStrongRelations: true } : {}), + options, + ...(importPreview ? { unmappedTarget: formValue.unmappedTarget || 'DEFAULT' } : {}), + }; +} diff --git a/chat2db-community-client/src/blocks/ImportAndExport/components/ImportFileModal/index.tsx b/chat2db-community-client/src/blocks/ImportAndExport/components/ImportFileModal/index.tsx index 700fb5a52e..843d311ca5 100644 --- a/chat2db-community-client/src/blocks/ImportAndExport/components/ImportFileModal/index.tsx +++ b/chat2db-community-client/src/blocks/ImportAndExport/components/ImportFileModal/index.tsx @@ -5,37 +5,28 @@ import i18n from '@/i18n'; import ImportExportFile, { ImportExportFileRef } from '../ImportExportFile'; import { useImportExportStore } from '@/store/importExport'; import ModalFooterButton from '@/components/Modal/ModalFooterButton'; -import importExportServices from '@/service/importExport'; +import importExportServices, { type ExportTaskParams, type ImportTaskParams } from '@/service/importExport'; import { ImportExportTaskStatus, ImportExportType } from '@/constants/importExport'; import Log from '@/blocks/ImportAndExport/components/Log'; import { ImportExportTaskDetails } from '@/typings/importExport'; -import ImportMappingContent from '@/blocks/ImportAndExport/components/ImportMappingContent'; import jcefApi from '@/jcef'; import { isDesktop } from '@/utils/env'; -import sqlService from '@/service/sql'; -import { prepareWebImportParams } from './submission'; import { IMPORT_TARGET_TABLE_REFRESH_EVENT, shouldRefreshImportTargetTable, } from '@/store/importExport/taskCenterUtils'; -import type { FileUrl } from '@/components/UploadLocalFile'; interface IProps { className?: string; } -const isPreviewFile = (file?: FileUrl) => { - const name = (file?.fileName || file?.file?.name)?.toLowerCase(); - return name?.endsWith('.csv') || name?.endsWith('.xls') || name?.endsWith('.xlsx'); -}; - export default memo((_props) => { const [isReady, setIsReady] = useState(false); const importExportFileRef = useRef(null); - const previousTaskDetailsRef = useRef(); const [taskId, setTaskId] = useState(); const [taskDetails, setTaskDetails] = useState(); - const [importFile, setImportFile] = useState(); + const previousTaskDetailsRef = useRef(); + const [submitting, setSubmitting] = useState(false); const { importExportDataBoundInfo, setImportExportDataBoundInfo, getTaskList } = useImportExportStore((state) => { return { @@ -47,33 +38,29 @@ export default memo((_props) => { useEffect(() => { if (!importExportDataBoundInfo) { + setIsReady(false); setTaskId(undefined); setTaskDetails(undefined); previousTaskDetailsRef.current = undefined; - setImportFile(undefined); } }, [importExportDataBoundInfo]); - const handleRunSQl = async () => { + const handleRunSQl = () => { + if (submitting) return; const params = importExportFileRef.current?.getValues(); if (!params) return; - let response; - if ('sourceFile' in params) { - let importParams = params; - if (!isDesktop) { - if (!importFile?.file) return; - importParams = await prepareWebImportParams(importParams, importFile.file, sqlService.uploadImportFile); - } - response = await importExportServices.submitImport(importParams); - } else { - response = await importExportServices.submitExport(params); - } - setTaskId(response.taskId); - getTaskList(); - }; - - const handleImportFileChange = (file?: FileUrl) => { - setImportFile(file); + setSubmitting(true); + const request = + params.taskType === 'DATA_FILE_IMPORT' || params.taskType === 'SQL_FILE_IMPORT' + ? importExportServices.submitImport(params as ImportTaskParams) + : importExportServices.submitExport(params as ExportTaskParams); + request + .then((res) => { + setTaskId(res.taskId); + getTaskList(); + }) + .catch(() => {}) + .finally(() => setSubmitting(false)); }; const renderFooter = () => { @@ -88,7 +75,7 @@ export default memo((_props) => { > {i18n('common.button.cancel')} - @@ -133,50 +120,41 @@ export default memo((_props) => { ); const handleTaskChange = (_taskDetails: ImportExportTaskDetails) => { - const previousTask = previousTaskDetailsRef.current; + const previous = previousTaskDetailsRef.current; previousTaskDetailsRef.current = _taskDetails; setTaskDetails(_taskDetails); - if (shouldRefreshImportTargetTable(previousTask, _taskDetails)) { - window.dispatchEvent( - new CustomEvent(IMPORT_TARGET_TABLE_REFRESH_EVENT, { - detail: _taskDetails.target, - }), - ); + if (shouldRefreshImportTargetTable(previous, _taskDetails)) { + window.dispatchEvent(new CustomEvent(IMPORT_TARGET_TABLE_REFRESH_EVENT, { detail: _taskDetails.target })); void getTaskList(); } }; - const importPreviewContext = - importExportDataBoundInfo?.type === ImportExportType.IMPORT && - isPreviewFile(importFile) && - importExportDataBoundInfo.dataSourceId != null && - importExportDataBoundInfo.databaseName != null && - importFile != null - ? { - dataSourceId: importExportDataBoundInfo.dataSourceId, - databaseName: importExportDataBoundInfo.databaseName, - schemaName: importExportDataBoundInfo.schemaName, - tableName: importExportDataBoundInfo.tableName || '', - file: importFile, - } - : null; - const showImportPreview = taskId == null && importPreviewContext != null; + const modalTitle = (() => { + if (importExportDataBoundInfo?.type === ImportExportType.IMPORT) { + return importExportDataBoundInfo.targetScope === 'TABLE' + ? i18n('workspace.menu.importData') + : i18n('workspace.menu.runSqlFile'); + } + if (importExportDataBoundInfo?.sqlExportScope === 'SCHEMA') { + return i18n('workspace.menu.exportStructure'); + } + if (importExportDataBoundInfo?.sqlExportScope === 'ALL') { + return i18n('workspace.menu.exportStructureData'); + } + return i18n('workspace.menu.exportData'); + })(); return ( { setImportExportDataBoundInfo(null); @@ -184,24 +162,8 @@ export default memo((_props) => { > {taskId ? ( - ) : importPreviewContext ? ( - { - setTaskId(submittedTaskId); - getTaskList(); - }} - /> ) : ( - + )} ); diff --git a/chat2db-community-client/src/blocks/ImportAndExport/components/ImportFileModal/submission.test.ts b/chat2db-community-client/src/blocks/ImportAndExport/components/ImportFileModal/submission.test.ts index 0f5420ac45..10599d267f 100644 --- a/chat2db-community-client/src/blocks/ImportAndExport/components/ImportFileModal/submission.test.ts +++ b/chat2db-community-client/src/blocks/ImportAndExport/components/ImportFileModal/submission.test.ts @@ -3,7 +3,7 @@ import { ImportExportFileType, ImportExportTaskType } from '@/constants/importEx import { prepareWebImportParams } from './submission'; const file = { name: 'users.json' } as File; -const params = { +const params: import('@/service/importExport').ImportTaskParams = { dataSourceId: 1, databaseName: 'app', tableName: 'users', diff --git a/chat2db-community-client/src/blocks/ImportAndExport/index.tsx b/chat2db-community-client/src/blocks/ImportAndExport/index.tsx index ef44b74378..fbdcb89480 100644 --- a/chat2db-community-client/src/blocks/ImportAndExport/index.tsx +++ b/chat2db-community-client/src/blocks/ImportAndExport/index.tsx @@ -1,6 +1,6 @@ import { memo } from 'react'; import RunSqlModal from './components/RunSqlModal'; -import ImportFileModal from './components/RunSql'; +import ImportFileModal from './components/ImportFileModal'; export default memo(() => { return ( diff --git a/chat2db-community-client/src/blocks/NewTree/hooks/useCreateRightClickMenu.tsx b/chat2db-community-client/src/blocks/NewTree/hooks/useCreateRightClickMenu.tsx index 598b9cb8a8..fb2a33b005 100644 --- a/chat2db-community-client/src/blocks/NewTree/hooks/useCreateRightClickMenu.tsx +++ b/chat2db-community-client/src/blocks/NewTree/hooks/useCreateRightClickMenu.tsx @@ -12,9 +12,10 @@ import { WorkspaceTabType, databaseTypeList, } from '@/constants'; -import { ImportExportType } from '@/constants/importExport'; +import { ImportExportFileType, ImportExportType } from '@/constants/importExport'; import { ShortcutAction } from '@/constants/shortcut'; import { TreeNodeData } from '@/typings'; +import type { ImportExportTargetScope } from '@/typings/importExport'; import { canImportExport } from '@/utils/env'; // ----- store ----- @@ -116,6 +117,13 @@ function handleMenuOptions(treeNodeType, databaseType) { return withDataSourceColorMenuOption(menuOptions, treeNodeType); } +function getImportExportTargetScope(treeNodeType: TreeNodeType): ImportExportTargetScope { + if (treeNodeType === TreeNodeType.TABLE) return 'TABLE'; + if (treeNodeType === TreeNodeType.SCHEMA) return 'SCHEMA'; + if (treeNodeType === TreeNodeType.DATABASE) return 'DATABASE'; + return 'DATA_SOURCE'; +} + // Node that can be double-clicked export const canBeDoubleClicked = [ TreeNodeType.TABLE, @@ -167,11 +175,10 @@ export const useCreateRightClickMenu = () => { }; }); - const { setImportExportDataBoundInfo, setRunSqlBoundInfo, getTaskList, openLogModal } = useImportExportStore( + const { setImportExportDataBoundInfo, getTaskList, openLogModal } = useImportExportStore( (state) => { return { setImportExportDataBoundInfo: state.setImportExportDataBoundInfo, - setRunSqlBoundInfo: state.setRunSqlBoundInfo, getTaskList: state.getTaskList, openLogModal: state.openLogModal, }; @@ -241,6 +248,7 @@ export const useCreateRightClickMenu = () => { }; const { supportSchema, supportDatabase } = getDatabaseSupport(databaseType); + const importExportTargetScope = getImportExportTargetScope(treeNodeType); const handelOpenCreateDatabaseModal = (type: 'database' | 'schema') => { const relyOnParams = { databaseType: treeNodeData.extraParams.databaseType!, @@ -1043,11 +1051,14 @@ export const useCreateRightClickMenu = () => { text: i18n('workspace.menu.runSqlFile'), icon: 'icon-run-sql', handle: () => { - setRunSqlBoundInfo({ + setImportExportDataBoundInfo({ dataSourceName: dataSourceName, dataSourceId: dataSourceId!, databaseName, schemaName, + targetScope: importExportTargetScope, + type: ImportExportType.IMPORT, + fileType: ImportExportFileType.SQL, }); }, discard: @@ -1164,6 +1175,7 @@ export const useCreateRightClickMenu = () => { databaseName, schemaName, tableName: tableName!, + targetScope: 'TABLE', type: ImportExportType.IMPORT, }); }, diff --git a/chat2db-community-client/src/constants/importExport.ts b/chat2db-community-client/src/constants/importExport.ts index eb7fb72542..f4a12998ed 100644 --- a/chat2db-community-client/src/constants/importExport.ts +++ b/chat2db-community-client/src/constants/importExport.ts @@ -9,9 +9,21 @@ export enum ImportExportFileType { XLS = 'XLS', XLSX = 'XLSX', JSON = 'JSON', + NDJSON = 'NDJSON', + MARKDOWN = 'MARKDOWN', SQL = 'SQL', } +export enum ImportExportCompression { + NONE = 'NONE', + GZIP = 'GZIP', +} + +export enum ImportOnError { + ABORT = 'ABORT', + SKIP = 'SKIP', +} + export enum ImportExportTaskType { QUERY_RESULT_EXPORT = 'QUERY_RESULT_EXPORT', SQL_EXPORT = 'SQL_EXPORT', diff --git a/chat2db-community-client/src/i18n/en-US/workspace.ts b/chat2db-community-client/src/i18n/en-US/workspace.ts index c9d361607d..74a936d79e 100644 --- a/chat2db-community-client/src/i18n/en-US/workspace.ts +++ b/chat2db-community-client/src/i18n/en-US/workspace.ts @@ -139,8 +139,7 @@ export default { 'workspace.importExport.ultraModeBenefit1': '✅ Supported table data imports and exports use multi-threaded parallel processing', 'workspace.importExport.ultraModeBenefit2': '✅ Adaptive thread count and batch size for the best configuration', - 'workspace.importExport.ultraModeBenefit3': - '✅ Runs full admission before workers start; small files use Standard and hard risks are rejected', + 'workspace.importExport.ultraModeBenefit3': '✅ Runs full admission before workers start; hard risks are rejected', 'workspace.importExport.ultraModeRisk1': '⚠️ Higher source-database connection pressure (MySQL pool ≥ 16 recommended)', 'workspace.importExport.ultraModeRisk2': '⚠️ Slightly higher memory usage (per-shard result set and buffer)', 'workspace.importExport.ultraModeRisk3': '⚠️ Not suitable for data with strict unprepared foreign keys (deadlocks possible)', @@ -156,7 +155,6 @@ export default { 'workspace.importExport.modeBadgeUltra': '[Ultra-Fast]', 'workspace.importExport.modeBadgeStandard': '[Standard]', 'workspace.importExport.charset': 'Charset', - 'workspace.importExport.delimiter': 'Delimiter', 'workspace.importExport.quoteChar': 'Quote character', 'workspace.importExport.skipRows': 'Skip rows', 'workspace.importExport.nullString': 'NULL string', diff --git a/chat2db-community-client/src/i18n/es-ES/workspace.ts b/chat2db-community-client/src/i18n/es-ES/workspace.ts index 8063d9b85d..22944981f8 100644 --- a/chat2db-community-client/src/i18n/es-ES/workspace.ts +++ b/chat2db-community-client/src/i18n/es-ES/workspace.ts @@ -435,7 +435,7 @@ export default { '✅ Las importaciones y exportaciones de datos de tabla compatibles usan procesamiento paralelo multihilo', 'workspace.importExport.ultraModeBenefit2': '✅ Ajuste adaptativo de hilos y tamaño de lote', 'workspace.importExport.ultraModeBenefit3': - '✅ Evalúa el archivo completo antes de iniciar workers; usa modo estándar para archivos pequeños y rechaza riesgos graves', + '✅ Evalúa el archivo completo antes de iniciar workers y rechaza riesgos graves', 'workspace.importExport.ultraModeRisk1': '⚠️ Mayor presión de conexiones en la base de datos origen (pool MySQL ≥ 16 recomendado)', 'workspace.importExport.ultraModeRisk2': '⚠️ Uso de memoria ligeramente mayor (ResultSet y buffer por fragmento)', 'workspace.importExport.ultraModeRisk3': '⚠️ No apto para datos con claves foráneas estrictas sin preparar (posibles deadlocks)', @@ -451,7 +451,6 @@ export default { 'workspace.importExport.modeBadgeUltra': '[Ultrarrápido]', 'workspace.importExport.modeBadgeStandard': '[Estándar]', 'workspace.importExport.charset': 'Conjunto de caracteres', - 'workspace.importExport.delimiter': 'Delimitador', 'workspace.importExport.quoteChar': 'Carácter de comillas', 'workspace.importExport.skipRows': 'Filas a omitir', 'workspace.importExport.nullString': 'Cadena NULL', diff --git a/chat2db-community-client/src/i18n/ja-JP/workspace.ts b/chat2db-community-client/src/i18n/ja-JP/workspace.ts index 3a4a7cec18..1eb51ee907 100644 --- a/chat2db-community-client/src/i18n/ja-JP/workspace.ts +++ b/chat2db-community-client/src/i18n/ja-JP/workspace.ts @@ -135,7 +135,7 @@ export default { '✅ 対応するテーブルデータのインポート/エクスポートはマルチスレッド並列処理を使用', 'workspace.importExport.ultraModeBenefit2': '✅ スレッド数とバッチサイズを適応的に調整し最適な構成に自動合わせ', 'workspace.importExport.ultraModeBenefit3': - '✅ ワーカー開始前にファイル全体を判定し、小さいファイルは標準モード、重大なリスクは明示的に拒否', + '✅ ワーカー開始前にファイル全体を判定し、重大なリスクは明示的に拒否', 'workspace.importExport.ultraModeRisk1': '⚠️ 高同時実行時にソース DB の接続圧力が増大（MySQL プール ≥ 16 推奨）', 'workspace.importExport.ultraModeRisk2': '⚠️ メモリ使用量がやや増加（シャードごとの ResultSet とバッファ）', 'workspace.importExport.ultraModeRisk3': '⚠️ 事前準備されていない厳格な外部キーのデータには不適（デッドロックの可能性）', diff --git a/chat2db-community-client/src/i18n/ko-KR/workspace.ts b/chat2db-community-client/src/i18n/ko-KR/workspace.ts index a8fc32bbba..26e39f804b 100644 --- a/chat2db-community-client/src/i18n/ko-KR/workspace.ts +++ b/chat2db-community-client/src/i18n/ko-KR/workspace.ts @@ -431,7 +431,7 @@ export default { '✅ 지원되는 테이블 데이터 가져오기/내보내기에 멀티스레드 병렬 처리 사용', 'workspace.importExport.ultraModeBenefit2': '✅ 스레드 수와 배치 크기를 적응적으로 조정', 'workspace.importExport.ultraModeBenefit3': - '✅ 작업자 시작 전 전체 파일을 판정하고 작은 파일은 표준 모드로, 심각한 위험은 명시적으로 거부', + '✅ 작업자 시작 전 전체 파일을 판정하고 심각한 위험은 명시적으로 거부', 'workspace.importExport.ultraModeRisk1': '⚠️ 높은 동시성으로 원본 데이터베이스 연결 부담 증가 (MySQL 풀 ≥ 16 권장)', 'workspace.importExport.ultraModeRisk2': '⚠️ 메모리 사용량 다소 증가 (샤드별 결과 집합 및 버퍼)', 'workspace.importExport.ultraModeRisk3': '⚠️ 사전 준비되지 않은 엄격한 외래 키 데이터에는 부적합 (교착 상태 가능)', @@ -447,7 +447,6 @@ export default { 'workspace.importExport.modeBadgeUltra': '[초고속]', 'workspace.importExport.modeBadgeStandard': '[표준]', 'workspace.importExport.charset': '문자셋', - 'workspace.importExport.delimiter': '구분자', 'workspace.importExport.quoteChar': '따옴표 문자', 'workspace.importExport.skipRows': '건너뛸 행 수', 'workspace.importExport.nullString': 'NULL 문자열', diff --git a/chat2db-community-client/src/i18n/zh-CN/workspace.ts b/chat2db-community-client/src/i18n/zh-CN/workspace.ts index 7257d5a46a..0ad1fe5ca5 100644 --- a/chat2db-community-client/src/i18n/zh-CN/workspace.ts +++ b/chat2db-community-client/src/i18n/zh-CN/workspace.ts @@ -132,7 +132,7 @@ export default { 'workspace.importExport.ultraModeConfirmIntro': '极速模式特点：', 'workspace.importExport.ultraModeBenefit1': '✅ 支持的表数据导入/导出使用多线程并行处理', 'workspace.importExport.ultraModeBenefit2': '✅ 自适应调整线程数与批大小，自动匹配最优配置', - 'workspace.importExport.ultraModeBenefit3': '✅ 启动线程前执行全量准入探测；小文件自动使用常规模式，硬风险明确拒绝', + 'workspace.importExport.ultraModeBenefit3': '✅ 启动线程前执行全量准入探测；硬风险明确拒绝', 'workspace.importExport.ultraModeRisk1': '⚠️ 高并发下源数据库连接压力增大（建议 MySQL 连接池 ≥ 16）', 'workspace.importExport.ultraModeRisk2': '⚠️ 内存占用略增（每个分片独占结果集与缓冲区）', 'workspace.importExport.ultraModeRisk3': '⚠️ 不适用于外键约束严格且未预处理的数据（可能导致死锁）', @@ -146,7 +146,6 @@ export default { 'workspace.importExport.modeBadgeUltra': '[极速模式]', 'workspace.importExport.modeBadgeStandard': '[常规模式]', 'workspace.importExport.charset': '字符集', - 'workspace.importExport.delimiter': '分隔符', 'workspace.importExport.quoteChar': '引号字符', 'workspace.importExport.skipRows': '跳过行数', 'workspace.importExport.nullString': 'NULL 文本', diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/task/ImportAdmissionFinding.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/task/ImportAdmissionFinding.java new file mode 100644 index 0000000000..36735d5321 --- /dev/null +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/task/ImportAdmissionFinding.java @@ -0,0 +1,25 @@ +package ai.chat2db.community.domain.api.model.task; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** One explainable rule result from the parallel-import admission gate. */ +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class ImportAdmissionFinding { + + private String code; + + /** BLOCKER or DEGRADATION. */ + private String severity; + + private String message; + + private String evidence; + + private String remediation; +} diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/task/ImportAdmissionReport.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/task/ImportAdmissionReport.java new file mode 100644 index 0000000000..7160953cef --- /dev/null +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/task/ImportAdmissionReport.java @@ -0,0 +1,37 @@ +package ai.chat2db.community.domain.api.model.task; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.List; + +/** Result of the mandatory, read-only gate that runs before an import starts workers. */ +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class ImportAdmissionReport { + + /** PARALLEL_SAFE, PARALLEL_DEGRADED, or PARALLEL_FORBIDDEN. */ + private String verdict; + + private String requestedMode; + + private String effectiveMode; + + private boolean parallelAllowed; + + private String fileFormat; + + private long fileSizeBytes; + + private long dataRows; + + private boolean fullScan; + + private boolean relationshipRiskAccepted; + + private List findings; +} diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/task/ImportPreview.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/task/ImportPreview.java index 63482f56cc..c285ede76b 100644 --- a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/task/ImportPreview.java +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/task/ImportPreview.java @@ -29,6 +29,9 @@ public class ImportPreview { private String detectedDelimiter; + /** Explainable result of the same admission gate enforced before task execution. */ + private ImportAdmissionReport parallelAdmission; + /** * Table columns the source file does not supply; they will be imported as {@code NULL} or * their defaults. diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/task/ImportTaskSpec.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/task/ImportTaskSpec.java index 5a8b001668..6b8b7a9138 100644 --- a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/task/ImportTaskSpec.java +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/task/ImportTaskSpec.java @@ -45,4 +45,11 @@ public class ImportTaskSpec implements TaskSpec { * Execution mode; {@code null} resolves to {@code STANDARD}. */ private String mode; + + /** + * Explicit operator assertion required for parallel row imports. It covers relationship and + * ordering semantics that metadata alone cannot prove (logical foreign keys, triggers and + * application-level parent/child ID dependencies); it does not bypass file-format blockers. + */ + private Boolean confirmedNoStrongRelations; } diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/task/TaskErrorCode.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/task/TaskErrorCode.java index 1618e76f8a..f82f153add 100644 --- a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/task/TaskErrorCode.java +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/task/TaskErrorCode.java @@ -12,5 +12,6 @@ public enum TaskErrorCode { FILE_READ_FAILED, FILE_WRITE_FAILED, EXPORT_FAILED, + IMPORT_PARALLEL_FORBIDDEN, IMPORT_FAILED } diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/task/TaskExecutionMode.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/task/TaskExecutionMode.java new file mode 100644 index 0000000000..c5c1272113 --- /dev/null +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/task/TaskExecutionMode.java @@ -0,0 +1,24 @@ +package ai.chat2db.community.domain.api.model.task; + +import org.apache.commons.lang3.StringUtils; + +/** + * Execution mode of a bulk import/export task. {@code ULTRA_FAST} enables the parallel machinery + * (keyset sharding, multi-worker batches, multi-row INSERT merging, adaptive tuning); + * {@code STANDARD} is the conservative single-threaded path with fixed small batches. Absent or + * unknown values resolve to {@code STANDARD} so older clients keep a well-defined behaviour. + */ +public final class TaskExecutionMode { + + public static final String ULTRA_FAST = "ULTRA_FAST"; + + public static final String STANDARD = "STANDARD"; + + private TaskExecutionMode() { + } + + /** True only for an explicit {@code ULTRA_FAST}; anything else (null, blank, unknown) is standard. */ + public static boolean isUltraFast(String mode) { + return ULTRA_FAST.equalsIgnoreCase(StringUtils.trimToEmpty(mode)); + } +} diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/AdaptiveBatchSizer.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/AdaptiveBatchSizer.java new file mode 100644 index 0000000000..e8495a3295 --- /dev/null +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/AdaptiveBatchSizer.java @@ -0,0 +1,60 @@ +package ai.chat2db.community.domain.core.impl.task; + +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Self-tuning row-batch size for bulk I/O. Producers report the wall time of each executed batch; + * the sizer doubles the batch when execution comes back cheap and halves it when execution is + * expensive, which keeps per-batch latency inside a band where the per-statement overhead is + * amortized without building oversized memory structures or holding long-running batches. The + * contract keeps the size inside [100, 100_000] rows so it stays sane under noisy measurements. + */ +public final class AdaptiveBatchSizer { + + private static final int MIN_BATCH = 100; + + private static final int MAX_BATCH = 100_000; + + private static final long FAST_NANOS = 4L * 1_000_000L; + + private static final long SLOW_NANOS = 40L * 1_000_000L; + + private final AtomicInteger batchSize; + + /** When {@code false} the sizer stays fixed at its initial size (standard mode). */ + private final boolean adaptive; + + public AdaptiveBatchSizer(int initialBatch) { + this(initialBatch, true); + } + + public AdaptiveBatchSizer(int initialBatch, boolean adaptive) { + this.batchSize = new AtomicInteger(clamp(initialBatch)); + this.adaptive = adaptive; + } + + public int batchSize() { + return batchSize.get(); + } + + /** + * Reports one executed batch of {@code rows} rows that took {@code nanos} wall time; later + * {@link #batchSize()} calls reflect the tuned size. + */ + public void record(int rows, long nanos) { + if (!adaptive || rows <= 0 || nanos <= 0) { + return; + } + int current = batchSize.get(); + int next = nanos < FAST_NANOS ? current * 2 + : nanos > SLOW_NANOS ? current / 2 + : current; + if (next != current) { + batchSize.compareAndSet(current, clamp(next)); + } + } + + private static int clamp(int value) { + return Math.max(MIN_BATCH, Math.min(MAX_BATCH, value)); + } +} diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/AdaptiveConcurrencyGate.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/AdaptiveConcurrencyGate.java new file mode 100644 index 0000000000..18b3567c1c --- /dev/null +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/AdaptiveConcurrencyGate.java @@ -0,0 +1,183 @@ +package ai.chat2db.community.domain.core.impl.task; + +import java.util.concurrent.Semaphore; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +import lombok.extern.slf4j.Slf4j; + +/** + * Dynamic concurrency gate for parallel task I/O. Workers acquire a permit around each unit of + * work; the permit count starts low and is tuned by a throughput observer in an AIMD style: every + * time {@link #WINDOW_ROWS} rows of data have flowed through since the last evaluation, the gate + * compares the throughput of the finished window with the previous one and grows by one permit on + * improvement, or gives back a quarter of the permits on regression. The fan-out therefore + * converges to the level the target system actually tolerates instead of a fixed guess, and it + * backs off on its own when the source or the target becomes the bottleneck. + * + *

Tuning never throws into the task: every adjustment runs under its own guard, so an observer + * failure degrades to keeping the current fan-out instead of failing the export or import. The + * total permit count is tracked explicitly and hard-capped at {@code maxPermits}, even while + * workers hold permits, so the fan-out can never exceed its configured ceiling, and tuning never + * shrinks it past {@link #MIN_PERMITS}. A stuck gate must not hang a task either: workers wait + * through {@link #admit(long)} with a timeout and proceed ungated on expiry, which degrades to the + * pre-adaptive unbounded concurrency instead of stalling. + */ +@Slf4j +public final class AdaptiveConcurrencyGate extends Semaphore { + + /** + * Tuning window: the observer evaluates once this much data (rows) has flowed through since + * the previous evaluation, independent of how many batches that took. + */ + static final long WINDOW_ROWS = 80_000L; + + /** + * Hard floor of the fan-out; a gate created below it can still grow, but tuning never shrinks + * it past this bound (bounded by the configured max when that is smaller). + */ + static final int MIN_PERMITS = 2; + + /** Minimum spacing between source-pressure cuts so one slow page cannot crash the fan-out. */ + private static final long PRESSURE_CUT_SPACING_NANOS = 1_000_000_000L; + + private final int maxPermits; + + private final int floor; + + /** Total permits in circulation; only the tuning paths change it, and never past maxPermits. */ + private final AtomicInteger totalPermits; + + private final AtomicLong windowRows = new AtomicLong(); + + private final AtomicLong windowNanos = new AtomicLong(); + + private volatile long lastPressureCutNanos; + + private double lastThroughput = -1.0D; + + private AdaptiveConcurrencyGate(int initialPermits, int maxPermits) { + super(Math.max(1, Math.min(initialPermits, maxPermits))); + this.maxPermits = Math.max(1, maxPermits); + this.floor = Math.min(MIN_PERMITS, this.maxPermits); + this.totalPermits = new AtomicInteger(Math.max(1, Math.min(initialPermits, maxPermits))); + } + + public static AdaptiveConcurrencyGate create(int initialPermits, int maxPermits) { + return new AdaptiveConcurrencyGate(initialPermits, maxPermits); + } + + /** + * Records one completed work unit ({@code rows} rows over {@code nanos} wall time); once the + * observation window fills, the fan-out is retuned. Never throws into the caller. + */ + public void record(long rows, long nanos) { + if (rows <= 0 || nanos <= 0) { + return; + } + windowRows.addAndGet(rows); + windowNanos.addAndGet(nanos); + if (windowRows.get() < WINDOW_ROWS) { + return; + } + synchronized (this) { + if (windowRows.get() < WINDOW_ROWS) { + // A concurrent caller already consumed this window. + return; + } + tuneThroughput(windowRows.getAndSet(0L), windowNanos.getAndSet(0L)); + } + } + + /** + * Source-pressure response for readers: a page query took noticeably longer than healthy, so + * give back a quarter of the fan-out immediately instead of waiting for the throughput window + * to notice, letting the source database recover. Cooldown-limited and failure-tolerant; the + * regular AIMD window tuning remains the recovery path once the source speeds up again. + */ + public void reduceForSourcePressure() { + long now = System.nanoTime(); + synchronized (this) { + if (now - lastPressureCutNanos < PRESSURE_CUT_SPACING_NANOS + || totalPermits.get() <= floor) { + return; + } + lastPressureCutNanos = now; + try { + int cut = Math.max(1, totalPermits.get() / 4); + int target = Math.max(floor, totalPermits.get() - cut); + while (totalPermits.get() > target) { + reducePermits(1); + totalPermits.decrementAndGet(); + } + } catch (Throwable tuningFailure) { + log.warn("Source-pressure permit reduction failed; keeping the current fan-out", + tuningFailure); + } + } + } + + /** + * Bounded permit wait for task workers: waits up to {@code timeoutMillis} and then reports + * failure instead of blocking forever, so a stuck gate degrades to ungated execution (the + * pre-adaptive behaviour) rather than hanging the task. + * + * @return whether a permit was taken and must later be returned via {@link #relinquish} + */ + public boolean admit(long timeoutMillis) { + try { + return tryAcquire(timeoutMillis, TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return false; + } + } + + /** Returns a permit taken by {@link #admit}; never throws into the worker. */ + public void relinquish(boolean permitted) { + if (!permitted) { + return; + } + try { + release(); + } catch (Throwable releaseFailure) { + // The lost permit is capacity, not data: the AIMD tuning re-grows it. + log.warn("Returning a gate permit failed; the AIMD tuning will restore the capacity", + releaseFailure); + } + } + + private void tuneThroughput(long rows, long nanos) { + try { + double throughput = rows * 1_000_000.0D / Math.max(1L, nanos); + if (lastThroughput > 0.0D) { + if (throughput > lastThroughput) { + // Additive increase, capped by the hard total so growth cannot overshoot the + // configured ceiling even while workers hold permits. + if (totalPermits.get() < maxPermits) { + release(); + totalPermits.incrementAndGet(); + } + } else if (throughput < lastThroughput && totalPermits.get() > floor) { + // Multiplicative decrease: a regression cuts fast, growth is careful so a + // lucky window cannot oversubscribe the target system. + int cut = Math.max(1, totalPermits.get() / 4); + int target = Math.max(floor, totalPermits.get() - cut); + while (totalPermits.get() > target) { + reducePermits(1); + totalPermits.decrementAndGet(); + } + } + } + lastThroughput = throughput; + } catch (Throwable tuningFailure) { + // Tuning must never break the task: keep the current fan-out and the next window. + log.warn("Adaptive gate tuning failed; keeping the current fan-out", tuningFailure); + } + } + + int currentPermits() { + return availablePermits(); + } +} diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/TaskResumeJournal.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/TaskResumeJournal.java new file mode 100644 index 0000000000..adde5da9d6 --- /dev/null +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/TaskResumeJournal.java @@ -0,0 +1,334 @@ +package ai.chat2db.community.domain.core.impl.task; + +import ai.chat2db.community.tools.util.ConfigUtils; +import com.alibaba.fastjson2.JSONObject; +import lombok.extern.slf4j.Slf4j; + +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.io.Writer; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.security.MessageDigest; +import java.util.Comparator; +import java.util.HexFormat; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +/** + * Three-layer resume persistence for bulk import/export tasks. + * + *

Layer 1 appends phase/progress records to a per-task NDJSON journal, so the freshest + * stage survives a hard kill even when the snapshot interval has not elapsed. Layer 2, the + * task-storage {@code ResumeState} checkpoints, is written by the callers through + * {@code TaskExecutionContext#checkpoint} — this class only contributes candidates to the recovery + * resolution. Layer 3 writes generational JSON snapshots and maintains a {@code committed} + * pointer that always names the generation before the last successfully committed one: + * the guaranteed-consistent fallback of the resume chain, mirroring "restore from before the last + * successful commit". + * + *

Every record carries a SHA-256 checksum of its payload; torn records (truncated writes, power + * loss mid-rename) fail validation and recovery falls back to the next candidate. Snapshots are + * written atomically (temp file + rename + force), so a reader never observes a half-written file. + * Recovery resolves the newest valid candidate: because every candidate stores a durable + * watermark (all rows below it are committed), any valid candidate is equally safe and the newest + * simply minimises reprocessing. + */ +@Slf4j +public final class TaskResumeJournal { + + /** Generations kept on disk: the newest and the committed fallback behind it. */ + private static final int KEEP_GENERATIONS = 2; + + private static final String PROGRESS_FILE = "progress.ndjson"; + + private static final String POINTER_FILE = "committed.txt"; + + private final File directory; + + private final Map identity; + + private final Writer progressWriter; + + private TaskResumeJournal(File directory, Map identity, Writer progressWriter) { + this.directory = directory; + this.identity = identity; + this.progressWriter = progressWriter; + } + + /** Task-scoped journal directory under the application state path. */ + public static File directoryFor(Long taskId) { + return new File(ConfigUtils.getBasePath(), + "task-journal" + File.separatorChar + "task-" + (taskId == null ? 0L : taskId)); + } + + /** + * Opens the journal for writing, or {@code null} when the state path is unusable — journaling + * is an enhancement and must never block the task itself. + * + * @param identity source identity (length/lastModified) stamped onto every record so recovery + * can reject checkpoints of a changed input + */ + public static TaskResumeJournal open(Long taskId, Map identity) { + return openDirectory(directoryFor(taskId), identity); + } + + /** Directory-injected variant for tests and callers that own their persistence path. */ + static TaskResumeJournal openDirectory(File directory, Map identity) { + try { + Files.createDirectories(directory.toPath()); + Writer writer = Files.newBufferedWriter(directory.toPath().resolve(PROGRESS_FILE), + StandardCharsets.UTF_8, StandardOpenOption.CREATE, StandardOpenOption.APPEND); + return new TaskResumeJournal(directory, identity, writer); + } catch (Throwable openFailure) { + log.warn("Resume journal unavailable; continuing without stage persistence", openFailure); + return null; + } + } + + /** Appends one progress record (Layer 1); flushes but never throws into the task. */ + public synchronized void progress(String phase, long rowsDone) { + if (progressWriter == null) { + return; + } + try { + JSONObject payload = basePayload("progress", rowsDone); + payload.put("phase", phase); + progressWriter.write(signed(payload)); + progressWriter.write("\n"); + progressWriter.flush(); + } catch (Throwable journalFailure) { + log.warn("Resume journal write failed; continuing", journalFailure); + } + } + + /** + * Commits the next full-state generation (Layer 3) and then repoints the committed fallback at + * the previous generation, so the recovery floor always stays one commit behind; older + * generations are pruned. Never throws into the task. + */ + public synchronized void snapshot(long rowsDone) { + try { + long seq = nextGeneration(); + JSONObject payload = basePayload("snapshot", rowsDone); + payload.put("seq", seq); + writeGeneration(seq, payload); + if (seq > 1) { + writePointer(seq - 1); + pruneGenerations(seq); + } + } catch (Throwable snapshotFailure) { + log.warn("Resume snapshot failed; the previous fallback stays in place", snapshotFailure); + } + } + + /** Closes the journal and, on a clean run, removes the whole directory. */ + public synchronized void cleanup() { + preserve(); + deleteQuietly(directory); + } + + /** Closes the writer while preserving every recovery candidate after a failed run. */ + public synchronized void preserve() { + try { + if (progressWriter != null) { + progressWriter.close(); + } + } catch (Throwable closeFailure) { + log.warn("Resume journal close failed", closeFailure); + } + } + + /** Newest intact generation snapshot, or empty when none validates. */ + public static Optional recoverNewest(File directory) { + return generations(directory).stream() + .map(TaskResumeJournal::readGeneration) + .flatMap(Optional::stream) + .max(Comparator.comparingLong(Snapshot::seq)); + } + + /** The committed fallback: the generation the pointer names (one behind the last commit). */ + public static Optional recoverCommitted(File directory) { + try { + File pointer = directory.toPath().resolve(POINTER_FILE).toFile(); + if (!pointer.isFile()) { + return Optional.empty(); + } + long seq = Long.parseLong(Files.readString(pointer.toPath(), StandardCharsets.UTF_8).trim()); + return readGeneration(directory.toPath().resolve("gen-" + seq + ".json").toFile()); + } catch (Throwable pointerFailure) { + log.debug("Committed fallback unreadable; skipping this candidate", pointerFailure); + return Optional.empty(); + } + } + + /** Watermark of the last valid progress record in the NDJSON tail. */ + public static Optional recoverTail(File directory) { + try { + File progress = directory.toPath().resolve(PROGRESS_FILE).toFile(); + if (!progress.isFile()) { + return Optional.empty(); + } + Optional newestValid = Optional.empty(); + try (BufferedReader reader = Files.newBufferedReader(progress.toPath(), StandardCharsets.UTF_8)) { + String line; + while ((line = reader.readLine()) != null) { + if (!line.isBlank()) { + Optional record = parse(line.trim()); + if (record.isPresent()) { + newestValid = record; + } + } + } + } + return newestValid; + } catch (Throwable tailFailure) { + log.debug("Journal tail unreadable; skipping this candidate", tailFailure); + return Optional.empty(); + } + } + + // --- internal persistence plumbing ---------------------------------------------------- + + private JSONObject basePayload(String kind, long rowsDone) { + JSONObject payload = new JSONObject(); + payload.put("kind", kind); + payload.put("rowsDone", rowsDone); + payload.put("ts", System.currentTimeMillis()); + if (identity != null) { + payload.put("identity", identity); + } + return payload; + } + + /** Signs the payload with a SHA-256 checksum of its own JSON and serialises the result. */ + private static String signed(JSONObject payload) { + String body = payload.toJSONString(); + JSONObject signed = JSONObject.parseObject(body); + signed.put("checksum", sha256(body)); + return signed.toJSONString(); + } + + /** Parses and checksum-validates one record; empty on any mismatch or parse problem. */ + private static Optional parse(String line) { + try { + JSONObject record = JSONObject.parseObject(line); + String checksum = record.getString("checksum"); + if (checksum == null) { + return Optional.empty(); + } + record.remove("checksum"); + if (!checksum.equals(sha256(record.toJSONString()))) { + return Optional.empty(); + } + if (!"progress".equals(record.getString("kind")) + && !"snapshot".equals(record.getString("kind"))) { + return Optional.empty(); + } + Long rowsDone = record.getLong("rowsDone"); + if (rowsDone == null || rowsDone < 0) { + return Optional.empty(); + } + return Optional.of(new Snapshot(rowsDone, record.getLongValue("seq"), + record.getLongValue("ts"), record.getJSONObject("identity"))); + } catch (Throwable parseFailure) { + log.debug("Resume record failed validation", parseFailure); + return Optional.empty(); + } + } + + private long nextGeneration() { + return generations(directory).stream().mapToLong(this::generationSeq).max().orElse(0L) + 1; + } + + private long generationSeq(File file) { + String name = file.getName(); + try { + return Long.parseLong(name.substring("gen-".length(), name.length() - ".json".length())); + } catch (NumberFormatException e) { + return 0L; + } + } + + private static List generations(File directory) { + File[] files = directory.listFiles((dir, name) -> name.startsWith("gen-") && name.endsWith(".json")); + return files == null ? List.of() : List.of(files); + } + + private void writeGeneration(long seq, JSONObject payload) throws IOException { + Path target = directory.toPath().resolve("gen-" + seq + ".json"); + writeAtomically(target, signed(payload)); + } + + private void writePointer(long seq) throws IOException { + writeAtomically(directory.toPath().resolve(POINTER_FILE), String.valueOf(seq)); + } + + /** Temp file + fsync + atomic rename: a reader either sees the old file or the complete new one. */ + private static void writeAtomically(Path target, String content) throws IOException { + Path temp = target.resolveSibling(target.getFileName() + ".tmp"); + try (var channel = java.nio.channels.FileChannel.open(temp, StandardOpenOption.CREATE, + StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING)) { + channel.write(java.nio.ByteBuffer.wrap(content.getBytes(StandardCharsets.UTF_8))); + channel.force(true); + } + Files.move(temp, target, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE); + } + + private void pruneGenerations(long newestSeq) { + long floor = newestSeq - (KEEP_GENERATIONS - 1); + for (File file : generations(directory)) { + long seq = generationSeq(file); + if (seq > 0 && seq < floor && !file.delete()) { + log.debug("Could not prune stale resume generation {}", file.getName()); + } + } + } + + private static Optional readGeneration(File file) { + try { + return parse(Files.readString(file.toPath(), StandardCharsets.UTF_8)); + } catch (Throwable readFailure) { + log.debug("Resume generation {} failed validation", file.getName(), readFailure); + return Optional.empty(); + } + } + + private static String sha256(String content) { + try { + MessageDigest digest = MessageDigest.getInstance("SHA-256"); + return HexFormat.of().formatHex(digest.digest(content.getBytes(StandardCharsets.UTF_8))); + } catch (Exception digestFailure) { + throw new IllegalStateException("SHA-256 unavailable", digestFailure); + } + } + + private static void deleteQuietly(File directory) { + File[] files = directory.listFiles(); + if (files != null) { + for (File file : files) { + if (file.isDirectory()) { + deleteQuietly(file); + } else if (!file.delete()) { + log.debug("Could not delete resume journal file {}", file.getName()); + } + } + } + if (!directory.delete()) { + log.debug("Could not delete resume journal directory {}", directory); + } + } + + /** + * One validated resume candidate. {@code rowsDone} means every source row with a position at + * or below it is durably applied; {@code identity} carries the source fingerprint for staleness + * checks; {@code seq} is 0 for journal-tail records. + */ + public record Snapshot(long rowsDone, long seq, long timestamp, Map identity) { + } +} diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/TaskServiceImpl.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/TaskServiceImpl.java index 9ee503a2e9..15135f4c23 100644 --- a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/TaskServiceImpl.java +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/TaskServiceImpl.java @@ -27,6 +27,7 @@ import ai.chat2db.community.domain.api.service.task.TaskStorage; import ai.chat2db.community.domain.core.impl.task.imports.ImportColumnResolver; import ai.chat2db.community.domain.core.impl.task.imports.ImportFileProbe; +import ai.chat2db.community.domain.core.impl.task.imports.ImportParallelAdmission; import ai.chat2db.community.domain.core.impl.task.imports.excel.ImportPreviewListener; import com.alibaba.excel.EasyExcel; import com.alibaba.excel.support.ExcelTypeEnum; @@ -181,7 +182,7 @@ private ImportPreview buildPreview(List> rows, List ta ImportTaskSpec spec, String detectedCharset, String detectedDelimiter) { List headers = rows.isEmpty() ? List.of() : rows.get(0); ImportColumnResolver.Resolution resolution = - ImportColumnResolver.resolve(tableColumns, headers, spec.getOptions()); + ImportColumnResolver.resolveForSpec(tableColumns, headers, spec); return ImportPreview.builder() .targetColumns(tableColumns.stream().map(column -> ai.chat2db.community.domain.api.model.db.ImportTargetColumn.builder() @@ -196,6 +197,7 @@ private ImportPreview buildPreview(List> rows, List ta : rows.subList(1, rows.size()).stream().map(row -> (List) row).toList()) .detectedCharset(detectedCharset) .detectedDelimiter(detectedDelimiter) + .parallelAdmission(ImportParallelAdmission.assess(spec, tableColumns)) .build(); } diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/executor/DataFileImportTaskExecutor.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/executor/DataFileImportTaskExecutor.java index 520fd7b5a5..e3a206e37e 100644 --- a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/executor/DataFileImportTaskExecutor.java +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/executor/DataFileImportTaskExecutor.java @@ -34,6 +34,7 @@ public Class specType() { @Override public void execute(ImportTaskSpec spec, TaskExecutionContext context) { + boolean completed = false; try { TaskExecutorSupport.requireReadableSource(spec.getSourceFile()); String format = TaskExecutorSupport.requireFormat(spec.getFormat()); @@ -47,13 +48,15 @@ public void execute(ImportTaskSpec spec, TaskExecutionContext context) { strategy.run(spec, context); context.reportProgress(95, TaskStage.IMPORTING.name(), "Data import completed"); context.logInfo(TaskEventCode.IMPORT_COMPLETED.name(), "Data import completed"); + completed = true; } catch (TaskCancelledException | TaskExecutionException e) { throw e; } catch (Exception e) { throw new TaskExecutionException(TaskErrorCode.IMPORT_FAILED.name(), "Could not import data file", e); } finally { - if (spec.getImportFileId() != null) { + // Interrupted imports still need the exact staged source to resume from checkpoints. + if (completed && spec.getImportFileId() != null) { importFileStagingService.release(spec.getImportFileId()); } } diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/executor/SqlFileImportTaskExecutor.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/executor/SqlFileImportTaskExecutor.java index 940f47f614..643ee5511f 100644 --- a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/executor/SqlFileImportTaskExecutor.java +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/executor/SqlFileImportTaskExecutor.java @@ -11,6 +11,7 @@ import ai.chat2db.community.domain.api.service.task.TaskExecutionContext; import ai.chat2db.community.domain.api.service.task.TaskExecutor; import ai.chat2db.community.domain.core.impl.task.imports.ImportFactory; +import ai.chat2db.community.domain.core.impl.task.imports.ImportParallelAdmission; import org.springframework.stereotype.Component; @Component @@ -34,6 +35,7 @@ public Class specType() { @Override public void execute(ImportTaskSpec spec, TaskExecutionContext context) { + boolean completed = false; try { TaskExecutorSupport.requireReadableSource(spec.getSourceFile()); String format = TaskExecutorSupport.requireFormat(spec.getFormat()); @@ -42,15 +44,18 @@ public void execute(ImportTaskSpec spec, TaskExecutionContext context) { "SQL import requires an SQL file"); } context.reportProgress(5, TaskStage.READING.name(), "Preparing SQL import"); + ImportParallelAdmission.enforce(spec, java.util.List.of(), context); ImportFactory.get(format).run(spec, context); context.reportProgress(95, TaskStage.IMPORTING.name(), "SQL import completed"); + completed = true; } catch (TaskCancelledException | TaskExecutionException e) { throw e; } catch (Exception e) { throw new TaskExecutionException(TaskErrorCode.IMPORT_FAILED.name(), "Could not import SQL file", e); } finally { - if (spec.getImportFileId() != null) { + // The exact staged source is required by a later resume attempt after interruption. + if (completed && spec.getImportFileId() != null) { importFileStagingService.release(spec.getImportFileId()); } } diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/BaseImporter.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/BaseImporter.java index 05fb295edd..6803ddbab6 100644 --- a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/BaseImporter.java +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/BaseImporter.java @@ -36,6 +36,8 @@ public void run(ImportTaskSpec spec, TaskExecutionContext context) { new TableMetadataRequest(connectInfo.getDatabaseName(), connectInfo.getSchemaName(), spec.getTarget().getTableName())); context.checkCancelled(); + ImportParallelAdmission.enforce(spec, tableColumns, context); + context.checkCancelled(); context.reportProgress(20, TaskStage.READING.name(), "Target table metadata loaded"); context.logInfo(TaskEventCode.TARGET_METADATA_LOADED.name(), "Target table metadata loaded"); context.logInfo(TaskEventCode.FILE_READ_STARTED.name(), "Reading import file"); @@ -54,7 +56,26 @@ public void run(ImportTaskSpec spec, TaskExecutionContext context) { protected abstract void doImportData(ImportTaskSpec spec, TaskExecutionContext context, - List tableColumns); + List tableColumns) throws Exception; + + /** + * One warning event describing how file columns resolved against table columns; unmatched file + * columns are skipped and missing table columns follow the requested DEFAULT or NULL policy. + */ + protected static void reportResolution(TaskExecutionContext context, + ImportColumnResolver.Resolution resolution) { + List extraFileColumns = resolution.matches().stream() + .filter(match -> !match.isMatched()) + .map(match -> match.getFileColumn()) + .filter(name -> name != null) + .toList(); + if (!extraFileColumns.isEmpty() || !resolution.missingTableColumns().isEmpty()) { + context.logWarn("IMPORT_COLUMN_MAPPING", "Import column mapping applied with warnings", + java.util.Map.of( + "unmatchedFileColumns", extraFileColumns, + "missingTableColumns", resolution.missingTableColumns())); + } + } protected SQLDataValue getSQLDataValue(String value, TableColumn column) { diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/ImportColumnResolver.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/ImportColumnResolver.java index 090e6a9337..db1ba2d225 100644 --- a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/ImportColumnResolver.java +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/ImportColumnResolver.java @@ -4,6 +4,8 @@ import ai.chat2db.community.domain.api.model.task.ImportColumnMapping; import ai.chat2db.community.domain.api.model.task.ImportColumnMatch; import ai.chat2db.community.domain.api.model.task.ImportOptions; +import ai.chat2db.community.domain.api.model.task.ImportTaskSpec; +import ai.chat2db.community.domain.api.model.task.UnmappedTargetStrategy; import ai.chat2db.community.tools.exception.ParamBusinessException; import org.apache.commons.lang3.StringUtils; @@ -32,13 +34,47 @@ private ImportColumnResolver() { public static Resolution resolve(List tableColumns, List fileHeaders, ImportOptions options) { + return resolve(tableColumns, fileHeaders, options == null ? null : options.getColumnMappings(), + UnmappedTargetStrategy.DEFAULT); + } + + public static Resolution resolveForSpec(List tableColumns, List fileHeaders, + ImportTaskSpec spec) { + List mappings = spec.getColumnMappings(); + if (mappings == null && spec.getOptions() != null) { + mappings = spec.getOptions().getColumnMappings(); + } + return resolve(tableColumns, fileHeaders, mappings, spec.getUnmappedTarget()); + } + + public static void validateForImport(List columns, Resolution resolution, ImportTaskSpec spec) { + if (resolution.fileIndexes().stream().noneMatch(java.util.Objects::nonNull)) { + throw new ParamBusinessException("At least one import column mapping is required"); + } + for (TableColumn column : columns) { + if (resolution.missingTableColumns().contains(column.getName()) + && Integer.valueOf(0).equals(column.getNullable()) + && !Boolean.TRUE.equals(column.getAutoIncrement()) + && (spec.getUnmappedTarget() == UnmappedTargetStrategy.NULL || column.getDefaultValue() == null)) { + throw new ParamBusinessException("Required import column is unmapped: " + column.getName()); + } + } + } + + private static Resolution resolve(List tableColumns, List fileHeaders, + List mappings, UnmappedTargetStrategy unmappedTarget) { Map byNormalizedName = new LinkedHashMap<>(); for (int index = 0; index < fileHeaders.size(); index++) { - byNormalizedName.putIfAbsent(normalize(fileHeaders.get(index)), index); + if (byNormalizedName.putIfAbsent(normalize(fileHeaders.get(index)), index) != null) { + throw new ParamBusinessException("Duplicate import source column: " + fileHeaders.get(index)); + } } Map explicitTargets = new LinkedHashMap<>(); - if (options != null && options.getColumnMappings() != null) { - for (ImportColumnMapping mapping : options.getColumnMappings()) { + java.util.Set explicitSources = new java.util.HashSet<>(); + java.util.Set knownTargets = tableColumns.stream().map(column -> normalize(column.getName())) + .collect(java.util.stream.Collectors.toSet()); + if (mappings != null) { + for (ImportColumnMapping mapping : mappings) { if (mapping == null || StringUtils.isBlank(mapping.getSourceColumn()) || StringUtils.isBlank(mapping.getTargetColumn())) { throw new ParamBusinessException("columnMappings"); @@ -48,7 +84,11 @@ public static Resolution resolve(List tableColumns, List fi if (sourceIndex == null) { throw new ParamBusinessException("columnMappings source: " + mapping.getSourceColumn()); } - explicitTargets.put(normalize(mapping.getTargetColumn()), sourceIndex); + String target = normalize(mapping.getTargetColumn()); + if (!knownTargets.contains(target) || !explicitSources.add(sourceIndex) + || explicitTargets.putIfAbsent(target, sourceIndex) != null) { + throw new ParamBusinessException("Duplicate or invalid import column mapping"); + } } } @@ -57,7 +97,7 @@ public static Resolution resolve(List tableColumns, List fi List missingTableColumns = new ArrayList<>(); for (TableColumn column : tableColumns) { Integer sourceIndex = explicitTargets.get(normalize(column.getName())); - if (sourceIndex == null) { + if (sourceIndex == null && mappings == null) { sourceIndex = byNormalizedName.get(normalize(column.getName())); } if (sourceIndex != null) { @@ -65,6 +105,11 @@ public static Resolution resolve(List tableColumns, List fi fileIndexes.add(sourceIndex); } else { missingTableColumns.add(column.getName()); + if (mappings != null && unmappedTarget == UnmappedTargetStrategy.NULL + && !Boolean.TRUE.equals(column.getAutoIncrement())) { + resolvedColumns.add(column); + fileIndexes.add(null); + } } } @@ -73,7 +118,7 @@ public static Resolution resolve(List tableColumns, List fi for (int index = 0; index < fileHeaders.size(); index++) { String tableColumn = null; for (int resolved = 0; resolved < fileIndexes.size(); resolved++) { - if (fileIndexes.get(resolved) == index) { + if (java.util.Objects.equals(fileIndexes.get(resolved), index)) { tableColumn = resolvedColumns.get(resolved).getName(); break; } @@ -89,6 +134,10 @@ public static Resolution resolve(List tableColumns, List fi private static Integer indexOfSource(String source, List fileHeaders, Map byNormalizedName) { + Integer namedIndex = byNormalizedName.get(normalize(source)); + if (namedIndex != null) { + return namedIndex; + } try { int index = Integer.parseInt(source); return index >= 0 && index < fileHeaders.size() ? index : null; diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/ImportParallelAdmission.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/ImportParallelAdmission.java new file mode 100644 index 0000000000..288ce5dd34 --- /dev/null +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/ImportParallelAdmission.java @@ -0,0 +1,270 @@ +package ai.chat2db.community.domain.core.impl.task.imports; + +import ai.chat2db.community.domain.api.model.metadata.TableColumn; +import ai.chat2db.community.domain.api.model.task.ImportAdmissionFinding; +import ai.chat2db.community.domain.api.model.task.ImportAdmissionReport; +import ai.chat2db.community.domain.api.model.task.ImportTaskSpec; +import ai.chat2db.community.domain.api.model.task.TaskErrorCode; +import ai.chat2db.community.domain.api.model.task.TaskExecutionException; +import ai.chat2db.community.domain.api.model.task.TaskExecutionMode; +import ai.chat2db.community.domain.api.service.task.TaskExecutionContext; +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVParser; +import org.apache.commons.csv.CSVRecord; +import org.apache.commons.lang3.StringUtils; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.Charset; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; + +/** + * Mandatory read-only admission gate for imports. It deliberately separates facts from mode + * selection: the report records every detected blocker/degradation, then {@link #enforce} + * applies the strict policy before an importer can construct worker threads. + */ +public final class ImportParallelAdmission { + + static final String SAFE = "PARALLEL_SAFE"; + static final String DEGRADED = "PARALLEL_DEGRADED"; + static final String FORBIDDEN = "PARALLEL_FORBIDDEN"; + static final String BLOCKER = "BLOCKER"; + static final String DEGRADATION = "DEGRADATION"; + + private ImportParallelAdmission() { + } + + public static ImportAdmissionReport assess(ImportTaskSpec spec, List tableColumns) { + File source = new File(StringUtils.defaultString(spec.getSourceFile())); + String format = StringUtils.upperCase(StringUtils.trimToEmpty(spec.getFormat()), Locale.ROOT); + boolean requestedParallel = TaskExecutionMode.isUltraFast(spec.getMode()); + boolean relationshipAccepted = Boolean.TRUE.equals(spec.getConfirmedNoStrongRelations()); + List findings = new ArrayList<>(); + long rows = -1L; + boolean fullScan = false; + + if (!source.isFile() || !source.canRead()) { + blocker(findings, "C4", "The import source is not a readable, seekable file", + source.getAbsolutePath(), "Select and stage a local file again."); + } else if (isCompressed(source)) { + blocker(findings, "C2", "Compressed or container input cannot enter parallel execution directly", + "ZIP/GZIP container signature detected", "Decompress to a supported plain-text format first."); + } else if ("CSV".equals(format)) { + CsvFacts facts = scanCsv(source, spec, findings); + rows = facts.dataRows(); + fullScan = facts.fullScan(); + assessCsvTarget(spec, tableColumns, facts.headers(), relationshipAccepted, findings); + } else if ("XLS".equals(format) || "XLSX".equals(format)) { + blocker(findings, "C2", "Excel containers are not safely shardable", + format + " is a compressed binary workbook", "Use STANDARD mode or export a UTF-8 CSV source."); + } else if ("SQL".equals(format)) { + blocker(findings, "P0", "Safe parallel SQL planning is not available", + "SQL execution currently preserves file order on one connection", + "Use STANDARD mode until statement-boundary sharding, dependency barriers and validation are available."); + } else if ("JSON".equals(format)) { + blocker(findings, "P0", "The JSON array importer has no parallel shard planner", + "JSON arrays are consumed as one ordered stream", "Use STANDARD mode or convert to a validated CSV source."); + } else { + blocker(findings, "C2", "The file format is not eligible for parallel import", + StringUtils.defaultIfBlank(format, "UNKNOWN"), "Choose a supported UTF-8 CSV source or STANDARD mode."); + } + + if (requestedParallel && "CSV".equals(format) && !relationshipAccepted) { + blocker(findings, "R1", "Strong relationship and ordering risks have not been acknowledged", + "Logical foreign keys, trigger ordering and application-level parent/child dependencies cannot be proven from column metadata", + "Confirm that the target has no strong relationship or ordering dependency, or use STANDARD mode."); + } + if (requestedParallel && StringUtils.isBlank(spec.getImportFileId())) { + blocker(findings, "G7", "Parallel input has not completed mandatory staging", + "No opaque staged-file identity is attached to the task", "Select the source through the import file picker and retry."); + } + + boolean blocked = findings.stream().anyMatch(item -> BLOCKER.equals(item.getSeverity())); + boolean degraded = findings.stream().anyMatch(item -> DEGRADATION.equals(item.getSeverity())); + String verdict = blocked ? FORBIDDEN : degraded ? DEGRADED : SAFE; + boolean parallelAllowed = requestedParallel && !blocked; + String effectiveMode = parallelAllowed ? TaskExecutionMode.ULTRA_FAST : TaskExecutionMode.STANDARD; + return ImportAdmissionReport.builder() + .verdict(verdict) + .requestedMode(requestedParallel ? TaskExecutionMode.ULTRA_FAST : TaskExecutionMode.STANDARD) + .effectiveMode(effectiveMode) + .parallelAllowed(parallelAllowed) + .fileFormat(format) + .fileSizeBytes(source.isFile() ? source.length() : 0L) + .dataRows(rows) + .fullScan(fullScan) + .relationshipRiskAccepted(relationshipAccepted) + .findings(List.copyOf(findings)) + .build(); + } + + public static ImportAdmissionReport enforce(ImportTaskSpec spec, List tableColumns, + TaskExecutionContext context) { + ImportAdmissionReport report = assess(spec, tableColumns); + Map details = details(report); + if (TaskExecutionMode.isUltraFast(spec.getMode()) && FORBIDDEN.equals(report.getVerdict())) { + context.logError("IMPORT_PARALLEL_ADMISSION", "Parallel import rejected before execution", details); + String codes = report.getFindings().stream() + .filter(item -> BLOCKER.equals(item.getSeverity())) + .map(ImportAdmissionFinding::getCode).distinct().reduce((left, right) -> left + ", " + right) + .orElse("UNKNOWN"); + throw new TaskExecutionException(TaskErrorCode.IMPORT_PARALLEL_FORBIDDEN.name(), + "Parallel import rejected by admission rules: " + codes + ". Review the task event for evidence and remediation."); + } + if (FORBIDDEN.equals(report.getVerdict())) { + context.logWarn("IMPORT_PARALLEL_ADMISSION", + "Serial import selected because parallel admission blockers were detected", details); + } else if (DEGRADED.equals(report.getVerdict())) { + context.logWarn("IMPORT_PARALLEL_ADMISSION", "Parallel import admitted with mandatory warnings", details); + } else { + context.logInfo("IMPORT_PARALLEL_ADMISSION", "Import admission completed", details); + } + if (report.isRelationshipRiskAccepted() && TaskExecutionMode.isUltraFast(report.getRequestedMode())) { + context.logWarn("IMPORT_RELATIONSHIP_RISK_ACCEPTED", + "Operator confirmed that the target has no strong relationship or ordering dependency", details); + } + return report; + } + + private static CsvFacts scanCsv(File source, ImportTaskSpec spec, List findings) { + try { + Charset charset = ImportFileProbe.effectiveCharset(source, + spec.getOptions() == null ? null : spec.getOptions().getCharset()); + char quote = ImportFileProbe.quoteChar( + spec.getOptions() == null ? null : spec.getOptions().getQuoteChar()); + char delimiter = ImportFileProbe.delimiterChar( + spec.getOptions() == null ? null : spec.getOptions().getDelimiter(), charset, source); + CSVFormat csvFormat = ImportFileProbe.csvFormat(delimiter, quote); + List headers = List.of(); + long records = 0L; + long previousLine = 0L; + int expectedWidth = -1; + try (CSVParser parser = ImportFileProbe.openParser(source, charset, csvFormat)) { + for (CSVRecord record : parser) { + records++; + long currentLine = parser.getCurrentLineNumber(); + if (currentLine - previousLine > 1L) { + blocker(findings, "C3", "A CSV field spans physical lines", + "Logical record " + record.getRecordNumber() + " ends on physical line " + currentLine, + "Use STANDARD mode or pre-process embedded newlines into escaped text."); + } + previousLine = currentLine; + if (expectedWidth < 0) { + expectedWidth = record.size(); + headers = List.copyOf(record.toList()); + validateHeaders(headers, findings); + } else if (record.size() != expectedWidth) { + blocker(findings, "D2", "CSV column count is inconsistent", + "Record " + record.getRecordNumber() + " has " + record.size() + + " columns; expected " + expectedWidth, + "Correct the CSV dialect or regenerate the source with a stable header."); + } + } + } + if (records == 0L) { + blocker(findings, "D2", "CSV has no header", "The file contains no logical records", + "Provide a CSV header and explicit column mapping."); + } + return new CsvFacts(headers, Math.max(0L, records - 1L), true); + } catch (Exception failure) { + blocker(findings, "D1", "The CSV source could not be decoded and parsed deterministically", + failure.getClass().getSimpleName() + ": " + StringUtils.defaultString(failure.getMessage()), + "Specify the correct charset and CSV dialect, then preview the file again."); + return new CsvFacts(List.of(), -1L, false); + } + } + + private static void validateHeaders(List headers, List findings) { + Set normalized = new HashSet<>(); + for (String header : headers) { + String value = StringUtils.trimToEmpty(header).toLowerCase(Locale.ROOT); + if (value.isEmpty() || !normalized.add(value)) { + blocker(findings, "D2", "CSV header cannot identify every source column", + value.isEmpty() ? "A blank column name was found" : "Duplicate column: " + header, + "Provide unique, non-empty headers and an explicit column mapping."); + } + } + } + + private static void assessCsvTarget(ImportTaskSpec spec, List columns, List headers, + boolean relationshipAccepted, List findings) { + if (headers.isEmpty() || columns == null || columns.isEmpty()) { + return; + } + try { + ImportColumnResolver.Resolution resolution = ImportColumnResolver.resolveForSpec(columns, headers, spec); + Set mapped = resolution.tableColumns().stream().map(TableColumn::getName) + .collect(java.util.stream.Collectors.toSet()); + List omittedGeneratedKeys = columns.stream() + .filter(column -> Boolean.TRUE.equals(column.getAutoIncrement())) + .map(TableColumn::getName).filter(name -> !mapped.contains(name)).toList(); + if (!omittedGeneratedKeys.isEmpty()) { + String evidence = "Generated key columns omitted from the input: " + omittedGeneratedKeys; + if (relationshipAccepted) { + degradation(findings, "B1", "Generated key allocation order may change under concurrency", + evidence, "The operator accepted this risk; verify dependent data after import."); + } else { + blocker(findings, "B1", "Generated key allocation may break parent/child identity ordering", + evidence, "Map explicit keys, confirm there are no strong relationships, or use STANDARD mode."); + } + } + } catch (RuntimeException invalidMapping) { + blocker(findings, "D2", "Column mapping is not deterministic", invalidMapping.getMessage(), + "Correct duplicate, missing or invalid source/target mappings."); + } + } + + private static boolean isCompressed(File source) { + try (var input = Files.newInputStream(source.toPath())) { + int first = input.read(); + int second = input.read(); + return (first == 0x1f && second == 0x8b) || (first == 0x50 && second == 0x4b); + } catch (IOException ignored) { + return false; + } + } + + private static Map details(ImportAdmissionReport report) { + Map details = new LinkedHashMap<>(); + details.put("verdict", report.getVerdict()); + details.put("requestedMode", report.getRequestedMode()); + details.put("effectiveMode", report.getEffectiveMode()); + details.put("parallelAllowed", report.isParallelAllowed()); + details.put("relationshipRiskAccepted", report.isRelationshipRiskAccepted()); + details.put("fileFormat", report.getFileFormat()); + details.put("fileSizeBytes", report.getFileSizeBytes()); + details.put("dataRows", report.getDataRows()); + details.put("fullScan", report.isFullScan()); + details.put("findings", report.getFindings().stream().map(item -> Map.of( + "code", item.getCode(), "severity", item.getSeverity(), "message", item.getMessage(), + "evidence", StringUtils.defaultString(item.getEvidence()), + "remediation", StringUtils.defaultString(item.getRemediation()))).toList()); + return details; + } + + private static void blocker(List findings, String code, String message, + String evidence, String remediation) { + findings.add(finding(code, BLOCKER, message, evidence, remediation)); + } + + private static void degradation(List findings, String code, String message, + String evidence, String remediation) { + findings.add(finding(code, DEGRADATION, message, evidence, remediation)); + } + + private static ImportAdmissionFinding finding(String code, String severity, String message, + String evidence, String remediation) { + return ImportAdmissionFinding.builder().code(code).severity(severity).message(message) + .evidence(evidence).remediation(remediation).build(); + } + + private record CsvFacts(List headers, long dataRows, boolean fullScan) { + } +} diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/ImportRowBatcher.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/ImportRowBatcher.java new file mode 100644 index 0000000000..b25ee55e51 --- /dev/null +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/ImportRowBatcher.java @@ -0,0 +1,936 @@ +package ai.chat2db.community.domain.core.impl.task.imports; + +import ai.chat2db.community.domain.api.model.metadata.DataType; +import ai.chat2db.community.domain.api.model.metadata.TableColumn; +import ai.chat2db.community.domain.api.model.task.CsvOptions; +import ai.chat2db.community.domain.api.model.task.ImportOptions; +import ai.chat2db.community.domain.api.model.task.ImportTaskSpec; +import ai.chat2db.community.domain.api.model.task.ResumeState; +import ai.chat2db.community.domain.api.model.task.TaskCancelledException; +import ai.chat2db.community.domain.api.model.task.TaskErrorCode; +import ai.chat2db.community.domain.api.model.task.TaskExecutionException; +import ai.chat2db.community.domain.api.model.task.TaskExecutionMode; +import ai.chat2db.community.domain.api.model.value.SQLDataValue; +import ai.chat2db.community.domain.api.service.task.TaskExecutionContext; +import ai.chat2db.community.domain.core.impl.task.AdaptiveBatchSizer; +import ai.chat2db.community.domain.core.impl.task.AdaptiveConcurrencyGate; +import ai.chat2db.community.domain.core.impl.task.TaskResumeJournal; +import ai.chat2db.community.domain.core.impl.task.imports.ImportColumnResolver.Resolution; +import ai.chat2db.community.domain.core.impl.task.imports.excel.CsvImportValueNormalizer; +import ai.chat2db.spi.ISqlBuilder; +import ai.chat2db.spi.IValueProcessor; +import ai.chat2db.spi.model.datasource.ConnectInfo; +import ai.chat2db.spi.model.request.SingleInsertSqlRequest; +import ai.chat2db.spi.sql.Chat2DBContext; +import ai.chat2db.spi.sql.ConnectionPool; +import com.alibaba.fastjson2.JSON; +import com.alibaba.fastjson2.JSONObject; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.SQLNonTransientConnectionException; +import java.sql.SQLRecoverableException; +import java.sql.SQLTransientConnectionException; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ConcurrentSkipListMap; +import java.util.stream.Stream; +import java.util.concurrent.atomic.LongAdder; + +/** + * Turns file rows into buffered {@code INSERT} statements and executes them in JDBC batches. + * With {@code onError=SKIP} a failing row is retried individually and recorded in a + * {@code REJECT}-role NDJSON sub-artifact instead of aborting the task. + * + *

Parallel execution: the worker count resolves from the + * {@code chat2db.task.import.parallelism} system property ({@code 0}, the default, picks the + * adaptive band {@code [2, min(16, CPU cores)]}, {@code 1} forces the serial path, and explicit + * values are clamped into the band), finished batches are handed to + * partitioned queues so batch {@code n} is always executed before batch {@code n + workerCount}: + * per worker the order is strict, while workers run in parallel. The number of active + * workers and the batch size are self-tuning (see {@link AdaptiveConcurrencyGate} and + * {@link AdaptiveBatchSizer}), so the pipeline converges to the throughput the target database + * actually sustains. Rows have no ordering constraints, so inter-worker interleaving is safe; the + * only visible effect is that auto-generated key values may interleave across workers. + */ +@Slf4j +public final class ImportRowBatcher implements AutoCloseable { + + private static final int DEFAULT_BATCH_ROWS = 500; + + private static final int QUEUE_CAPACITY = 4; + + /** Upper bound of the adaptive worker band, also capped by the machine's CPU count. */ + private static final int MAX_WORKERS = 16; + + /** How long a worker waits for an adaptive gate permit before degrading to ungated execution. */ + private static final long GATE_WAIT_MILLIS = 30_000L; + + private static final String RESUME_KIND_IMPORT = "IMPORT_WATERMARK"; + + private static final String ON_ERROR_SKIP = "SKIP"; + + private static final String REJECT_ROLE = "REJECT"; + + private final ImportTaskSpec spec; + + private final TaskExecutionContext context; + + private final Resolution resolution; + + private final ImportOptions options; + + private final CsvOptions csvOptions; + + private final IValueProcessor valueProcessor; + + private final ISqlBuilder sqlBuilder; + + private final ConnectInfo connectInfo; + + private final ImportSqlExecutor sqlExecutor; + + private final AdaptiveBatchSizer batchSizer = new AdaptiveBatchSizer(DEFAULT_BATCH_ROWS); + + private final LongAdder importedCount = new LongAdder(); + + private final Object rejectLock = new Object(); + + private final List bufferedSqls = new ArrayList<>(DEFAULT_BATCH_ROWS); + + private final List bufferedRows = new ArrayList<>(DEFAULT_BATCH_ROWS); + + private final List bufferedRowNumbers = new ArrayList<>(DEFAULT_BATCH_ROWS); + + private BufferedWriter rejectWriter; + + private long rejectedRowCount; + + // --- parallel-execution state, null on the serial path --- + private final int workerCount; + + private final List> queues; + + private final ExecutorService workerPool; + + private final AdaptiveConcurrencyGate gate; + + private final AtomicBoolean aborted = new AtomicBoolean(); + + private final AtomicReference failure = new AtomicReference<>(); + + private final AtomicInteger inFlightBatches = new AtomicInteger(); + + private final AtomicInteger peakInFlightBatches = new AtomicInteger(); + + private final Object quiesceMonitor = new Object(); + + private long submittedBatches; + + private final long createdNanos = System.nanoTime(); + + private volatile long totalImportNanos; + + private final AtomicBoolean ungatedWarned = new AtomicBoolean(); + + // --- three-layer resume state: rows below the durable watermark are committed --------- + private final Map sourceIdentity; + + private final long resumeBelowRow; + + private final TaskResumeJournal journal; + + /** Standard mode: serial path with a fixed batch size (see {@link TaskExecutionMode}). */ + private final boolean standardMode; + + /** Batch sequence -> first row number, for submitted-but-not-durable batches. */ + private final ConcurrentSkipListMap inFlightFirstRows = + new ConcurrentSkipListMap<>(); + + private long lastAcceptedRow; + + private long bufferedFirstRow = Long.MAX_VALUE; + + private long batchesSinceCheckpoint; + + // Intervals are read per construction (not class-load) so tests can retune them reliably. + /** Batch interval of the Layer-1 journal progress records. */ + private final int journalProgressInterval = + Integer.getInteger("chat2db.task.import.journal-interval", 8); + + /** Batch interval of the Layer-2 task-storage checkpoints. */ + private final int checkpointInterval = + Integer.getInteger("chat2db.task.import.checkpoint-interval", 64); + + /** Batch interval of the Layer-3 committed-snapshot generations. */ + private final int snapshotInterval = + Integer.getInteger("chat2db.task.import.snapshot-interval", 256); + + public ImportRowBatcher(ImportTaskSpec spec, TaskExecutionContext context, Resolution resolution, + IValueProcessor valueProcessor) { + this.spec = spec; + this.context = context; + this.resolution = resolution; + this.options = spec.getOptions() == null ? new ImportOptions() : spec.getOptions(); + this.csvOptions = spec.getCsvOptions() == null ? null : spec.getCsvOptions().validate(); + this.valueProcessor = valueProcessor; + this.sqlBuilder = Chat2DBContext.getSqlBuilder(); + this.connectInfo = Chat2DBContext.getConnectInfo(); + this.standardMode = !TaskExecutionMode.isUltraFast(spec.getMode()); + this.sqlExecutor = new ImportSqlExecutor(context); + this.sourceIdentity = sourceIdentity(spec); + this.resumeBelowRow = resolveResumeBelowRow(spec, context); + if (resumeBelowRow > 0) { + log.info("Import resume: the first {} rows are durable from the interrupted run; " + + "they will be skipped", resumeBelowRow); + } + this.journal = TaskResumeJournal.open(context.taskId(), sourceIdentity); + if (journal != null && resumeBelowRow > 0) { + journal.progress("RESUMED", resumeBelowRow); + } + int requestedWorkers = standardMode ? 1 : effectiveWorkerCount(connectInfo); + List> builtQueues = null; + AdaptiveConcurrencyGate builtGate = null; + ExecutorService builtPool = null; + if (requestedWorkers > 1) { + try { + builtQueues = new ArrayList<>(requestedWorkers); + for (int index = 0; index < requestedWorkers; index++) { + builtQueues.add(new ArrayBlockingQueue<>(QUEUE_CAPACITY)); + } + builtGate = AdaptiveConcurrencyGate.create(2, requestedWorkers); + builtPool = Executors.newFixedThreadPool(requestedWorkers, runnable -> { + Thread thread = new Thread(runnable, "chat2db-import-" + context.taskId()); + thread.setDaemon(true); + return thread; + }); + } catch (Throwable parallelStartupFailure) { + // Adaptive parallel plumbing must never block the import: fall back to the exact + // serial path, which stays fully supported. + log.warn("Parallel import infrastructure failed to start; degrading to serial execution", + parallelStartupFailure); + if (builtPool != null) { + builtPool.shutdownNow(); + } + builtQueues = null; + builtGate = null; + builtPool = null; + requestedWorkers = 1; + } + } + this.workerCount = requestedWorkers; + this.queues = builtQueues; + this.gate = builtGate; + this.workerPool = builtPool; + if (this.workerPool != null) { + for (int index = 0; index < this.workerCount; index++) { + int workerIndex = index; + this.workerPool.execute(() -> runWorker(workerIndex)); + } + } + warnIfSelfReferencing(spec, this.workerCount > 1); + } + + public void accept(long fileRowNumber, List fileValues) { + try { + acceptRow(fileRowNumber, fileValues); + } catch (RuntimeException taskFailure) { + recordFailure(taskFailure); + throw taskFailure; + } + } + + private void acceptRow(long fileRowNumber, List fileValues) { + context.checkCancelled(); + throwIfFailed(); + if (fileRowNumber <= resumeBelowRow) { + // Durable from the interrupted run; skipping keeps the target duplicate-free. + return; + } + String sql; + String raw; + try { + sql = buildInsert(fileRowNumber, fileValues); + raw = JSON.toJSONString(fileValues); + } catch (RuntimeException conversionFailure) { + handleFailedRow(fileRowNumber, fileValues, conversionFailure); + return; + } + bufferedSqls.add(sql); + bufferedRows.add(raw); + bufferedRowNumbers.add(fileRowNumber); + if (fileRowNumber > lastAcceptedRow) { + lastAcceptedRow = fileRowNumber; + } + if (bufferedFirstRow == Long.MAX_VALUE) { + bufferedFirstRow = fileRowNumber; + } + if (bufferedSqls.size() >= batchSizer.batchSize()) { + flushBufferedBatch(); + } + } + + public long importedRows() { + return importedCount.sum(); + } + + public long rejectedRows() { + synchronized (rejectLock) { + return rejectedRowCount; + } + } + + /** Final adaptive batch size; observability for tests and ops dashboards. */ + public int finalBatchSize() { + return batchSizer.batchSize(); + } + + /** Available permits of the adaptive gate at call time (1 on the serial path). */ + public int gatePermits() { + return gate == null ? 1 : gate.availablePermits(); + } + + /** Wall time of the import measured in {@link #close()}; 0 before the first close. */ + public long elapsedNanos() { + return totalImportNanos; + } + + /** + * Final adaptive state of the most recently closed batcher. A process-wide snapshot because + * callers that drive the importer through {@code CSVImporter} never hold the instance; the + * last closed batcher wins when several imports run at once. + */ + public record ImportTuningSnapshot(int workers, long batches, long rows, long nanos, + int batchSize, int gatePermits, int peakInFlightBatches) { } + + private static final AtomicReference LAST_TUNING = new AtomicReference<>(); + + public static ImportTuningSnapshot lastTuningSnapshot() { + return LAST_TUNING.get(); + } + + /** + * Executes whatever is buffered; called at end of stream and whenever the caller needs a sync + * point. In parallel mode this waits until every submitted batch finished. + */ + public void flush() { + try { + context.checkCancelled(); + throwIfFailed(); + flushBufferedBatch(); + if (workerCount > 1) { + awaitQuiesce(); + } + } catch (RuntimeException taskFailure) { + recordFailure(taskFailure); + throw taskFailure; + } + } + + /** Submits the current buffer without turning normal producer flow into a global barrier. */ + private void flushBufferedBatch() { + if (bufferedSqls.isEmpty()) { + return; + } + long firstRowNumber = bufferedFirstRow; + PendingBatch batch = new PendingBatch(List.copyOf(bufferedSqls), List.copyOf(bufferedRows), + List.copyOf(bufferedRowNumbers), submittedBatches, firstRowNumber); + submittedBatches++; + bufferedSqls.clear(); + bufferedRows.clear(); + bufferedRowNumbers.clear(); + bufferedFirstRow = Long.MAX_VALUE; + executeBatch(batch); + } + + private void executeBatch(PendingBatch batch) { + inFlightFirstRows.put(batch.seq(), batch.firstRowNumber()); + if (workerCount > 1) { + submitBatch(batch); + } else { + executeWithTolerance(batch); + } + } + + /** + * Executes a finished batch in the calling (serial) or a worker (parallel) context and reports + * the measured cost to the adaptive sizer and gate. + */ + private void executeWithTolerance(PendingBatch batch) { + long started = System.nanoTime(); + int rows = batch.sqls().size(); + boolean fullyHandled = false; + try { + try { + sqlExecutor.executeBatch(batch.sqls()); + importedCount.add(rows); + fullyHandled = true; + } catch (TaskCancelledException cancellation) { + throw cancellation; + } catch (RuntimeException batchFailure) { + context.logError("IMPORT_BATCH_FAILED", "Could not import batch", Map.of( + "statementCount", rows, + "message", StringUtils.defaultString(batchFailure.getMessage()))); + if (!isSkipMode()) { + throw batchFailure; + } + // In SKIP mode every row ends handled (imported or recorded in the reject file), + // so the watermark may advance past the batch once the replay finishes. + replayIndividually(batch); + fullyHandled = true; + } + } finally { + long elapsed = System.nanoTime() - started; + if (gate != null) { + gate.record(rows, elapsed); + } + if (!standardMode) { + batchSizer.record(rows, elapsed); + } + if (fullyHandled) { + // Removed only on full handling: a failed batch keeps its rows un-durable, so the + // watermark must stay below it or a resume would skip live rows. + inFlightFirstRows.remove(batch.seq()); + maybeCheckpoint(); + } + if (workerCount > 1) { + batchCompleted(); + } + } + } + + /** Retries a failed batch row by row so genuinely bad rows can be skipped. */ + private void replayIndividually(PendingBatch batch) { + for (int index = 0; index < batch.sqls().size(); index++) { + try { + sqlExecutor.executeBatch(List.of(batch.sqls().get(index))); + importedCount.increment(); + } catch (RuntimeException rowFailure) { + if (isConnectionFailure(rowFailure)) { + throw rowFailure; + } + handleRejectedRow(batch.rowNumbers().get(index), batch.rows().get(index), + rootMessage(rowFailure)); + } + } + } + + private static boolean isConnectionFailure(Throwable failure) { + Throwable current = failure; + while (current != null) { + if (current instanceof SQLNonTransientConnectionException + || current instanceof SQLRecoverableException + || current instanceof SQLTransientConnectionException) { + return true; + } + if (current instanceof SQLException sqlException + && StringUtils.startsWith(sqlException.getSQLState(), "08")) { + return true; + } + current = current.getCause() == current ? null : current.getCause(); + } + return false; + } + + // --- parallel plumbing --------------------------------------------------------------- + + /** + * Resolves the worker count from the {@code chat2db.task.import.parallelism} system property: + * {@code 0}, the default, picks the adaptive band ceiling {@code max(2, min(16, CPU cores))}; + * {@code 1} forces the serial path; explicit values are clamped into the [2, ceiling] band so + * a pinned value can neither exceed the machine nor drop below the minimum fan-out. Parallel + * workers each need their own connection, so without a JDBC url (test fixtures and + * non-relational sources bind a prebuilt connection instead) the batcher stays serial. + */ + private static int effectiveWorkerCount(ConnectInfo connectInfo) { + if (StringUtils.isBlank(connectInfo.getUrl())) { + return 1; + } + int cores = Math.max(1, Runtime.getRuntime().availableProcessors()); + int ceiling = Math.max(2, Math.min(MAX_WORKERS, cores)); + int configured = Integer.getInteger("chat2db.task.import.parallelism", 0); + if (configured == 1) { + return 1; + } + if (configured > 1) { + return Math.max(2, Math.min(ceiling, configured)); + } + return ceiling; + } + + private void submitBatch(PendingBatch batch) { + throwIfFailed(); + int inFlight = inFlightBatches.incrementAndGet(); + peakInFlightBatches.accumulateAndGet(inFlight, Math::max); + BlockingQueue queue = queues.get((int) (batch.seq() % workerCount)); + try { + // Bounded offer with failure checks: when every worker died there is nobody left to + // drain the queues, and a blocking put would hang the import forever. + while (!queue.offer(batch, 200L, TimeUnit.MILLISECONDS)) { + throwIfFailed(); + context.checkCancelled(); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + inFlightBatches.decrementAndGet(); + throw new TaskExecutionException(TaskErrorCode.IMPORT_FAILED.name(), "Import was interrupted"); + } + } + + private void batchCompleted() { + if (inFlightBatches.decrementAndGet() == 0) { + synchronized (quiesceMonitor) { + quiesceMonitor.notifyAll(); + } + } + } + + private void awaitQuiesce() { + synchronized (quiesceMonitor) { + while (inFlightBatches.get() > 0) { + throwIfFailed(); + try { + quiesceMonitor.wait(50L); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new TaskExecutionException(TaskErrorCode.IMPORT_FAILED.name(), + "Import was interrupted"); + } + } + } + throwIfFailed(); + } + + /** + * The durable watermark: every source row below it is committed. It is the first row of the + * earliest in-flight batch (batches complete out of order), the first still-buffered row, or + * one past the last accepted row when nothing is in flight. + */ + private long durableWatermark() { + long firstInFlight = inFlightFirstRows.isEmpty() + ? Long.MAX_VALUE : inFlightFirstRows.firstEntry().getValue(); + long firstBuffered = bufferedSqls.isEmpty() ? Long.MAX_VALUE : bufferedFirstRow; + return Math.min(firstInFlight, Math.min(firstBuffered, lastAcceptedRow + 1)); + } + + /** Layered cadence: journal every 8, storage checkpoint every 64, snapshot every 256 batches. */ + private void maybeCheckpoint() { + batchesSinceCheckpoint++; + long rowsDone = durableWatermark() - 1; + try { + if (journal != null && batchesSinceCheckpoint % journalProgressInterval == 0) { + journal.progress("IMPORTING", rowsDone); + } + if (batchesSinceCheckpoint % checkpointInterval == 0) { + context.checkpoint(ResumeState.builder() + .shardNo(0) + .kind(RESUME_KIND_IMPORT) + .cursorJson(resumeCursorJson(rowsDone)) + .rowsDone(rowsDone) + .updatedAt(new Date()) + .build()); + } + if (journal != null && batchesSinceCheckpoint % snapshotInterval == 0) { + journal.snapshot(rowsDone); + } + } catch (TaskCancelledException cancellation) { + throw cancellation; + } catch (Throwable checkpointFailure) { + // A failed checkpoint never blocks the import; the next cadence retries. + log.warn("Import resume checkpoint failed; continuing", checkpointFailure); + } + } + + private String resumeCursorJson(long rowsDone) { + JSONObject cursor = new JSONObject(); + cursor.put("watermark", rowsDone + 1); + cursor.putAll(sourceIdentity); + return cursor.toJSONString(); + } + + private Map sourceIdentity(ImportTaskSpec spec) { + File source = new File(StringUtils.defaultString(spec.getSourceFile())); + Map identity = new HashMap<>(); + identity.put("sourcePath", source.toPath().toAbsolutePath().normalize().toString()); + identity.put("sourceLength", source.isFile() ? source.length() : -1L); + identity.put("sourceLastModified", source.isFile() ? source.lastModified() : -1L); + return identity; + } + + /** + * Resolves the resume position: the newest VALID candidate among the journal tail, the newest + * and committed snapshot generations, and the task-storage checkpoints. A candidate whose + * source identity no longer matches (the file was rewritten between runs) is discarded so a + * stale checkpoint can never skip live rows. + */ + private long resolveResumeBelowRow(ImportTaskSpec spec, TaskExecutionContext context) { + try { + File dir = TaskResumeJournal.directoryFor(context.taskId()); + long below = 0L; + for (TaskResumeJournal.Snapshot candidate : Stream + .of(TaskResumeJournal.recoverNewest(dir), TaskResumeJournal.recoverCommitted(dir), + TaskResumeJournal.recoverTail(dir)) + .flatMap(Optional::stream).toList()) { + if (identityMatches(candidate.identity()) && candidate.rowsDone() > below) { + below = candidate.rowsDone(); + } + } + for (ResumeState state : context.resumeStates()) { + if (state == null || !RESUME_KIND_IMPORT.equals(state.getKind()) + || state.getRowsDone() == null || state.getRowsDone() < 0) { + continue; + } + if (identityMatches(parseIdentity(state.getCursorJson())) + && state.getRowsDone() > below) { + below = state.getRowsDone(); + } + } + return below; + } catch (Throwable resumeProbeFailure) { + log.warn("Import resume probe failed; the import restarts from the first row", + resumeProbeFailure); + return 0L; + } + } + + private boolean identityMatches(Map stored) { + if (stored == null || stored.isEmpty()) { + return false; + } + Long storedLength = asLong(stored.get("sourceLength")); + Long storedModified = asLong(stored.get("sourceLastModified")); + Long currentLength = asLong(sourceIdentity.get("sourceLength")); + Long currentModified = asLong(sourceIdentity.get("sourceLastModified")); + String storedPath = stored.get("sourcePath") instanceof String path + ? StringUtils.trimToNull(path) : null; + String currentPath = sourceIdentity.get("sourcePath") instanceof String path + ? StringUtils.trimToNull(path) : null; + return storedPath != null && storedPath.equals(currentPath) + && storedLength != null && storedModified != null && currentLength != null + && currentModified != null && storedLength >= 0 + && storedLength.equals(currentLength) && storedModified.equals(currentModified); + } + + private static Long asLong(Object value) { + if (value instanceof Number number) { + return number.longValue(); + } + try { + return value == null ? null : Long.parseLong(String.valueOf(value)); + } catch (NumberFormatException e) { + return null; + } + } + + private Map parseIdentity(String cursorJson) { + try { + JSONObject cursor = JSONObject.parseObject(StringUtils.defaultString(cursorJson)); + if (cursor == null || !cursor.containsKey("sourcePath") + || !cursor.containsKey("sourceLength")) { + return Map.of(); + } + Map identity = new HashMap<>(); + identity.put("sourcePath", cursor.getString("sourcePath")); + identity.put("sourceLength", cursor.getLong("sourceLength")); + identity.put("sourceLastModified", cursor.getLong("sourceLastModified")); + return identity; + } catch (Exception parseFailure) { + return Map.of(); + } + } + + /** + * Warns once when the target references itself (e.g. {@code category.parent_id}): parallel + * batches carry no parent-before-child order, so an enforced self-referencing foreign key + * needs the serial path or a deferred constraint. Purely advisory — never fails the import. + */ + private void warnIfSelfReferencing(ImportTaskSpec spec, boolean parallel) { + if (!parallel) { + return; + } + try (ResultSet keys = Chat2DBContext.getConnection().getMetaData() + .getImportedKeys(null, null, spec.getTarget().getTableName())) { + while (keys.next()) { + String referencing = keys.getString("FKTABLE_NAME"); + String referenced = keys.getString("PKTABLE_NAME"); + if (referencing != null && referencing.equalsIgnoreCase(referenced)) { + log.warn("Target table {} references itself; parallel batch import carries no " + + "parent-before-child order — if the foreign key is enforced, use the " + + "serial path (chat2db.task.import.parallelism=1) or defer the constraint", + spec.getTarget().getTableName()); + break; + } + } + } catch (Throwable probeFailure) { + log.debug("Self-reference probe skipped", probeFailure); + } + } + + private void runWorker(int workerIndex) { + Thread.currentThread().setName("chat2db-import-" + context.taskId() + "-" + workerIndex); + // Created on first use and owned by this worker until it exits; copy() carries no + // connection, so Chat2DBContext.getConnection() builds a dedicated one per worker. + ConnectInfo isolated = null; + try { + isolated = connectInfo.copy(); + isolated.setLoginUser("task-" + context.taskId() + "#import-" + workerIndex); + Chat2DBContext.putContext(isolated); + while (true) { + PendingBatch batch = queues.get(workerIndex).take(); + if (batch == END_OF_QUEUE) { + return; + } + boolean permitted = gate.admit(GATE_WAIT_MILLIS); + if (!permitted && ungatedWarned.compareAndSet(false, true)) { + log.warn("Adaptive import gate did not admit within {}ms; executing batches " + + "ungated until the gate recovers (degraded concurrency)", GATE_WAIT_MILLIS); + } + try { + throwIfFailed(); + executeWithTolerance(batch); + } finally { + gate.relinquish(permitted); + } + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } catch (Throwable t) { + recordFailure(t); + } finally { + if (isolated != null) { + // Hand the dedicated connection back to the pool (or close it) instead of + // leaking it until the JVM exits. + ConnectionPool.close(isolated); + } + Chat2DBContext.removeContext(); + } + } + + private void recordFailure(Throwable taskFailure) { + failure.compareAndSet(null, taskFailure); + aborted.set(true); + synchronized (quiesceMonitor) { + quiesceMonitor.notifyAll(); + } + } + + private void throwIfFailed() { + if (aborted.get()) { + Throwable cause = failure.get(); + throw cause instanceof RuntimeException runtime ? runtime + : new TaskExecutionException(TaskErrorCode.IMPORT_FAILED.name(), + "Import failed", cause); + } + } + + private static final PendingBatch END_OF_QUEUE = + new PendingBatch(List.of(), List.of(), List.of(), -1L, Long.MAX_VALUE); + + private record PendingBatch(List sqls, List rows, List rowNumbers, + long seq, long firstRowNumber) { + } + + // --- rejected-row bookkeeping --------------------------------------------------------- + + private void handleFailedRow(long fileRowNumber, List fileValues, RuntimeException failure) { + handleRejectedRow(fileRowNumber, JSON.toJSONString(fileValues), rootMessage(failure)); + } + + @SuppressWarnings("unused") + private void handleFailedRowText(String rawRow, RuntimeException failure) { + handleRejectedRow(null, rawRow, rootMessage(failure)); + } + + private void handleRejectedRow(Long fileRowNumber, String rawRow, String reason) { + if (!isSkipMode()) { + throw new TaskExecutionException(TaskErrorCode.IMPORT_FAILED.name(), + "Import row failed: " + reason); + } + synchronized (rejectLock) { + rejectedRowCount++; + Integer maxErrors = options.getMaxErrors(); + if (maxErrors != null && maxErrors >= 0 && rejectedRowCount > maxErrors) { + throw new TaskExecutionException(TaskErrorCode.IMPORT_FAILED.name(), + "Import aborted after " + rejectedRowCount + " rejected rows"); + } + try { + rejectWriter().write(JSON.toJSONString(Map.of( + "row", fileRowNumber == null ? -1L : fileRowNumber, + "line", rawRow, + "reason", reason == null ? "unknown" : reason))); + rejectWriter().write("\n"); + } catch (IOException e) { + throw new UncheckedIOException("Could not write reject file", e); + } + } + context.logWarn("IMPORT_ROW_REJECTED", "Import row rejected: " + reason, + Map.of("rejectedRows", rejectedRows())); + } + + private BufferedWriter rejectWriter() throws IOException { + if (rejectWriter == null) { + String fileName = StringUtils.firstNonBlank( + new java.io.File(StringUtils.defaultString(spec.getSourceFile())).getName(), "import") + + ".rejects.ndjson"; + var draft = context.createArtifact(REJECT_ROLE, + StringUtils.substringBeforeLast(spec.getSourceFile(), java.io.File.separator), + fileName, "application/x-ndjson"); + rejectWriter = Files.newBufferedWriter(draft.getTemporaryFile().toPath(), StandardCharsets.UTF_8); + } + return rejectWriter; + } + + private String buildInsert(long fileRowNumber, List fileValues) { + List tableColumnNames = new ArrayList<>(resolution.tableColumns().size()); + List values = new ArrayList<>(resolution.tableColumns().size()); + for (int index = 0; index < resolution.tableColumns().size(); index++) { + TableColumn column = resolution.tableColumns().get(index); + Integer sourceIndex = resolution.fileIndexes().get(index); + String raw = sourceIndex != null && sourceIndex < fileValues.size() + ? fileValues.get(sourceIndex) : null; + tableColumnNames.add(column.getName()); + values.add(toSqlLiteral(column, raw, fileRowNumber)); + } + return sqlBuilder.dml().buildInsert(SingleInsertSqlRequest.builder() + .databaseName(connectInfo.getDatabaseName()) + .schemaName(connectInfo.getSchemaName()) + .tableName(spec.getTarget().getTableName()) + .columnList(tableColumnNames) + .valueList(values) + .build()); + } + + private String toSqlLiteral(TableColumn column, String raw, long fileRowNumber) { + if (raw == null || (options.getNullString() != null && options.getNullString().equals(raw))) { + return null; + } + if (raw.isEmpty()) { + return null; + } + if (csvOptions != null) { + raw = CsvImportValueNormalizer.normalize(raw, column, csvOptions, fileRowNumber); + } + DataType dataType = new DataType(); + dataType.setDataTypeName(column.getColumnType()); + dataType.setScale(column.getDecimalDigits()); + dataType.setPrecision(column.getColumnSize()); + SQLDataValue sqlDataValue = new SQLDataValue(); + sqlDataValue.setDataType(dataType); + sqlDataValue.setValue(raw); + return valueProcessor.getSqlValueString(sqlDataValue); + } + + private boolean isSkipMode() { + return ON_ERROR_SKIP.equalsIgnoreCase(StringUtils.trimToEmpty(options.getOnError())); + } + + private static String rootMessage(Throwable failure) { + Throwable current = failure; + while (current.getCause() != null && current.getCause() != current) { + current = current.getCause(); + } + return current.getMessage(); + } + + /** Stops pending writes when the source parser fails outside the batch executor. */ + public void abort(RuntimeException sourceFailure) { + recordFailure(sourceFailure); + } + + @Override + public void close() { + Throwable existingFailure = failure.get(); + try { + if (existingFailure == null) { + flush(); + } + } finally { + if (workerPool != null) { + for (BlockingQueue queue : queues) { + while (!queue.offer(END_OF_QUEUE)) { + if (aborted.get()) { + break; + } + } + } + if (aborted.get()) { + workerPool.shutdownNow(); + } else { + workerPool.shutdown(); + } + try { + if (!workerPool.awaitTermination(30L, TimeUnit.SECONDS)) { + workerPool.shutdownNow(); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + workerPool.shutdownNow(); + } + } + totalImportNanos = System.nanoTime() - createdNanos; + long importedRows = importedCount.sum(); + double seconds = totalImportNanos / 1_000_000_000.0D; + long rowsPerSecond = seconds > 0 ? (long) (importedRows / seconds) : 0L; + // Final adaptive state: how far the AIMD gate grew/shrank and where the batch sizer + // settled, for production observability and stress-test reporting. + log.info("Import batcher finished: workers={}, batches={}, imported rows={} in {}s " + + "-> {} rows/s, final batch size={}, final gate permits={}", + workerCount, submittedBatches, importedRows, Math.round(seconds), rowsPerSecond, + batchSizer.batchSize(), gate == null ? 1 : gate.availablePermits()); + LAST_TUNING.set(new ImportTuningSnapshot(workerCount, submittedBatches, importedRows, + totalImportNanos, batchSizer.batchSize(), + gate == null ? 1 : gate.availablePermits(), peakInFlightBatches.get())); + if (failure.get() == null) { + // Tail checkpoint: after the final flush everything accepted is durable. + try { + long rowsDone = durableWatermark() - 1; + context.checkpoint(ResumeState.builder() + .shardNo(0) + .kind(RESUME_KIND_IMPORT) + .cursorJson(resumeCursorJson(rowsDone)) + .rowsDone(rowsDone) + .updatedAt(new Date()) + .build()); + } catch (Throwable tailCheckpointFailure) { + log.warn("Final import resume checkpoint failed", tailCheckpointFailure); + } + } + if (journal != null) { + if (failure.get() == null) { + journal.cleanup(); + } else { + journal.progress("FAILED", durableWatermark() - 1); + journal.preserve(); + } + } + if (rejectWriter != null) { + try { + rejectWriter.flush(); + rejectWriter.close(); + } catch (IOException e) { + log.warn("Could not close import reject writer", e); + } + } + } + } +} diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/excel/BaseExcelImporter.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/excel/BaseExcelImporter.java index 7958050975..558200a014 100644 --- a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/excel/BaseExcelImporter.java +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/excel/BaseExcelImporter.java @@ -1,55 +1,51 @@ package ai.chat2db.community.domain.core.impl.task.imports.excel; -import ai.chat2db.community.domain.core.impl.task.imports.BaseImporter; -import ai.chat2db.community.domain.core.impl.task.imports.ImportSqlExecutor; -import ai.chat2db.community.domain.api.model.task.TaskConstants; -import ai.chat2db.community.domain.api.model.task.TaskCancelledException; +import ai.chat2db.community.domain.api.model.metadata.TableColumn; import ai.chat2db.community.domain.api.model.task.ImportTaskSpec; -import ai.chat2db.community.domain.api.model.task.ImportColumnMapping; -import ai.chat2db.community.domain.api.model.task.CsvOptions; -import ai.chat2db.community.domain.api.model.task.TaskEventCode; -import ai.chat2db.community.domain.api.model.task.TaskStage; -import ai.chat2db.community.domain.api.model.task.UnmappedTargetStrategy; import ai.chat2db.community.domain.api.service.task.TaskExecutionContext; -import ai.chat2db.spi.ISqlBuilder; +import ai.chat2db.community.domain.core.impl.task.imports.BaseImporter; +import ai.chat2db.community.domain.core.impl.task.imports.ImportColumnResolver; +import ai.chat2db.community.domain.core.impl.task.imports.ImportRowBatcher; import ai.chat2db.spi.IValueProcessor; -import ai.chat2db.community.domain.api.model.metadata.TableColumn; import ai.chat2db.spi.sql.Chat2DBContext; -import ai.chat2db.spi.model.datasource.ConnectInfo; -import ai.chat2db.spi.model.request.SingleInsertSqlRequest; import com.alibaba.excel.EasyExcel; import com.alibaba.excel.context.AnalysisContext; import com.alibaba.excel.event.AnalysisEventListener; import com.alibaba.excel.metadata.data.ReadCellData; import com.alibaba.excel.support.ExcelTypeEnum; import com.alibaba.excel.util.ConverterUtils; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; import java.io.File; -import java.util.*; - - -@Slf4j +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** + * XLSX/XLS import through EasyExcel, sharing column resolution, batching and reject handling with + * the CSV path. + */ public abstract class BaseExcelImporter extends BaseImporter { + @Override protected void doImportData(ImportTaskSpec spec, TaskExecutionContext context, List columns) { - context.checkCancelled(); ExcelTypeEnum excelType = getExcelType(); - NoModelDataListener noModelDataListener = new NoModelDataListener(spec, context, columns); - EasyExcel.read(new File(spec.getSourceFile()), noModelDataListener) - .excelType(excelType) - .sheet() - .headRowNumber(1) - .doRead(); - context.checkCancelled(); + try (NoModelDataListener listener = new NoModelDataListener(spec, context, columns, + Chat2DBContext.getDbMetaData().getValueProcessor())) { + try { + EasyExcel.read(new File(spec.getSourceFile()), listener) + .excelType(excelType).sheet().headRowNumber(1).doRead(); + context.checkCancelled(); + listener.finish(); + } catch (RuntimeException failure) { + listener.abort(failure); + throw failure; + } + } } protected abstract ExcelTypeEnum getExcelType(); - - public class NoModelDataListener extends AnalysisEventListener> { - + public class NoModelDataListener extends AnalysisEventListener> implements AutoCloseable { private final ImportTaskSpec spec; @@ -57,225 +53,85 @@ public class NoModelDataListener extends AnalysisEventListener columns; - private Map headMap; - - private Map mappedHeadMap; - - private List tableColumns; - - private List tableColumnList; - - private List sqlList; - - private long successCount; - - private long skippedCount; - - private static final int BATCH_SIZE = 1000; - private final IValueProcessor valueProcessor; - private final ConnectInfo connectInfo; - - private final ISqlBuilder sqlBuilder; + private ImportColumnResolver.Resolution resolution; - private final ImportSqlExecutor sqlExecutor; + private ImportRowBatcher batcher; - private final CsvOptions csvOptions; + private long rowNumber; - private long sourceRowNumber; - - public NoModelDataListener(ImportTaskSpec spec, TaskExecutionContext taskContext, - List columns) { + private NoModelDataListener(ImportTaskSpec spec, TaskExecutionContext taskContext, + List columns, IValueProcessor valueProcessor) { this.spec = spec; - this.columns = columns; this.taskContext = taskContext; - this.valueProcessor = Chat2DBContext.getDbMetaData().getValueProcessor(); - this.connectInfo = Chat2DBContext.getConnectInfo(); - this.sqlBuilder = Chat2DBContext.getSqlBuilder(); - this.sqlExecutor = new ImportSqlExecutor(taskContext); - this.csvOptions = spec.getCsvOptions() == null ? null : spec.getCsvOptions().validate(); + this.columns = columns; + this.valueProcessor = valueProcessor; } - @Override - public void invokeHead(Map> headMap, AnalysisContext context) { - acceptHead(ConverterUtils.convertToStringMap(headMap, context)); - } - - void acceptHead(Map map) { + public void invokeHead(Map> headCells, AnalysisContext context) { this.taskContext.checkCancelled(); - this.headMap = invertMap(map); - this.mappedHeadMap = mappedHeadMap(); - this.tableColumns = getTableColumns(columns, this.headMap); + acceptHead(ConverterUtils.convertToStringMap(headCells, context)); } - private List getTableColumns(List columns, Map headMap) { - List tableColumns = new ArrayList<>(); - this.tableColumnList = new ArrayList<>(); - for (TableColumn column : columns) { - if (shouldInclude(column)) { - tableColumns.add(column); - this.tableColumnList.add(column.getName()); - } - } - return tableColumns; - } - - private Map invertMap(Map map) { - Map out = new HashMap(map.size()); - Iterator it = map.entrySet().iterator(); - while (it.hasNext()) { - Map.Entry entry = (Map.Entry) it.next(); - if (entry.getValue() != null) { - out.put(entry.getValue().toUpperCase(Locale.ROOT), entry.getKey()); - } - } - return out; + void acceptHead(Map headMap) { + this.taskContext.checkCancelled(); + List headers = values(headMap); + resolution = ImportColumnResolver.resolveForSpec(columns, headers, spec); + reportResolution(taskContext, resolution); + ImportColumnResolver.validateForImport(columns, resolution, spec); + batcher = new ImportRowBatcher(spec, taskContext, resolution, valueProcessor); } - @Override public void invoke(Map data, AnalysisContext context) { - acceptRow(data); - } - - void acceptRow(Map data) { - acceptRow(data, 0); + acceptRow(data, ++rowNumber); } void acceptRow(Map data, long sourceRowNumber) { this.taskContext.checkCancelled(); - this.sourceRowNumber = sourceRowNumber; - if (data == null || data.isEmpty()) { - skippedCount++; - return; - } - List values = getValueList(data); - - String sql = getInsertSql(values); - - if (StringUtils.isBlank(sql)) { - skippedCount++; + if (data == null || data.isEmpty() || batcher == null || resolution == null) { return; } - if (sqlList == null) { - sqlList = new ArrayList<>(); - } - sqlList.add(sql); - if (sqlList.size() >= BATCH_SIZE) { - executeBatchInsert(); - } else { - - } + batcher.accept(sourceRowNumber, values(data)); } - private List getValueList(Map data) { - List values = new ArrayList<>(); - for (TableColumn column : tableColumns) { - Integer index = sourceIndex(column.getName()); - if (index == null) { - values.add(null); - continue; - } - String value = data.get(index); - if (value == null) { - values.add(null); - } else { - if (csvOptions != null) { - value = CsvImportValueNormalizer.normalize(value, column, csvOptions, sourceRowNumber); - } - String stringValue = valueProcessor.getSqlValueString(getSQLDataValue(value, column)); - values.add(stringValue); - } + private List values(Map cells) { + int count = cells.keySet().stream().mapToInt(Integer::intValue).max().orElse(-1) + 1; + List values = new ArrayList<>(count); + for (int index = 0; index < count; index++) { + values.add(cells.get(index)); } return values; } - private Map mappedHeadMap() { - Map mapped = new HashMap<>(); - if (spec.getColumnMappings() == null) { - return mapped; - } - for (ImportColumnMapping mapping : spec.getColumnMappings()) { - String source = mapping.getSourceColumn(); - String target = mapping.getTargetColumn(); - Integer sourceIndex = headMap.get(source == null ? null : source.toUpperCase(Locale.ROOT)); - if (sourceIndex != null && StringUtils.isNotBlank(target)) { - mapped.put(target.toUpperCase(Locale.ROOT), sourceIndex); - } - } - return mapped; - } - - private Integer sourceIndex(String targetColumn) { - String target = targetColumn.toUpperCase(Locale.ROOT); - if (spec.getColumnMappings() != null) { - return mappedHeadMap.get(target); - } - return headMap.get(target); - } - - private boolean shouldInclude(TableColumn column) { - if (spec.getColumnMappings() == null) { - return sourceIndex(column.getName()) != null; - } - if (sourceIndex(column.getName()) != null) { - return true; - } - return spec.getUnmappedTarget() == UnmappedTargetStrategy.NULL - && !Boolean.TRUE.equals(column.getAutoIncrement()); - } - - private String getInsertSql(List values) { - return sqlBuilder.dml().buildInsert(SingleInsertSqlRequest.builder() - .databaseName(connectInfo.getDatabaseName()) - .schemaName(connectInfo.getSchemaName()) - .tableName(spec.getTarget().getTableName()) - .columnList(this.tableColumnList) - .valueList(values) - .build()); - } - @Override public void doAfterAllAnalysed(AnalysisContext context) { - finish(); + this.taskContext.checkCancelled(); } void finish() { - this.taskContext.checkCancelled(); - executeBatchInsert(); + if (batcher == null) { + return; + } + batcher.flush(); + taskContext.logInfo("IMPORT_SUMMARY", "Excel import finished", Map.of( + "importedRows", batcher.importedRows(), + "rejectedRows", batcher.rejectedRows())); } - private void executeBatchInsert() { - taskContext.checkCancelled(); - if (sqlList != null && !sqlList.isEmpty()) { - taskContext.logInfo(TaskEventCode.BATCH_EXECUTED.name(), - String.format("Executing batch insert: %s", sqlList.size())); - int statementCount = sqlList.size(); - try { - sqlExecutor.executeBatch(sqlList); - successCount += statementCount; - reportImportProgress(); - } catch (TaskCancelledException e) { - throw e; - } catch (Exception e) { - taskContext.logError(TaskEventCode.IMPORT_BATCH_FAILED.name(), "Could not import batch", Map.of( - "statementCount", statementCount, - "message", StringUtils.defaultString(e.getMessage()))); - throw e; - } + void abort(RuntimeException failure) { + if (batcher != null) { + batcher.abort(failure); } - sqlList = new ArrayList<>(); } - private void reportImportProgress() { - long processedRows = successCount + skippedCount; - int progress = (int) Math.min(TaskConstants.MAX_RUNNING_PROGRESS, - 20 + Math.min(70, processedRows / 100)); - taskContext.reportProgress(progress, TaskStage.IMPORTING.name(), - String.format("Imported %s rows", successCount)); + @Override + public void close() { + if (batcher != null) { + batcher.close(); + } } } - } diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/excel/CSVImporter.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/excel/CSVImporter.java index bdd2422daf..8a9473c3df 100644 --- a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/excel/CSVImporter.java +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/excel/CSVImporter.java @@ -5,44 +5,88 @@ import ai.chat2db.community.domain.api.model.task.ImportTaskSpec; import ai.chat2db.community.domain.api.service.task.TaskExecutionContext; import ai.chat2db.community.domain.core.impl.db.CsvParser; +import ai.chat2db.community.domain.core.impl.task.imports.BaseImporter; +import ai.chat2db.community.domain.core.impl.task.imports.ImportColumnResolver; +import ai.chat2db.community.domain.core.impl.task.imports.ImportRowBatcher; import ai.chat2db.community.domain.core.impl.task.imports.IImportStrategy; -import com.alibaba.excel.support.ExcelTypeEnum; +import ai.chat2db.spi.sql.Chat2DBContext; import java.nio.file.Path; +import java.util.ArrayList; import java.util.LinkedHashMap; -import java.util.Map; import java.util.List; +import java.util.Map; -public class CSVImporter extends BaseExcelImporter implements IImportStrategy { +/** CSV import driven by the validated {@link CsvOptions} contract and the resumable row batcher. */ +public class CSVImporter extends BaseImporter implements IImportStrategy { @Override - protected void doImportData(ImportTaskSpec spec, TaskExecutionContext context, List columns) { + protected void doImportData(ImportTaskSpec spec, TaskExecutionContext context, + List columns) { CsvOptions options = (spec.getCsvOptions() == null ? CsvOptions.defaults() : spec.getCsvOptions()).validate(); - NoModelDataListener listener = new NoModelDataListener(spec, context, columns); - boolean[] initialized = {false}; + spec.setCsvOptions(options); + int skipRows = spec.getOptions() == null || spec.getOptions().getSkipRows() == null + ? 0 : Math.max(0, spec.getOptions().getSkipRows()); + ImportRowBatcher[] batcher = {null}; int[] sourceRow = {0}; - new CsvParser(options).forEachRow(Path.of(spec.getSourceFile()), row -> { - int rowNumber = ++sourceRow[0]; - if (Boolean.TRUE.equals(options.getHasHeader()) && rowNumber == options.getHeaderRow()) { - initialized[0] = true; - listener.acceptHead(row); - return; + int[] skippedDataRows = {0}; + try { + new CsvParser(options).forEachRow(Path.of(spec.getSourceFile()), row -> { + int rowNumber = ++sourceRow[0]; + if (Boolean.TRUE.equals(options.getHasHeader()) && rowNumber == options.getHeaderRow()) { + batcher[0] = createBatcher(spec, context, columns, values(row)); + return; + } + if (rowNumber < options.getDataStartRow() + || options.getDataEndRow() != null && rowNumber > options.getDataEndRow()) { + return; + } + if (skippedDataRows[0] < skipRows) { + skippedDataRows[0]++; + return; + } + if (batcher[0] == null) { + int width = Math.max(row.size(), mappedSourceColumnCount(spec)); + batcher[0] = createBatcher(spec, context, columns, values(syntheticHeader(width))); + } + batcher[0].accept(rowNumber, values(row)); + }, context::checkCancelled); + if (batcher[0] != null) { + batcher[0].flush(); + context.logInfo("IMPORT_SUMMARY", "CSV import finished", Map.of( + "importedRows", batcher[0].importedRows(), + "rejectedRows", batcher[0].rejectedRows())); } - if (rowNumber < options.getDataStartRow() - || options.getDataEndRow() != null && rowNumber > options.getDataEndRow()) { - return; + } catch (RuntimeException failure) { + if (batcher[0] != null) { + batcher[0].abort(failure); } - if (!initialized[0]) { - initialized[0] = true; - listener.acceptHead(syntheticHeader(Math.max(row.size(), mappedSourceColumnCount(spec)))); + throw failure; + } finally { + if (batcher[0] != null) { + batcher[0].close(); } - listener.acceptRow(row, rowNumber); - }, context::checkCancelled); - if (initialized[0]) { - listener.finish(); } } + private ImportRowBatcher createBatcher(ImportTaskSpec spec, TaskExecutionContext context, + List columns, List headers) { + ImportColumnResolver.Resolution resolution = ImportColumnResolver.resolveForSpec(columns, headers, spec); + reportResolution(context, resolution); + ImportColumnResolver.validateForImport(columns, resolution, spec); + return new ImportRowBatcher(spec, context, resolution, + Chat2DBContext.getDbMetaData().getValueProcessor()); + } + + private static List values(Map row) { + int count = row.keySet().stream().mapToInt(Integer::intValue).max().orElse(-1) + 1; + List values = new ArrayList<>(count); + for (int index = 0; index < count; index++) { + values.add(row.get(index)); + } + return values; + } + private static Map syntheticHeader(int columnCount) { Map header = new LinkedHashMap<>(); for (int index = 0; index < columnCount; index++) { @@ -68,9 +112,4 @@ private static int mappedSourceColumnCount(ImportTaskSpec spec) { .max() .orElse(0); } - - @Override - protected ExcelTypeEnum getExcelType() { - return ExcelTypeEnum.CSV; - } } diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/excel/CsvImportValueNormalizer.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/excel/CsvImportValueNormalizer.java index 1f4ff57b0c..cecc79bb48 100644 --- a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/excel/CsvImportValueNormalizer.java +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/excel/CsvImportValueNormalizer.java @@ -23,7 +23,7 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; -final class CsvImportValueNormalizer { +public final class CsvImportValueNormalizer { private static final Map MONTHS = monthNames(); private static final Set NUMERIC_TYPES = Set.of( @@ -33,7 +33,7 @@ final class CsvImportValueNormalizer { private CsvImportValueNormalizer() { } - static String normalize(String value, TableColumn column, CsvOptions options, long sourceRow) { + public static String normalize(String value, TableColumn column, CsvOptions options, long sourceRow) { if (value == null || column == null) { return value; } diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/AdaptiveBatchSizerTest.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/AdaptiveBatchSizerTest.java new file mode 100644 index 0000000000..59e3b231f1 --- /dev/null +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/AdaptiveBatchSizerTest.java @@ -0,0 +1,64 @@ +package ai.chat2db.community.domain.core.impl.task; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Tuning behaviour of the batch-size observer: cheap batches grow, expensive batches shrink, + * and the size always stays inside its bounds. + */ +class AdaptiveBatchSizerTest { + + private static final long FAST_NANOS = 1L * 1_000_000L; + + private static final long SLOW_NANOS = 100L * 1_000_000L; + + @Test + void clampsInitialValueIntoBounds() { + assertEquals(100, new AdaptiveBatchSizer(1).batchSize()); + assertEquals(500, new AdaptiveBatchSizer(500).batchSize()); + assertEquals(50_000, new AdaptiveBatchSizer(50_000).batchSize()); + assertEquals(100_000, new AdaptiveBatchSizer(200_000).batchSize()); + } + + @Test + void doublesWhileBatchesComeBackCheap() { + AdaptiveBatchSizer sizer = new AdaptiveBatchSizer(500); + for (int round = 0; round < 4; round++) { + sizer.record(500, FAST_NANOS); + } + assertEquals(8_000, sizer.batchSize()); + } + + @Test + void halvesWhileBatchesComeBackExpensive() { + AdaptiveBatchSizer sizer = new AdaptiveBatchSizer(5_000); + sizer.record(5_000, SLOW_NANOS); + assertEquals(2_500, sizer.batchSize()); + sizer.record(2_500, SLOW_NANOS); + assertEquals(1_250, sizer.batchSize()); + } + + @Test + void neverLeavesTheBounds() { + AdaptiveBatchSizer sizer = new AdaptiveBatchSizer(5_000); + for (int round = 0; round < 10; round++) { + sizer.record(5_000, SLOW_NANOS); + } + assertEquals(100, sizer.batchSize()); + for (int round = 0; round < 10; round++) { + sizer.record(100, FAST_NANOS); + } + assertEquals(100_000, sizer.batchSize()); + } + + @Test + void ignoresInvalidObservations() { + AdaptiveBatchSizer sizer = new AdaptiveBatchSizer(500); + sizer.record(0, FAST_NANOS); + sizer.record(500, 0L); + sizer.record(-1, FAST_NANOS); + assertEquals(500, sizer.batchSize()); + } +} diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/AdaptiveConcurrencyGateTest.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/AdaptiveConcurrencyGateTest.java new file mode 100644 index 0000000000..33ed6270ec --- /dev/null +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/AdaptiveConcurrencyGateTest.java @@ -0,0 +1,120 @@ +package ai.chat2db.community.domain.core.impl.task; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * AIMD tuning of the concurrency gate: permits grow on throughput improvement, shrink fast on + * regression, stay hard-capped at the configured max even while workers hold permits, and react + * to source pressure by cutting a quarter of the fan-out. Tuning windows are row-based: one + * observation with at least {@link AdaptiveConcurrencyGate#WINDOW_ROWS} rows triggers one + * evaluation, so successive windows model faster execution with smaller wall times. + */ +class AdaptiveConcurrencyGateTest { + + private static final long MILLI = 1_000_000L; + + /** Feeds one full tuning window executed in {@code millis} (rows are the window size). */ + private void tune(AdaptiveConcurrencyGate gate, long millis) { + gate.record(AdaptiveConcurrencyGate.WINDOW_ROWS, millis * MILLI); + } + + @Test + void growsOnImprovementUpToTheCap() { + AdaptiveConcurrencyGate gate = AdaptiveConcurrencyGate.create(2, 4); + assertEquals(2, gate.currentPermits()); + tune(gate, 10); // first window only establishes the baseline + assertEquals(2, gate.currentPermits()); + tune(gate, 5); + assertEquals(3, gate.currentPermits()); + tune(gate, 2); + assertEquals(4, gate.currentPermits()); + tune(gate, 1); + assertEquals(4, gate.currentPermits(), "permits must never exceed the configured max"); + } + + @Test + void growsFromBelowTheFloor() { + AdaptiveConcurrencyGate gate = AdaptiveConcurrencyGate.create(1, 4); + assertEquals(1, gate.currentPermits()); + tune(gate, 10); // baseline + tune(gate, 5); + assertEquals(2, gate.currentPermits()); + tune(gate, 2); + assertEquals(3, gate.currentPermits()); + tune(gate, 1); + assertEquals(4, gate.currentPermits()); + } + + @Test + void hardCapHoldsEvenWhileWorkersHoldPermits() { + AdaptiveConcurrencyGate gate = AdaptiveConcurrencyGate.create(2, 4); + gate.tryAcquire(); + gate.tryAcquire(); // both initial permits are held by workers now + assertEquals(0, gate.currentPermits()); + tune(gate, 10); // baseline + tune(gate, 5); + tune(gate, 2); + tune(gate, 1); + gate.relinquish(true); + gate.relinquish(true); + // The old available-permits-based guard let the total drift one past the max; the hard + // total cap must keep it at exactly the configured ceiling. + assertEquals(4, gate.currentPermits()); + } + + @Test + void shrinksFastOnRegressionAndStaysAtTheFloor() { + AdaptiveConcurrencyGate gate = AdaptiveConcurrencyGate.create(4, 4); + tune(gate, 2); // baseline (fast windows: high throughput) + tune(gate, 10); // regression: a quarter of 4 permits cut + assertEquals(3, gate.currentPermits()); + tune(gate, 20); // regression: at least one permit cut + assertEquals(2, gate.currentPermits()); + tune(gate, 40); + tune(gate, 80); + tune(gate, 100); + assertEquals(2, gate.currentPermits(), "the fan-out must never drop below the floor"); + } + + @Test + void keepsAStableThroughputFlat() { + AdaptiveConcurrencyGate gate = AdaptiveConcurrencyGate.create(2, 4); + tune(gate, 10); // baseline + tune(gate, 10); // identical throughput: neither grow nor cut + assertEquals(2, gate.currentPermits()); + } + + @Test + void ignoresInvalidObservations() { + AdaptiveConcurrencyGate gate = AdaptiveConcurrencyGate.create(2, 4); + gate.record(0, MILLI); + gate.record(100, 0L); + assertEquals(2, gate.currentPermits()); + } + + @Test + void sourcePressureCutsAQuarterImmediatelyButNeverPastTheFloor() { + AdaptiveConcurrencyGate gate = AdaptiveConcurrencyGate.create(4, 4); + gate.reduceForSourcePressure(); + assertEquals(3, gate.currentPermits(), "a slow page cuts a quarter of the fan-out"); + AdaptiveConcurrencyGate floored = AdaptiveConcurrencyGate.create(2, 4); + floored.reduceForSourcePressure(); + assertEquals(2, floored.currentPermits(), "the floor holds under source pressure"); + } + + @Test + void boundedAdmitDegradesInsteadOfHanging() { + AdaptiveConcurrencyGate gate = AdaptiveConcurrencyGate.create(1, 4); + gate.tryAcquire(); // the only permit is held + assertFalse(gate.admit(1L), "a stuck gate must report failure instead of blocking forever"); + gate.relinquish(false); // no permit taken: a no-op + gate.relinquish(true); + assertTrue(gate.admit(1L), "the returned permit is admitted again"); + gate.relinquish(true); + assertEquals(1, gate.currentPermits()); + } +} diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/CsvImportPipelineTest.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/CsvImportPipelineTest.java new file mode 100644 index 0000000000..c820d99459 --- /dev/null +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/CsvImportPipelineTest.java @@ -0,0 +1,261 @@ +package ai.chat2db.community.domain.core.impl.task; + +import ai.chat2db.community.domain.api.config.DBConfig; +import ai.chat2db.community.domain.api.config.DriverConfig; +import ai.chat2db.community.domain.api.model.PageResponse; +import ai.chat2db.community.domain.api.model.task.ImportColumnMapping; +import ai.chat2db.community.domain.api.model.task.ImportOptions; +import ai.chat2db.community.domain.api.model.task.ImportTaskSpec; +import ai.chat2db.community.domain.api.model.task.ResumeState; +import ai.chat2db.community.domain.api.model.task.Task; +import ai.chat2db.community.domain.api.model.task.TaskArtifact; +import ai.chat2db.community.domain.api.model.task.TaskEvent; +import ai.chat2db.community.domain.api.model.task.TaskProgress; +import ai.chat2db.community.domain.api.model.task.TaskQuery; +import ai.chat2db.community.domain.api.model.task.TaskStage; +import ai.chat2db.community.domain.api.model.task.TaskStatus; +import ai.chat2db.community.domain.api.model.task.TaskStatusPatch; +import ai.chat2db.community.domain.api.model.task.TaskTargetSnapshot; +import ai.chat2db.community.domain.api.service.task.TaskStorage; +import ai.chat2db.community.domain.core.impl.task.imports.excel.CSVImporter; +import ai.chat2db.spi.DefaultMetaService; +import ai.chat2db.spi.IDbMetaData; +import ai.chat2db.spi.IPlugin; +import ai.chat2db.spi.model.datasource.ConnectInfo; +import ai.chat2db.spi.sql.Chat2DBContext; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * The CSV import path end to end: commons-csv grammar, explicit column mapping, error tolerance + * with a REJECT sub-artifact, and true batched inserts against the target table. + */ +class CsvImportPipelineTest { + + private static final String DB_TYPE = "CSV_IMPORT_TEST"; + + @TempDir + Path tempDirectory; + + private java.sql.Connection connection; + private IPlugin previousPlugin; + private InMemoryTaskStorage storage; + + @BeforeEach + void setUp() throws Exception { + DBConfig config = new DBConfig(); + config.setDbType(DB_TYPE); + config.setDefaultDriverConfig(new DriverConfig()); + previousPlugin = Chat2DBContext.PLUGIN_MAP.put(DB_TYPE, new IPlugin() { + @Override + public DBConfig getDBConfig() { + return config; + } + + @Override + public IDbMetaData getDbMetaData() { + return new DefaultMetaService(); + } + }); + connection = DriverManager.getConnection("jdbc:h2:mem:csv_import"); + try (Statement statement = connection.createStatement()) { + statement.execute("CREATE TABLE TARGET_ROWS (ID INT PRIMARY KEY, NAME VARCHAR(10))"); + } + ConnectInfo connectInfo = new ConnectInfo(); + connectInfo.setDbType(DB_TYPE); + connectInfo.setDriverConfig(new DriverConfig()); + connectInfo.setConnection(connection); + Chat2DBContext.putContext(connectInfo); + storage = new InMemoryTaskStorage(); + } + + @AfterEach + void tearDown() throws Exception { + Chat2DBContext.removeContext(); + if (previousPlugin == null) { + Chat2DBContext.PLUGIN_MAP.remove(DB_TYPE); + } else { + Chat2DBContext.PLUGIN_MAP.put(DB_TYPE, previousPlugin); + } + connection.close(); + } + + @Test + void skipsBadRowsIntoRejectArtifactAndMapsColumnsExplicitly() throws Exception { + Path csv = tempDirectory.resolve("input.csv"); + Files.writeString(csv, "ROW_ID,ROW_NAME,EXTRA\n1,ok,ignored\n2,this-value-is-too-long,x\n", + StandardCharsets.UTF_8); + + ImportTaskSpec spec = ImportTaskSpec.builder() + .taskType("DATA_FILE_IMPORT") + .sourceFile(csv.toString()) + .format("CSV") + .target(TaskTargetSnapshot.builder().dataSourceId(1L).tableName("TARGET_ROWS").build()) + .options(ImportOptions.builder() + .charset("UTF-8") + .delimiter(",") + .onError("SKIP") + .maxErrors(5) + .columnMappings(List.of( + new ImportColumnMapping("ROW_ID", "ID"), + new ImportColumnMapping("ROW_NAME", "NAME"))) + .build()) + .build(); + + Long taskId = storage.create(Task.builder().type("DATA_FILE_IMPORT").name("import") + .target(spec.getTarget()).build(), TaskEvent.builder() + .level("INFO").code("TASK_CREATED").message("created").build()).getId(); + TaskExecutionContextImpl context = new TaskExecutionContextImpl(taskId, new RunningTask(taskId), + storage, new ArtifactServiceImpl()); + + new CSVImporter().run(spec, context); + + try (Statement statement = connection.createStatement(); + ResultSet rows = statement.executeQuery("SELECT ID FROM TARGET_ROWS ORDER BY ID")) { + List ids = new ArrayList<>(); + while (rows.next()) { + ids.add(rows.getInt(1)); + } + assertEquals(List.of(1), ids, "the over-long value row must be rejected, not truncated"); + } + + List rejectDrafts; + try (var files = Files.list(tempDirectory)) { + rejectDrafts = files.filter(path -> path.getFileName().toString().contains("rejects.ndjson")) + .toList(); + } + assertEquals(1, rejectDrafts.size()); + String rejects = Files.readString(rejectDrafts.get(0), StandardCharsets.UTF_8); + assertTrue(rejects.contains("this-value-is-too-long"), rejects); + assertTrue(rejects.contains("\"row\":3"), rejects); + + List codes = storage.listEvents(taskId, 0L, 100).stream().map(TaskEvent::getCode).toList(); + assertTrue(codes.contains("IMPORT_COLUMN_MAPPING"), "unmatched EXTRA column reported: " + codes); + assertTrue(codes.contains("IMPORT_ROW_REJECTED"), codes.toString()); + assertTrue(codes.contains("IMPORT_SUMMARY"), codes.toString()); + } + + /** + * Task storage good enough for the import pipeline: the interesting behaviour is the events + * and the reject artifact it records. + */ + private static final class InMemoryTaskStorage implements TaskStorage { + + private final List tasks = new ArrayList<>(); + private final List events = new ArrayList<>(); + private final List artifacts = new ArrayList<>(); + private final List states = new ArrayList<>(); + private long sequence; + + @Override + public Task create(Task task, TaskEvent createdEvent) { + task.setId(1L); + task.setStatus(TaskStatus.PENDING.name()); + tasks.add(task); + createdEvent.setTaskId(task.getId()); + appendEvent(createdEvent); + return task; + } + + @Override + public Optional get(Long taskId) { + return tasks.stream().filter(task -> task.getId().equals(taskId)).findFirst(); + } + + @Override + public PageResponse list(TaskQuery query) { + return PageResponse.of(tasks, (long) tasks.size(), 1, 20); + } + + @Override + public boolean compareAndSetStatus(Long taskId, String expectedStatus, String targetStatus, + TaskStatusPatch patch, TaskEvent lifecycleEvent) { + return false; + } + + @Override + public boolean updateProgressIfRunning(Long taskId, TaskProgress progress) { + return true; + } + + @Override + public TaskEvent appendEvent(TaskEvent event) { + event.setSequence(++sequence); + events.add(event); + return event; + } + + @Override + public List listEvents(Long taskId, long afterSequence, int limit) { + return events.stream().filter(event -> event.getSequence() > afterSequence).limit(limit).toList(); + } + + @Override + public List listEventsBefore(Long taskId, Long beforeSequence, int limit) { + return events.stream().filter(event -> beforeSequence == null || event.getSequence() < beforeSequence) + .toList(); + } + + @Override + public List listNonTerminalTasks() { + return List.of(); + } + + @Override + public boolean deleteTerminalTask(Long taskId, Runnable commitAction) { + return false; + } + + @Override + public List listArtifacts(Long taskId) { + return List.copyOf(artifacts); + } + + @Override + public void saveArtifact(Long taskId, TaskArtifact artifact) { + artifacts.add(artifact); + } + + @Override + public void deleteArtifact(Long taskId, String artifactId) { + artifacts.removeIf(artifact -> artifact.getArtifactId().equals(artifactId)); + } + + @Override + public List listResumableTasks() { + return List.of(); + } + + @Override + public void saveResumeState(Long taskId, ResumeState state) { + states.add(state); + } + + @Override + public List listResumeStates(Long taskId) { + return List.copyOf(states); + } + + @Override + public void clearResumeStates(Long taskId) { + states.clear(); + } + + } +} diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/ImportResumeRoundTripTest.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/ImportResumeRoundTripTest.java new file mode 100644 index 0000000000..7f8c30ee1e --- /dev/null +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/ImportResumeRoundTripTest.java @@ -0,0 +1,339 @@ +package ai.chat2db.community.domain.core.impl.task; + +import ai.chat2db.community.domain.api.config.DBConfig; +import ai.chat2db.community.domain.api.config.DriverConfig; +import ai.chat2db.community.domain.api.model.PageResponse; +import ai.chat2db.community.domain.api.model.task.ImportColumnMapping; +import ai.chat2db.community.domain.api.model.task.ImportOptions; +import ai.chat2db.community.domain.api.model.task.ImportTaskSpec; +import ai.chat2db.community.domain.api.model.task.ResumeState; +import ai.chat2db.community.domain.api.model.task.Task; +import ai.chat2db.community.domain.api.model.task.TaskArtifact; +import ai.chat2db.community.domain.api.model.task.TaskEvent; +import ai.chat2db.community.domain.api.model.task.TaskExecutionException; +import ai.chat2db.community.domain.api.model.task.TaskProgress; +import ai.chat2db.community.domain.api.model.task.TaskQuery; +import ai.chat2db.community.domain.api.model.task.TaskStatusPatch; +import ai.chat2db.community.domain.api.model.task.TaskTargetSnapshot; +import ai.chat2db.community.domain.api.service.task.TaskStorage; +import ai.chat2db.community.domain.core.impl.task.imports.excel.CSVImporter; +import ai.chat2db.community.domain.core.impl.task.imports.ImportColumnResolver; +import ai.chat2db.community.tools.constant.JdbcDriverConstants; +import ai.chat2db.spi.DefaultMetaService; +import ai.chat2db.spi.IDbMetaData; +import ai.chat2db.spi.IPlugin; +import ai.chat2db.spi.model.datasource.ConnectInfo; +import ai.chat2db.spi.sql.Chat2DBContext; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * The three-layer import resume, end to end: a FAIL_FAST run aborted mid-file leaves durable + * watermarks in the journal (Layer 1) and task storage (Layer 2); a resumed SKIP run must skip + * exactly those rows, finish the remainder, and land every id exactly once — no loss, no + * duplicates. A clean run removes its Layer-3 journal entirely. + */ +class ImportResumeRoundTripTest { + + private static final String DB_TYPE = "IMPORT_RESUME_TEST"; + private static final String PARALLELISM_PROPERTY = "chat2db.task.import.parallelism"; + private static final String JOURNAL_INTERVAL_PROPERTY = "chat2db.task.import.journal-interval"; + private static final String CHECKPOINT_INTERVAL_PROPERTY = "chat2db.task.import.checkpoint-interval"; + private static final String SNAPSHOT_INTERVAL_PROPERTY = "chat2db.task.import.snapshot-interval"; + private static final String H2_DRIVER_NAME = "import-resume-h2.jar"; + private static final int ROWS = 5000; + private static final int POISON_ID = 1500; + + private static String previousUserHome; + + @TempDir + Path tempDirectory; + + private Connection connection; + private IPlugin previousPlugin; + private RecordingStorage storage; + + @BeforeAll + static void isolateHomeAndSeedDriver() throws Exception { + previousUserHome = System.getProperty("user.home"); + File tempHome = Files.createTempDirectory("chat2db-import-resume-home").toFile(); + System.setProperty("user.home", tempHome.getAbsolutePath()); + File libDir = new File(JdbcDriverConstants.DRIVER_LIB_PATH); + libDir.mkdirs(); + File h2Jar = new File(org.h2.Driver.class.getProtectionDomain().getCodeSource().getLocation().toURI()); + Files.copy(h2Jar.toPath(), new File(libDir, H2_DRIVER_NAME).toPath(), + StandardCopyOption.REPLACE_EXISTING); + } + + @AfterAll + static void restoreHome() { + System.setProperty("user.home", previousUserHome); + } + + @BeforeEach + void setUp() throws Exception { + System.clearProperty(PARALLELISM_PROPERTY); + System.setProperty(JOURNAL_INTERVAL_PROPERTY, "1"); + System.setProperty(CHECKPOINT_INTERVAL_PROPERTY, "1"); + System.setProperty(SNAPSHOT_INTERVAL_PROPERTY, "8"); + DBConfig config = new DBConfig(); + config.setDbType(DB_TYPE); + config.setDefaultDriverConfig(new DriverConfig()); + previousPlugin = Chat2DBContext.PLUGIN_MAP.put(DB_TYPE, new IPlugin() { + @Override + public DBConfig getDBConfig() { + return config; + } + + @Override + public IDbMetaData getDbMetaData() { + return new DefaultMetaService(); + } + }); + connection = DriverManager.getConnection("jdbc:h2:mem:resume_rt"); + try (Statement statement = connection.createStatement()) { + statement.execute("CREATE TABLE BULK_ROWS (ID INT PRIMARY KEY, NAME VARCHAR(50))"); + // The poison row duplicates CSV id 1500: a FAIL_FAST run aborts inside the batch that + // contains it, after the first committed batch has been checkpointed. + statement.execute("INSERT INTO BULK_ROWS VALUES (" + POISON_ID + ", 'poison')"); + } + ConnectInfo connectInfo = new ConnectInfo(); + connectInfo.setDbType(DB_TYPE); + DriverConfig driverConfig = new DriverConfig(); + driverConfig.setJdbcDriverClass("org.h2.Driver"); + driverConfig.setJdbcDriver(H2_DRIVER_NAME); + connectInfo.setDriverConfig(driverConfig); + connectInfo.setUrl("jdbc:h2:mem:resume_rt"); + connectInfo.setConnection(connection); + Chat2DBContext.putContext(connectInfo); + storage = new RecordingStorage(); + } + + @AfterEach + void tearDown() throws Exception { + Chat2DBContext.removeContext(); + if (previousPlugin == null) { + Chat2DBContext.PLUGIN_MAP.remove(DB_TYPE); + } else { + Chat2DBContext.PLUGIN_MAP.put(DB_TYPE, previousPlugin); + } + System.clearProperty(PARALLELISM_PROPERTY); + System.clearProperty(JOURNAL_INTERVAL_PROPERTY); + System.clearProperty(CHECKPOINT_INTERVAL_PROPERTY); + System.clearProperty(SNAPSHOT_INTERVAL_PROPERTY); + connection.close(); + } + + @Test + void resumeContinuesAfterMidImportFailureWithoutDuplicates() throws Exception { + Path csv = writeCsv(); + + // Run 1 (FAIL_FAST): aborts inside the batch holding the poison row. + assertThrows(TaskExecutionException.class, + () -> new CSVImporter().run(csvSpec(csv, "FAIL_FAST"), contextFor()), + "the poison row must abort a FAIL_FAST import"); + long watermarkRows = storage.resumeStates.stream() + .filter(state -> state.getRowsDone() != null) + .mapToLong(ResumeState::getRowsDone) + .max().orElse(0L); + assertTrue(watermarkRows > 0, "a durable checkpoint must have survived the abort"); + assertTrue(watermarkRows < ROWS, "the abort must leave a partial watermark"); + // Layer-1/3 journaling is best-effort by design (its directory presence depends on the + // hosting state path); the Layer-2 storage watermark above carries the resume guarantee + // and the journal file semantics are covered by TaskResumeJournalTest. + assertEquals(1, countIds(POISON_ID), "only the pre-inserted poison row exists so far"); + + // Run 2 (SKIP): resumes below the watermark, rejects the poison duplicate, finishes. + new CSVImporter().run(csvSpec(csv, "SKIP"), contextFor()); + + assertEquals(ROWS, countRows(), "every id must be present exactly once after the resume"); + assertEquals(ROWS, countDistinctIds(), "the resume must not duplicate durable rows"); + assertEquals(1, countIds(POISON_ID)); + } + + private Path writeCsv() throws Exception { + Path csv = tempDirectory.resolve("resume.csv"); + StringBuilder content = new StringBuilder("ID,NAME\n"); + for (int id = 1; id <= ROWS; id++) { + content.append(id).append(",name-").append(id).append('\n'); + } + Files.writeString(csv, content.toString(), StandardCharsets.UTF_8); + return csv; + } + + private ImportTaskSpec csvSpec(Path csv, String onError) { + return ImportTaskSpec.builder() + .taskType("DATA_FILE_IMPORT") + .sourceFile(csv.toString()) + .format("CSV") + .target(TaskTargetSnapshot.builder().dataSourceId(1L).tableName("BULK_ROWS").build()) + .options(ImportOptions.builder() + .charset("UTF-8") + .delimiter(",") + .onError(onError) + .maxErrors(100) + .columnMappings(List.of( + new ImportColumnMapping("ID", "ID"), + new ImportColumnMapping("NAME", "NAME"))) + .build()) + .build(); + } + + private TaskExecutionContextImpl contextFor() { + Long taskId = storage.create(Task.builder().type("DATA_FILE_IMPORT").name("resume") + .target(TaskTargetSnapshot.builder().dataSourceId(1L).build()).build(), + TaskEvent.builder().level("INFO").code("TASK_CREATED").message("created").build()).getId(); + return new TaskExecutionContextImpl(taskId, new RunningTask(taskId), storage, new ArtifactServiceImpl()); + } + + private int countRows() throws Exception { + try (Statement statement = connection.createStatement(); + ResultSet rows = statement.executeQuery("SELECT COUNT(*) FROM BULK_ROWS")) { + rows.next(); + return rows.getInt(1); + } + } + + private int countDistinctIds() throws Exception { + try (Statement statement = connection.createStatement(); + ResultSet rows = statement.executeQuery("SELECT COUNT(DISTINCT ID) FROM BULK_ROWS")) { + rows.next(); + return rows.getInt(1); + } + } + + private int countIds(int id) throws Exception { + try (Statement statement = connection.createStatement(); + ResultSet rows = statement.executeQuery("SELECT COUNT(*) FROM BULK_ROWS WHERE ID = " + id)) { + rows.next(); + return rows.getInt(1); + } + } + + /** Task storage that records the resume states the batcher checkpoints. */ + private static final class RecordingStorage implements TaskStorage { + + private final List tasks = new ArrayList<>(); + private final List events = new ArrayList<>(); + private final List artifacts = new ArrayList<>(); + private final List resumeStates = new ArrayList<>(); + private long sequence; + + @Override + public Task create(Task task, TaskEvent createdEvent) { + task.setId(1L); + task.setStatus("PENDING"); + tasks.add(task); + createdEvent.setTaskId(task.getId()); + appendEvent(createdEvent); + return task; + } + + @Override + public Optional get(Long taskId) { + return tasks.stream().filter(task -> task.getId().equals(taskId)).findFirst(); + } + + @Override + public PageResponse list(TaskQuery query) { + return PageResponse.of(tasks, (long) tasks.size(), 1, 20); + } + + @Override + public boolean compareAndSetStatus(Long taskId, String expectedStatus, String targetStatus, + TaskStatusPatch patch, TaskEvent lifecycleEvent) { + return true; + } + + @Override + public boolean updateProgressIfRunning(Long taskId, TaskProgress progress) { + return true; + } + + @Override + public TaskEvent appendEvent(TaskEvent event) { + event.setSequence(++sequence); + events.add(event); + return event; + } + + @Override + public List listEvents(Long taskId, long afterSequence, int limit) { + return events.stream().filter(event -> event.getSequence() > afterSequence).limit(limit).toList(); + } + + @Override + public List listEventsBefore(Long taskId, Long beforeSequence, int limit) { + return events.stream() + .filter(event -> beforeSequence == null || event.getSequence() < beforeSequence) + .toList(); + } + + @Override + public List listNonTerminalTasks() { + return List.of(); + } + + @Override + public boolean deleteTerminalTask(Long taskId, Runnable commitAction) { + return false; + } + + @Override + public List listArtifacts(Long taskId) { + return List.copyOf(artifacts); + } + + @Override + public void saveArtifact(Long taskId, TaskArtifact artifact) { + artifacts.add(artifact); + } + + @Override + public void deleteArtifact(Long taskId, String artifactId) { + artifacts.removeIf(artifact -> artifact.getArtifactId().equals(artifactId)); + } + + @Override + public List listResumableTasks() { + return List.of(); + } + + @Override + public void saveResumeState(Long taskId, ResumeState state) { + resumeStates.add(state); + } + + @Override + public List listResumeStates(Long taskId) { + return List.copyOf(resumeStates); + } + + @Override + public void clearResumeStates(Long taskId) { + resumeStates.clear(); + } + } +} diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/ImportRowBatcherParallelTest.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/ImportRowBatcherParallelTest.java new file mode 100644 index 0000000000..0d4e5a6ed6 --- /dev/null +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/ImportRowBatcherParallelTest.java @@ -0,0 +1,351 @@ +package ai.chat2db.community.domain.core.impl.task; + +import ai.chat2db.community.domain.api.config.DBConfig; +import ai.chat2db.community.domain.api.config.DriverConfig; +import ai.chat2db.community.domain.api.model.PageResponse; +import ai.chat2db.community.domain.api.model.task.ImportColumnMapping; +import ai.chat2db.community.domain.api.model.task.ImportOptions; +import ai.chat2db.community.domain.api.model.task.ImportTaskSpec; +import ai.chat2db.community.domain.api.model.task.ResumeState; +import ai.chat2db.community.domain.api.model.task.Task; +import ai.chat2db.community.domain.api.model.task.TaskArtifact; +import ai.chat2db.community.domain.api.model.task.TaskEvent; +import ai.chat2db.community.domain.api.model.task.TaskQuery; +import ai.chat2db.community.domain.api.model.task.TaskStatusPatch; +import ai.chat2db.community.domain.api.model.task.TaskTargetSnapshot; +import ai.chat2db.community.domain.api.service.task.TaskStorage; +import ai.chat2db.community.domain.api.model.task.TaskProgress; +import ai.chat2db.community.domain.core.impl.task.imports.excel.CSVImporter; +import ai.chat2db.community.domain.core.impl.task.imports.ImportRowBatcher; +import ai.chat2db.community.domain.api.model.task.TaskStage; +import ai.chat2db.community.domain.api.model.task.TaskStatus; +import ai.chat2db.community.tools.constant.JdbcDriverConstants; +import ai.chat2db.spi.DefaultMetaService; +import ai.chat2db.spi.IDbMetaData; +import ai.chat2db.spi.IPlugin; +import ai.chat2db.spi.model.datasource.ConnectInfo; +import ai.chat2db.spi.sql.Chat2DBContext; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * The parallel import path end to end: multiple workers, each on its own dedicated connection, + * execute partitioned batches while the caller keeps producing. Verifies that parallel workers + * insert every row exactly once and that SKIP replay distinguishes bad data from connection + * failures. + */ +class ImportRowBatcherParallelTest { + + private static final String DB_TYPE = "PARALLEL_IMPORT_TEST"; + + private static final String PARALLELISM_PROPERTY = "chat2db.task.import.parallelism"; + + private static final String H2_DRIVER_NAME = "parallel-import-test-h2.jar"; + + private static String previousUserHome; + + @TempDir + Path tempDirectory; + + private java.sql.Connection connection; + private IPlugin previousPlugin; + private InMemoryTaskStorage storage; + private String previousParallelism; + + @BeforeAll + static void isolateHomeAndSeedDriver() throws Exception { + // JdbcJarUtils resolves driver names against the driver library under user.home and + // cannot load an absolute jar path, so seed a copy of the H2 jar and isolate the home + // directory exactly like ShardedKeysetExportTest does. + previousUserHome = System.getProperty("user.home"); + File tempHome = Files.createTempDirectory("chat2db-parallel-import-home").toFile(); + System.setProperty("user.home", tempHome.getAbsolutePath()); + File libDir = new File(JdbcDriverConstants.DRIVER_LIB_PATH); + libDir.mkdirs(); + File h2Jar = new File(org.h2.Driver.class.getProtectionDomain().getCodeSource().getLocation().toURI()); + Files.copy(h2Jar.toPath(), new File(libDir, H2_DRIVER_NAME).toPath(), + StandardCopyOption.REPLACE_EXISTING); + } + + @AfterAll + static void restoreHome() { + System.setProperty("user.home", previousUserHome); + } + + @BeforeEach + void setUp() throws Exception { + previousParallelism = System.clearProperty(PARALLELISM_PROPERTY); + DBConfig config = new DBConfig(); + config.setDbType(DB_TYPE); + config.setDefaultDriverConfig(new DriverConfig()); + previousPlugin = Chat2DBContext.PLUGIN_MAP.put(DB_TYPE, new IPlugin() { + @Override + public DBConfig getDBConfig() { + return config; + } + + @Override + public IDbMetaData getDbMetaData() { + return new DefaultMetaService(); + } + }); + connection = DriverManager.getConnection("jdbc:h2:mem:parallel_import"); + try (Statement statement = connection.createStatement()) { + statement.execute("CREATE TABLE BULK_ROWS (ID INT PRIMARY KEY, NAME VARCHAR(50))"); + } + ConnectInfo connectInfo = new ConnectInfo(); + connectInfo.setDbType(DB_TYPE); + // A real URL plus driver config so parallel workers can build their own dedicated + // connections through ConnectionPool, exactly like the production call path. + connectInfo.setUrl("jdbc:h2:mem:parallel_import"); + connectInfo.setDriverConfig(h2DriverConfig()); + connectInfo.setConnection(connection); + Chat2DBContext.putContext(connectInfo); + storage = new InMemoryTaskStorage(); + } + + @AfterEach + void tearDown() throws Exception { + Chat2DBContext.removeContext(); + if (previousPlugin == null) { + Chat2DBContext.PLUGIN_MAP.remove(DB_TYPE); + } else { + Chat2DBContext.PLUGIN_MAP.put(DB_TYPE, previousPlugin); + } + if (previousParallelism == null) { + System.clearProperty(PARALLELISM_PROPERTY); + } else { + System.setProperty(PARALLELISM_PROPERTY, previousParallelism); + } + connection.close(); + } + + private static DriverConfig h2DriverConfig() { + DriverConfig driverConfig = new DriverConfig(); + driverConfig.setJdbcDriver(H2_DRIVER_NAME); + driverConfig.setJdbcDriverClass("org.h2.Driver"); + return driverConfig; + } + + private TaskExecutionContextImpl contextFor(ImportTaskSpec spec) { + Long taskId = storage.create(Task.builder().type("DATA_FILE_IMPORT").name("import") + .target(spec.getTarget()).build(), TaskEvent.builder() + .level("INFO").code("TASK_CREATED").message("created").build()).getId(); + return new TaskExecutionContextImpl(taskId, new RunningTask(taskId), + storage, new ArtifactServiceImpl()); + } + + private ImportTaskSpec csvSpec(Path csv, String onError) { + return ImportTaskSpec.builder() + .taskType("DATA_FILE_IMPORT") + .sourceFile(csv.toString()) + .importFileId("parallel-import-test-source") + .format("CSV") + .target(TaskTargetSnapshot.builder().dataSourceId(1L).tableName("BULK_ROWS").build()) + .mode("ULTRA_FAST") + .confirmedNoStrongRelations(true) + .options(ImportOptions.builder() + .charset("UTF-8") + .delimiter(",") + .onError(onError) + .maxErrors(1000) + .columnMappings(List.of( + new ImportColumnMapping("ID", "ID"), + new ImportColumnMapping("NAME", "NAME"))) + .build()) + .build(); + } + + private Path writeCsv(String... lines) throws Exception { + Path csv = tempDirectory.resolve("bulk.csv"); + StringBuilder content = new StringBuilder("ID,NAME\n"); + for (String line : lines) { + content.append(line).append('\n'); + } + Files.writeString(csv, content.toString(), StandardCharsets.UTF_8); + return csv; + } + + private List importedIds() throws Exception { + try (Statement statement = connection.createStatement(); + ResultSet rows = statement.executeQuery("SELECT ID FROM BULK_ROWS ORDER BY ID")) { + List ids = new ArrayList<>(); + while (rows.next()) { + ids.add(rows.getInt(1)); + } + return ids; + } + } + + @Test + void parallelWorkersInsertEveryRowExactlyOnce() throws Exception { + System.setProperty(PARALLELISM_PROPERTY, "4"); + int rows = 2000; + String[] lines = new String[rows]; + for (int index = 0; index < rows; index++) { + lines[index] = (index + 1) + ",name-" + (index + 1); + } + Path csv = writeCsv(lines); + ImportTaskSpec spec = csvSpec(csv, "FAIL_FAST"); + + new CSVImporter().run(spec, contextFor(spec)); + + List ids = importedIds(); + assertEquals(rows, ids.size(), "parallel import must not lose or duplicate rows"); + assertEquals(1, ids.get(0)); + assertEquals(rows, ids.get(rows - 1)); + assertTrue(ImportRowBatcher.lastTuningSnapshot().peakInFlightBatches() > 1, + "the producer must have more than one submitted batch in flight"); + } + + @Test + void skipReplayRejectsConsecutiveBadRowsWithoutAbortingHealthyRows() throws Exception { + System.setProperty(PARALLELISM_PROPERTY, "2"); + Path csv = writeCsv("1,ok", "1,dup-a", "1,dup-b", "1,dup-c", "2,ok"); + ImportTaskSpec spec = csvSpec(csv, "SKIP"); + + new CSVImporter().run(spec, contextFor(spec)); + + List ids = importedIds(); + assertEquals(List.of(1, 2), ids, + "adjacent constraint violations are rejected without hiding healthy rows"); + } + + @Test + void isolatedBadRowsAreStillSkippedWhenSurroundedBySuccessfulRows() throws Exception { + System.setProperty(PARALLELISM_PROPERTY, "2"); + Path csv = writeCsv("1,ok", "1,isolated-dup", "2,ok", "3,ok"); + ImportTaskSpec spec = csvSpec(csv, "SKIP"); + + new CSVImporter().run(spec, contextFor(spec)); + + List ids = importedIds(); + assertEquals(List.of(1, 2, 3), ids, "the isolated duplicate row is rejected, others imported"); + } + + /** + * Task storage good enough for the import pipeline; mirrors the stub used by + * {@code CsvImportPipelineTest}. + */ + private static final class InMemoryTaskStorage implements TaskStorage { + + private final List tasks = new ArrayList<>(); + private final List events = new ArrayList<>(); + private final List artifacts = new ArrayList<>(); + private final List states = new ArrayList<>(); + private long sequence; + + @Override + public Task create(Task task, TaskEvent createdEvent) { + task.setId(1L); + task.setStatus(TaskStatus.PENDING.name()); + tasks.add(task); + createdEvent.setTaskId(task.getId()); + appendEvent(createdEvent); + return task; + } + + @Override + public Optional get(Long taskId) { + return tasks.stream().filter(task -> task.getId().equals(taskId)).findFirst(); + } + + @Override + public PageResponse list(TaskQuery query) { + return PageResponse.of(tasks, (long) tasks.size(), 1, 20); + } + + @Override + public boolean compareAndSetStatus(Long taskId, String expectedStatus, String targetStatus, + TaskStatusPatch patch, TaskEvent lifecycleEvent) { + return true; + } + + @Override + public boolean updateProgressIfRunning(Long taskId, TaskProgress progress) { + return true; + } + + @Override + public TaskEvent appendEvent(TaskEvent event) { + event.setSequence(++sequence); + events.add(event); + return event; + } + + @Override + public List listEvents(Long taskId, long afterSequence, int limit) { + return events.stream().filter(event -> event.getSequence() > afterSequence).limit(limit).toList(); + } + + @Override + public List listEventsBefore(Long taskId, Long beforeSequence, int limit) { + return events.stream().filter(event -> beforeSequence == null || event.getSequence() < beforeSequence) + .toList(); + } + + @Override + public List listNonTerminalTasks() { + return List.of(); + } + + @Override + public boolean deleteTerminalTask(Long taskId, Runnable commitAction) { + return false; + } + + @Override + public List listArtifacts(Long taskId) { + return List.copyOf(artifacts); + } + + @Override + public void saveArtifact(Long taskId, TaskArtifact artifact) { + artifacts.add(artifact); + } + + @Override + public void deleteArtifact(Long taskId, String artifactId) { + artifacts.removeIf(artifact -> artifact.getArtifactId().equals(artifactId)); + } + + @Override + public List listResumableTasks() { + return List.of(); + } + + @Override + public void saveResumeState(Long taskId, ResumeState state) { + states.add(state); + } + + @Override + public List listResumeStates(Long taskId) { + return List.copyOf(states); + } + + @Override + public void clearResumeStates(Long taskId) { + states.clear(); + } + } +} diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/TaskResumeJournalTest.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/TaskResumeJournalTest.java new file mode 100644 index 0000000000..a51497513f --- /dev/null +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/TaskResumeJournalTest.java @@ -0,0 +1,113 @@ +package ai.chat2db.community.domain.core.impl.task; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Map; +import java.util.Optional; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * The generational resume journal: snapshots are checksummed and atomically rotated, the committed + * pointer stays one commit behind the newest generation, torn files fail validation, and recovery + * falls back down the chain instead of trusting a damaged candidate. + */ +class TaskResumeJournalTest { + + @TempDir + Path tempDir; + + private File dir() { + return tempDir.resolve("journal").toFile(); + } + + @Test + void snapshotRoundTripsThroughRecovery() { + TaskResumeJournal journal = TaskResumeJournal.openDirectory(dir(), Map.of()); + journal.snapshot(100); + Optional newest = TaskResumeJournal.recoverNewest(dir()); + assertTrue(newest.isPresent()); + assertEquals(100, newest.get().rowsDone()); + assertEquals(1, newest.get().seq()); + } + + @Test + void committedFallbackStaysOneCommitBehind() { + TaskResumeJournal journal = TaskResumeJournal.openDirectory(dir(), Map.of()); + journal.snapshot(100); + journal.snapshot(200); + assertEquals(200, TaskResumeJournal.recoverNewest(dir()).orElseThrow().rowsDone()); + assertEquals(100, TaskResumeJournal.recoverCommitted(dir()).orElseThrow().rowsDone(), + "the fallback must be the state before the last successful commit"); + } + + @Test + void tornNewestGenerationFallsBackToCommitted() throws Exception { + TaskResumeJournal journal = TaskResumeJournal.openDirectory(dir(), Map.of()); + journal.snapshot(100); + journal.snapshot(200); + Files.writeString(dir().toPath().resolve("gen-2.json"), "{torn", StandardCharsets.UTF_8); + assertEquals(100, TaskResumeJournal.recoverNewest(dir()).orElseThrow().rowsDone(), + "a checksum-invalid generation must be skipped, not trusted"); + } + + @Test + void journalTailCarriesTheFreshestWatermark() { + TaskResumeJournal journal = TaskResumeJournal.openDirectory(dir(), Map.of()); + journal.progress("IMPORTING", 120); + journal.progress("IMPORTING", 150); + assertEquals(150, TaskResumeJournal.recoverTail(dir()).orElseThrow().rowsDone()); + } + + @Test + void tornTailRecordIsIgnored() throws Exception { + TaskResumeJournal journal = TaskResumeJournal.openDirectory(dir(), Map.of()); + journal.progress("IMPORTING", 150); + Files.writeString(dir().toPath().resolve("progress.ndjson"), + "{\"kind\":\"progress\",\"rowsDone\":999,\"ts\":1,\"checksum\":\"deadbeef\"}\n", + StandardCharsets.UTF_8, java.nio.file.StandardOpenOption.APPEND); + assertEquals(150, TaskResumeJournal.recoverTail(dir()).orElseThrow().rowsDone(), + "a checksum-invalid tail record must be ignored"); + } + + @Test + void staleIdentityIsCarriedForCallerSideValidation() { + Map identity = Map.of("sourceLength", 10L, "sourceLastModified", 20L); + TaskResumeJournal journal = TaskResumeJournal.openDirectory(dir(), identity); + journal.snapshot(100); + TaskResumeJournal.Snapshot snapshot = TaskResumeJournal.recoverNewest(dir()).orElseThrow(); + // fastjson2 renders small JSON numbers as Integer; compare numerically + assertEquals(10L, ((Number) snapshot.identity().get("sourceLength")).longValue()); + assertEquals(20L, ((Number) snapshot.identity().get("sourceLastModified")).longValue()); + } + + @Test + void cleanupRemovesEverything() { + TaskResumeJournal journal = TaskResumeJournal.openDirectory(dir(), Map.of()); + journal.snapshot(100); + assertTrue(dir().exists()); + journal.cleanup(); + assertFalse(dir().exists(), "a clean run leaves no journal behind"); + assertTrue(TaskResumeJournal.recoverNewest(dir()).isEmpty()); + } + + @Test + void preserveClosesWriterWithoutDeletingRecoveryState() throws Exception { + TaskResumeJournal journal = TaskResumeJournal.openDirectory(dir(), Map.of()); + journal.progress("IMPORTING", 75); + + journal.preserve(); + + assertTrue(dir().exists()); + assertEquals(75, TaskResumeJournal.recoverTail(dir()).orElseThrow().rowsDone()); + Files.move(dir().toPath().resolve("progress.ndjson"), + dir().toPath().resolve("progress.closed.ndjson")); + } +} diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/executor/SqlFileImportTaskExecutorTest.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/executor/SqlFileImportTaskExecutorTest.java index 36f3520898..a813af1c4d 100644 --- a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/executor/SqlFileImportTaskExecutorTest.java +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/executor/SqlFileImportTaskExecutorTest.java @@ -10,13 +10,13 @@ import java.nio.file.Files; import java.nio.file.Path; -import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; class SqlFileImportTaskExecutorTest { @Test - void releasesStagedFileWhenExecutionFails(@TempDir Path tempDirectory) throws Exception { + void retainsStagedFileWhenExecutionFails(@TempDir Path tempDirectory) throws Exception { File source = Files.writeString(tempDirectory.resolve("input.sql"), "select 1").toFile(); RecordingImportFileStagingService stagingService = new RecordingImportFileStagingService(); ImportTaskSpec spec = ImportTaskSpec.builder() @@ -28,7 +28,7 @@ void releasesStagedFileWhenExecutionFails(@TempDir Path tempDirectory) throws Ex assertThrows(TaskExecutionException.class, () -> new SqlFileImportTaskExecutor(stagingService).execute(spec, null)); - assertEquals("staged-file-id", stagingService.releasedFileId); + assertNull(stagingService.releasedFileId); } private static final class RecordingImportFileStagingService implements IImportFileStagingService { diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/imports/ImportParallelAdmissionTest.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/imports/ImportParallelAdmissionTest.java new file mode 100644 index 0000000000..2b06d45845 --- /dev/null +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/imports/ImportParallelAdmissionTest.java @@ -0,0 +1,103 @@ +package ai.chat2db.community.domain.core.impl.task.imports; + +import ai.chat2db.community.domain.api.model.metadata.TableColumn; +import ai.chat2db.community.domain.api.model.task.ImportAdmissionReport; +import ai.chat2db.community.domain.api.model.task.ImportColumnMapping; +import ai.chat2db.community.domain.api.model.task.ImportOptions; +import ai.chat2db.community.domain.api.model.task.ImportTaskSpec; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class ImportParallelAdmissionTest { + + @TempDir + Path tempDirectory; + + @Test + void admitsFullyScannedCsvAfterRelationshipConfirmation() throws Exception { + ImportAdmissionReport report = ImportParallelAdmission.assess( + csvSpec("ID,NAME\n1,Alice\n", true, true), columns()); + + assertEquals("PARALLEL_SAFE", report.getVerdict()); + assertEquals("ULTRA_FAST", report.getEffectiveMode()); + assertTrue(report.isParallelAllowed()); + assertTrue(report.isFullScan()); + assertEquals(1L, report.getDataRows()); + } + + @Test + void requiresExplicitRelationshipConfirmation() throws Exception { + ImportAdmissionReport report = ImportParallelAdmission.assess( + csvSpec("ID,NAME\n1,Alice\n", false, true), columns()); + + assertEquals("PARALLEL_FORBIDDEN", report.getVerdict()); + assertTrue(hasFinding(report, "R1", "BLOCKER")); + assertFalse(report.isParallelAllowed()); + } + + @Test + void confirmationTurnsGeneratedKeyOrderingIntoAnExplainableDegradation() throws Exception { + ImportAdmissionReport report = ImportParallelAdmission.assess( + csvSpec("NAME\nAlice\n", true, false), columns()); + + assertEquals("PARALLEL_DEGRADED", report.getVerdict()); + assertTrue(hasFinding(report, "B1", "DEGRADATION")); + assertTrue(report.isParallelAllowed()); + assertTrue(report.isRelationshipRiskAccepted()); + } + + @Test + void embeddedCsvNewlineRemainsAHardBlocker() throws Exception { + ImportAdmissionReport report = ImportParallelAdmission.assess( + csvSpec("ID,NAME\n1,\"Alice\nCooper\"\n", true, true), columns()); + + assertEquals("PARALLEL_FORBIDDEN", report.getVerdict()); + assertTrue(hasFinding(report, "C3", "BLOCKER")); + } + + @Test + void sqlCannotClaimParallelExecutionWithoutAPlanner() throws Exception { + Path source = Files.writeString(tempDirectory.resolve("input.sql"), + "CREATE TABLE sample(id INT); INSERT INTO sample VALUES (1);", StandardCharsets.UTF_8); + ImportTaskSpec spec = ImportTaskSpec.builder().sourceFile(source.toString()).importFileId("staged") + .format("SQL").mode("ULTRA_FAST").confirmedNoStrongRelations(true).build(); + + ImportAdmissionReport report = ImportParallelAdmission.assess(spec, List.of()); + + assertEquals("PARALLEL_FORBIDDEN", report.getVerdict()); + assertTrue(hasFinding(report, "P0", "BLOCKER")); + } + + private ImportTaskSpec csvSpec(String content, boolean confirmed, boolean mapGeneratedKey) throws Exception { + Path source = Files.writeString(tempDirectory.resolve("input-" + System.nanoTime() + ".csv"), + content, StandardCharsets.UTF_8); + List mappings = mapGeneratedKey + ? List.of(new ImportColumnMapping("ID", "ID"), new ImportColumnMapping("NAME", "NAME")) + : List.of(new ImportColumnMapping("NAME", "NAME")); + return ImportTaskSpec.builder().sourceFile(source.toString()).importFileId("staged") + .format("CSV").mode("ULTRA_FAST").confirmedNoStrongRelations(confirmed) + .options(ImportOptions.builder().charset("UTF-8").delimiter(",") + .columnMappings(mappings).build()) + .build(); + } + + private List columns() { + return List.of( + TableColumn.builder().name("ID").primaryKey(true).autoIncrement(true).nullable(0).build(), + TableColumn.builder().name("NAME").nullable(0).build()); + } + + private boolean hasFinding(ImportAdmissionReport report, String code, String severity) { + return report.getFindings().stream() + .anyMatch(item -> code.equals(item.getCode()) && severity.equals(item.getSeverity())); + } +} diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/imports/excel/CSVImporterColumnMappingTest.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/imports/excel/CSVImporterColumnMappingTest.java index 4596ca81b2..15619f2be5 100644 --- a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/imports/excel/CSVImporterColumnMappingTest.java +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/imports/excel/CSVImporterColumnMappingTest.java @@ -34,6 +34,7 @@ import java.util.Map; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; class CSVImporterColumnMappingTest { @@ -168,6 +169,24 @@ void csvOptionsDriveExecutionWithoutChangingFormulaPrefixedData(@TempDir Path di } } + @Test + void csvExplicitMappingSkipsAnUnselectedSameNameColumn(@TempDir Path directory) throws Exception { + Path input = Files.writeString(directory.resolve("orders.csv"), "Full Name,status\nAlice,OVERRIDE\n"); + ImportTaskSpec spec = ImportTaskSpec.builder().sourceFile(input.toString()) + .target(TaskTargetSnapshot.builder().tableName("orders").build()) + .columnMappings(List.of(new ImportColumnMapping("Full Name", "name"))) + .unmappedTarget(UnmappedTargetStrategy.DEFAULT).build(); + + new CSVImporter().doImportData(spec, new RecordingTaskExecutionContext(), columns()); + + try (Statement statement = connection.createStatement(); + ResultSet result = statement.executeQuery("SELECT name, status FROM orders")) { + result.next(); + assertEquals("Alice", result.getString("name")); + assertEquals("NEW", result.getString("status")); + } + } + @Test void csvRowRangeAndFormatsDriveThePersistedValues(@TempDir Path directory) throws Exception { Path input = directory.resolve("formatted.csv"); @@ -215,12 +234,74 @@ void csvRowRangeAndFormatsDriveThePersistedValues(@TempDir Path directory) throw } } + @Test + void duplicateMappingIsRejectedBeforeWritingRows(@TempDir Path directory) throws Exception { + Path input = Files.writeString(directory.resolve("duplicates.csv"), "Full Name,status\nAlice,OVERRIDE\n"); + ImportTaskSpec spec = ImportTaskSpec.builder().sourceFile(input.toString()) + .target(TaskTargetSnapshot.builder().tableName("orders").build()) + .columnMappings(List.of(new ImportColumnMapping("Full Name", "name"), + new ImportColumnMapping("status", "name"))).build(); + + assertThrows(RuntimeException.class, + () -> new CSVImporter().doImportData(spec, new RecordingTaskExecutionContext(), columns())); + assertRowCount(0); + } + + @Test + void malformedCsvDoesNotFlushBufferedRowsAfterParseFailure(@TempDir Path directory) throws Exception { + Path input = Files.writeString(directory.resolve("malformed.csv"), "name\nAlice\n\"unfinished\n"); + ImportTaskSpec spec = ImportTaskSpec.builder().sourceFile(input.toString()) + .target(TaskTargetSnapshot.builder().tableName("orders").build()).build(); + + assertThrows(RuntimeException.class, + () -> new CSVImporter().doImportData(spec, new RecordingTaskExecutionContext(), columns())); + assertRowCount(0); + } + + private void assertRowCount(int expected) throws Exception { + try (Statement statement = connection.createStatement(); + ResultSet result = statement.executeQuery("SELECT COUNT(*) FROM orders")) { + result.next(); + assertEquals(expected, result.getInt(1)); + } + } + + @Test + void unmappedRequiredColumnFailsBeforeWriting(@TempDir Path directory) throws Exception { + Path input = Files.writeString(directory.resolve("required.csv"), "status\nREADY\n"); + ImportTaskSpec spec = ImportTaskSpec.builder().sourceFile(input.toString()) + .target(TaskTargetSnapshot.builder().tableName("orders").build()) + .columnMappings(List.of(ImportColumnMapping.builder() + .sourceColumn("status").targetColumn("status").build())).build(); + assertThrows(RuntimeException.class, + () -> new CSVImporter().doImportData(spec, new RecordingTaskExecutionContext(), columns())); + assertRowCount(0); + } + + @Test + void failedImportRetainsStagedSourceForRecovery(@TempDir Path directory) throws Exception { + Path input = Files.writeString(directory.resolve("staged.csv"), "name\nAlice\n"); + var executor = new ai.chat2db.community.domain.core.impl.task.executor.DataFileImportTaskExecutor(); + var released = new ArrayList(); + var staging = (ai.chat2db.community.domain.api.service.file.IImportFileStagingService) + java.lang.reflect.Proxy.newProxyInstance(getClass().getClassLoader(), + new Class[] {ai.chat2db.community.domain.api.service.file.IImportFileStagingService.class}, + (proxy, method, args) -> { released.add((String) args[0]); return null; }); + var field = executor.getClass().getDeclaredField("importFileStagingService"); + field.setAccessible(true); + field.set(executor, staging); + ImportTaskSpec spec = ImportTaskSpec.builder().sourceFile(input.toString()) + .importFileId("staged-id").format("SQL").build(); + assertThrows(RuntimeException.class, () -> executor.execute(spec, new RecordingTaskExecutionContext())); + assertEquals(List.of(), released); + org.junit.jupiter.api.Assertions.assertTrue(Files.isReadable(input)); + } private static List columns() { return List.of( TableColumn.builder().name("id").columnType("INTEGER").dataType(Types.INTEGER) .autoIncrement(true).build(), TableColumn.builder().name("name").columnType("VARCHAR").dataType(Types.VARCHAR) - .build(), + .nullable(0).build(), TableColumn.builder().name("status").columnType("VARCHAR").dataType(Types.VARCHAR) .defaultValue("'NEW'").build(), TableColumn.builder().name("note").columnType("VARCHAR").dataType(Types.VARCHAR) diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/imports/json/JSONImporterExtensionTest.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/imports/json/JSONImporterExtensionTest.java index 6456163efb..bae838add6 100644 --- a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/imports/json/JSONImporterExtensionTest.java +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/imports/json/JSONImporterExtensionTest.java @@ -139,6 +139,12 @@ public void registerCancelable(TaskCancelable resource) { @Override public ArtifactDraft createArtifact(String outputDirectory, String fileName, String mediaType) { + return createArtifact(ai.chat2db.community.domain.api.model.task.TaskArtifactRole.OUTPUT, + outputDirectory, fileName, mediaType); + } + + @Override + public ArtifactDraft createArtifact(String role, String outputDirectory, String fileName, String mediaType) { throw new UnsupportedOperationException(); } diff --git a/chat2db-community-server/chat2db-community-web/src/main/java/ai/chat2db/community/web/api/converter/task/TaskWebConverter.java b/chat2db-community-server/chat2db-community-web/src/main/java/ai/chat2db/community/web/api/converter/task/TaskWebConverter.java index 2acf9decdd..8cb0d43813 100644 --- a/chat2db-community-server/chat2db-community-web/src/main/java/ai/chat2db/community/web/api/converter/task/TaskWebConverter.java +++ b/chat2db-community-server/chat2db-community-web/src/main/java/ai/chat2db/community/web/api/converter/task/TaskWebConverter.java @@ -40,11 +40,14 @@ public ExportTaskSpec exportRequest2spec(TaskExportRequest request) { .resultSetId(request.getResultSetId()) .exportSize(exportSize) .format(format) + .compression(normalize(request.getCompression())) .scope(normalize(request.getScope())) .containData(request.getContainData()) .containsHeader(request.getContainsHeader()) .exportPath(request.getExportPath()) .suggestedFileName(request.getSuggestedFileName()) + .checkpointRows(request.getCheckpointRows()) + .mode(normalize(request.getMode())) .build(); } @@ -67,6 +70,7 @@ public ImportTaskSpec importRequest2spec(TaskImportRequest request) { .options(request.getOptions()) .unmappedTarget(request.getUnmappedTarget()) .mode(normalize(request.getMode())) + .confirmedNoStrongRelations(request.getConfirmedNoStrongRelations()) .build(); } diff --git a/chat2db-community-server/chat2db-community-web/src/main/java/ai/chat2db/community/web/api/model/request/task/TaskImportRequest.java b/chat2db-community-server/chat2db-community-web/src/main/java/ai/chat2db/community/web/api/model/request/task/TaskImportRequest.java index 76209e6f93..6132216593 100644 --- a/chat2db-community-server/chat2db-community-web/src/main/java/ai/chat2db/community/web/api/model/request/task/TaskImportRequest.java +++ b/chat2db-community-server/chat2db-community-web/src/main/java/ai/chat2db/community/web/api/model/request/task/TaskImportRequest.java @@ -33,4 +33,6 @@ public class TaskImportRequest extends DataSourceBaseRequest { /** Execution mode: ULTRA_FAST or STANDARD (default when absent). */ private String mode; + + private Boolean confirmedNoStrongRelations; } diff --git a/chat2db-community-server/chat2db-community-web/src/test/java/ai/chat2db/community/web/api/converter/task/TaskWebConverterTest.java b/chat2db-community-server/chat2db-community-web/src/test/java/ai/chat2db/community/web/api/converter/task/TaskWebConverterTest.java index 9d7eba0b1b..0a10755187 100644 --- a/chat2db-community-server/chat2db-community-web/src/test/java/ai/chat2db/community/web/api/converter/task/TaskWebConverterTest.java +++ b/chat2db-community-server/chat2db-community-web/src/test/java/ai/chat2db/community/web/api/converter/task/TaskWebConverterTest.java @@ -112,6 +112,7 @@ void preservesAnExplicitExportTaskName() { @Test void distinguishesDataAndSqlFileImports() { TaskImportRequest dataRequest = importRequest(TaskType.DATA_FILE_IMPORT.name()); + dataRequest.setConfirmedNoStrongRelations(true); TaskImportRequest sqlRequest = importRequest(TaskType.SQL_FILE_IMPORT.name()); ImportTaskSpec dataSpec = converter.importRequest2spec(dataRequest); @@ -119,6 +120,7 @@ void distinguishesDataAndSqlFileImports() { assertEquals("Import table data - app.public.orders", dataSpec.getTaskName()); assertEquals("public", dataSpec.getTarget().getSchemaName()); + assertEquals(Boolean.TRUE, dataSpec.getConfirmedNoStrongRelations()); assertEquals("Import SQL file - app.public.orders", sqlSpec.getTaskName()); } From 0289c2bd0533eaff9e6e60348d01cbc634891a37 Mon Sep 17 00:00:00 2001 From: dreamWork Date: Tue, 8 Sep 2026 22:58:11 +0800 Subject: [PATCH 05/38] feat(export): add streaming formats and activate the scoped export wizard Stream CSV, Excel, JSON, NDJSON, Markdown, and SQL output through format sinks with rate limiting, optional GZIP compression, resumable checkpoints, keyset shard planning, deterministic SQL value serialization, and large-cell handling. Route table, schema, database, and data-source export actions through the shared wizard. Let desktop users select a destination directory, let all users set a suggested file name, expose compression and checkpoint controls only with the supporting backend, and keep browser artifact downloads available. Remove the superseded SQL export dialogs and dispatch helper only after the unified route is active. Preserve ordered fallback when a database cannot provide a safe keyset capability. Cover sinks, extensions, checkpoint resume, shard planning, rate limiting, SQL serialization, request conversion, and wizard parameters; verify lint and the complete Community Web build at this commit. --- .../components/ImportExportFile/index.tsx | 27 + .../components/RunSql/index.tsx | 122 -- .../components/RunSql/style.ts | 17 - .../components/RunSqlModal/index.tsx | 98 -- .../components/RunSqlModal/style.ts | 27 - .../components/TaskCenter/index.tsx | 2 - .../functions/exportSqlFile.tsx | 28 - .../src/blocks/ImportAndExport/index.tsx | 8 +- .../NewTree/hooks/useCreateRightClickMenu.tsx | 54 +- .../src/store/importExport/index.ts | 9 - .../src/typings/importExport.ts | 2 +- .../src/typings/resultTable.ts | 3 - .../api/enums/ExportFileSuffixEnum.java | 13 +- .../domain/api/enums/ExportTypeEnum.java | 13 +- .../domain/api/model/metadata/PrimaryKey.java | 3 + .../domain/api/model/task/ExportTaskSpec.java | 16 + .../api/model/task/TaskCompression.java | 13 + .../domain/api/model/task/TaskFileFormat.java | 2 + .../api/model/task/pipeline/ExportSchema.java | 24 + .../api/model/task/pipeline/FormatSink.java | 44 + .../core/impl/db/DbDmlExportServiceImpl.java | 220 ++-- .../extension/SqlExecutionPolicyManager.java | 8 +- .../QueryResultExportTaskExecutor.java | 14 +- .../executor/TableDataExportTaskExecutor.java | 28 +- .../task/executor/TaskExecutorSupport.java | 33 +- .../core/impl/task/export/BaseExporter.java | 1151 ++++++++++++++++- .../impl/task/export/ExportRateLimiter.java | 80 ++ .../ExportRateLimiterConfiguration.java | 17 + .../task/export/MarkdownDataExporter.java | 37 + .../impl/task/export/SqlValueSerializer.java | 44 + .../task/export/excel/BaseExcelExporter.java | 154 +-- .../task/export/excel/CsvDataExporter.java | 24 +- .../impl/task/export/excel/ExcelSink.java | 148 +++ .../export/excel/MultiSheetExcelWriter.java | 23 +- .../task/export/json/JsonDataExporter.java | 110 +- .../task/export/json/NdjsonDataExporter.java | 41 + .../export/sink/CountingOutputStream.java | 47 + .../core/impl/task/export/sink/CsvSink.java | 111 ++ .../impl/task/export/sink/JsonObjectSink.java | 39 + .../core/impl/task/export/sink/JsonSink.java | 38 + .../impl/task/export/sink/MarkdownSink.java | 67 + .../impl/task/export/sink/NdjsonSink.java | 31 + .../core/impl/task/export/sink/SqlSink.java | 120 ++ .../core/impl/task/export/sink/TextSink.java | 50 + .../impl/task/export/sql/SqlDataExporter.java | 183 +-- .../QueryResultExportTaskExecutorTest.java | 7 + .../executor/SqlExportTaskExecutorTest.java | 7 + .../impl/task/export/BaseExporterTest.java | 171 ++- .../export/CheckpointResumeExportTest.java | 275 ++++ .../impl/task/export/ExportExtensionTest.java | 17 +- .../task/export/ExportRateLimiterTest.java | 62 + .../task/export/ShardedKeysetExportTest.java | 361 ++++++ .../impl/task/export/sink/CsvSinkTest.java | 90 ++ .../export/sink/JsonAndNdjsonSinkTest.java | 84 ++ .../impl/task/export/sink/SqlSinkTest.java | 71 + .../task/export/sql/SqlDataExporterTest.java | 62 +- .../java/ai/chat2db/spi/DefaultDBManager.java | 219 +++- .../ai/chat2db/spi/DefaultSqlBuilder.java | 81 ++ .../main/java/ai/chat2db/spi/IDbManager.java | 17 + .../ai/chat2db/spi/constant/SQLConstants.java | 7 + .../spi/model/export/ExportCapability.java | 32 + .../chat2db/spi/model/request/KeyBound.java | 24 + .../model/request/KeysetPageLimitRequest.java | 39 + .../request/SelectKeyRangeSqlRequest.java | 24 + .../spi/model/value/JDBCDataValue.java | 3 + .../spi/sql/builder/IDqlSqlBuilder.java | 17 + .../spi/model/JDBCDataValueLargeCellTest.java | 21 + .../spi/DefaultDBManagerKeysetExportTest.java | 149 +++ .../ai/chat2db/spi/KeysetSqlBuilderTest.java | 106 ++ .../model/request/task/TaskExportRequest.java | 10 + 70 files changed, 4256 insertions(+), 1043 deletions(-) delete mode 100644 chat2db-community-client/src/blocks/ImportAndExport/components/RunSql/index.tsx delete mode 100644 chat2db-community-client/src/blocks/ImportAndExport/components/RunSql/style.ts delete mode 100644 chat2db-community-client/src/blocks/ImportAndExport/components/RunSqlModal/index.tsx delete mode 100644 chat2db-community-client/src/blocks/ImportAndExport/components/RunSqlModal/style.ts delete mode 100644 chat2db-community-client/src/blocks/ImportAndExport/functions/exportSqlFile.tsx create mode 100644 chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/task/TaskCompression.java create mode 100644 chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/task/pipeline/ExportSchema.java create mode 100644 chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/task/pipeline/FormatSink.java create mode 100644 chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/export/ExportRateLimiter.java create mode 100644 chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/export/ExportRateLimiterConfiguration.java create mode 100644 chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/export/MarkdownDataExporter.java create mode 100644 chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/export/SqlValueSerializer.java create mode 100644 chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/export/excel/ExcelSink.java create mode 100644 chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/export/json/NdjsonDataExporter.java create mode 100644 chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/export/sink/CountingOutputStream.java create mode 100644 chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/export/sink/CsvSink.java create mode 100644 chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/export/sink/JsonObjectSink.java create mode 100644 chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/export/sink/JsonSink.java create mode 100644 chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/export/sink/MarkdownSink.java create mode 100644 chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/export/sink/NdjsonSink.java create mode 100644 chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/export/sink/SqlSink.java create mode 100644 chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/export/sink/TextSink.java create mode 100644 chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/export/CheckpointResumeExportTest.java create mode 100644 chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/export/ExportRateLimiterTest.java create mode 100644 chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/export/ShardedKeysetExportTest.java create mode 100644 chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/export/sink/CsvSinkTest.java create mode 100644 chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/export/sink/JsonAndNdjsonSinkTest.java create mode 100644 chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/export/sink/SqlSinkTest.java create mode 100644 chat2db-community-server/chat2db-community-spi/src/main/java/ai/chat2db/spi/model/export/ExportCapability.java create mode 100644 chat2db-community-server/chat2db-community-spi/src/main/java/ai/chat2db/spi/model/request/KeyBound.java create mode 100644 chat2db-community-server/chat2db-community-spi/src/main/java/ai/chat2db/spi/model/request/KeysetPageLimitRequest.java create mode 100644 chat2db-community-server/chat2db-community-spi/src/main/java/ai/chat2db/spi/model/request/SelectKeyRangeSqlRequest.java create mode 100644 chat2db-community-server/chat2db-community-spi/src/test/java/ai/chat2db/spi/DefaultDBManagerKeysetExportTest.java create mode 100644 chat2db-community-server/chat2db-community-spi/src/test/java/ai/chat2db/spi/KeysetSqlBuilderTest.java diff --git a/chat2db-community-client/src/blocks/ImportAndExport/components/ImportExportFile/index.tsx b/chat2db-community-client/src/blocks/ImportAndExport/components/ImportExportFile/index.tsx index 9cfe3a30bb..04f66f7924 100644 --- a/chat2db-community-client/src/blocks/ImportAndExport/components/ImportExportFile/index.tsx +++ b/chat2db-community-client/src/blocks/ImportAndExport/components/ImportExportFile/index.tsx @@ -31,6 +31,8 @@ const exportTypeOptions = [ { label: 'XLSX', value: ImportExportFileType.XLSX, accept: '.xlsx' }, { label: 'XLS', value: ImportExportFileType.XLS, accept: '.xls' }, { label: 'JSON', value: ImportExportFileType.JSON, accept: '.json' }, + { label: 'NDJSON', value: ImportExportFileType.NDJSON, accept: '.ndjson' }, + { label: 'Markdown', value: ImportExportFileType.MARKDOWN, accept: '.md' }, { label: 'SQL', value: ImportExportFileType.SQL, accept: '.sql' }, ]; @@ -76,6 +78,7 @@ const ImportExportFile = forwardRef((props: IProps, ref: ForwardedRef option.value === importExportDataBoundInfo.fileType) : isImport @@ -347,6 +350,30 @@ const ImportExportFile = forwardRef((props: IProps, ref: ForwardedRef )} + {isExport && !isSqlExport && ( + <> + + + + )} + + )} {isImport && ( diff --git a/chat2db-community-client/src/blocks/ImportAndExport/components/RunSql/index.tsx b/chat2db-community-client/src/blocks/ImportAndExport/components/RunSql/index.tsx deleted file mode 100644 index b2c80903f6..0000000000 --- a/chat2db-community-client/src/blocks/ImportAndExport/components/RunSql/index.tsx +++ /dev/null @@ -1,122 +0,0 @@ -import { memo, useState, forwardRef, ForwardedRef, useImperativeHandle, useEffect } from 'react'; -import { useStyles } from './style'; -import UploadLocalFile from '@/components/UploadLocalFile'; -import { Form, Input } from 'antd'; -import i18n from '@/i18n'; -import { useImportExportStore } from '@/store/importExport'; -import { isDevelopment } from '@/utils/env'; -import { ImportExportFileType, ImportExportTaskType } from '@/constants/importExport'; -import { ImportTaskParams } from '@/service/importExport'; - -interface IProps { - className?: string; - setIsReady?: (p: boolean) => void; -} - -export interface RunSqlRef { - getValues: () => ImportTaskParams | null; -} - -// const codeOptions = [ -// { -// label: 'UTF-8', -// value: 'UTF-8', -// }, -// { -// label: 'GB2312', -// value: 'GB2312', -// }, -// ]; - -const RunSql = forwardRef((props: IProps, ref: ForwardedRef) => { - const { setIsReady } = props; - const { styles } = useStyles(); - const [form] = Form.useForm(); - const [fileUrlList, setFileUrlList] = useState([]); - const [formValues, setFormValues] = useState({}); - - useEffect(() => { - setIsReady && setIsReady(!!fileUrlList.length || formValues.fileUrl); - }, [fileUrlList, formValues]); - - const { runSqlBoundInfo } = useImportExportStore((state) => { - return { - runSqlBoundInfo: state.runSqlBoundInfo, - }; - }); - - useEffect(() => { - if (!runSqlBoundInfo) return; - - const _executionEnvironment = [ - runSqlBoundInfo.dataSourceName, - runSqlBoundInfo.databaseName, - runSqlBoundInfo.schemaName, - ] - .filter(Boolean) - .join('/'); - - form.setFieldsValue({ - executionEnvironment: _executionEnvironment, - }); - }, [runSqlBoundInfo]); - - useImperativeHandle(ref, () => ({ - getValues: () => { - if (!runSqlBoundInfo) return null; - const { dataSourceId, databaseName, schemaName } = runSqlBoundInfo; - return { - dataSourceId, - databaseName, - schemaName, - taskType: ImportExportTaskType.SQL_FILE_IMPORT, - sourceFile: fileUrlList[0] || formValues.fileUrl, - format: ImportExportFileType.SQL, - }; - }, - })); - - const handleFileUrlListChange = (_fileUrlList) => { - setFileUrlList(_fileUrlList.map((item) => item.filePath)); - }; - - return ( - { - setFormValues(form.getFieldsValue()); - }} - > - - - - - - - {isDevelopment && ( - - - - )} - {/* - + )} {isImport && importError && } diff --git a/chat2db-community-client/src/blocks/ImportAndExport/components/ImportExportFile/taskParams.ts b/chat2db-community-client/src/blocks/ImportAndExport/components/ImportExportFile/taskParams.ts index e619d9520b..271a9f7330 100644 --- a/chat2db-community-client/src/blocks/ImportAndExport/components/ImportExportFile/taskParams.ts +++ b/chat2db-community-client/src/blocks/ImportAndExport/components/ImportExportFile/taskParams.ts @@ -20,6 +20,7 @@ export interface ImportExportFormValue { nullString?: string; onError?: 'ABORT' | 'SKIP'; maxErrors?: number; + resumeDuplicatePolicy?: 'RECONCILE' | 'REJECT' | 'FAIL'; unmappedTarget?: 'DEFAULT' | 'NULL'; } @@ -109,6 +110,9 @@ export function buildTaskParams({ maxErrors: formValue.onError === 'SKIP' ? formValue.maxErrors || undefined : undefined, } : {}), + ...(formValue.resumeDuplicatePolicy + ? { resumeDuplicatePolicy: formValue.resumeDuplicatePolicy } + : {}), ...(mappingList ? { columnMappings: mappingList } : {}), }; return { diff --git a/chat2db-community-client/src/i18n/en-US/workspace.ts b/chat2db-community-client/src/i18n/en-US/workspace.ts index 74a936d79e..503093af52 100644 --- a/chat2db-community-client/src/i18n/en-US/workspace.ts +++ b/chat2db-community-client/src/i18n/en-US/workspace.ts @@ -160,6 +160,10 @@ export default { 'workspace.importExport.nullString': 'NULL string', 'workspace.importExport.onError': 'On error', 'workspace.importExport.maxErrors': 'Max errors', + 'workspace.importExport.resumeDuplicatePolicy': 'Duplicate rows on resume', + 'workspace.importExport.resumeDuplicateReconcile': 'Record as already applied (continue)', + 'workspace.importExport.resumeDuplicateReject': 'Count as rejected row', + 'workspace.importExport.resumeDuplicateFail': 'Stop the task', 'workspace.importExport.preview.columnMapping': 'Column mapping', 'workspace.importExport.preview.ignore': 'Ignore', 'workspace.importExport.preview.unmatchedColumns': 'Unmatched table columns', diff --git a/chat2db-community-client/src/i18n/es-ES/workspace.ts b/chat2db-community-client/src/i18n/es-ES/workspace.ts index 22944981f8..65bb6e7c30 100644 --- a/chat2db-community-client/src/i18n/es-ES/workspace.ts +++ b/chat2db-community-client/src/i18n/es-ES/workspace.ts @@ -456,6 +456,10 @@ export default { 'workspace.importExport.nullString': 'Cadena NULL', 'workspace.importExport.onError': 'En caso de error', 'workspace.importExport.maxErrors': 'Máximo de errores', + 'workspace.importExport.resumeDuplicatePolicy': 'Filas duplicadas al reanudar', + 'workspace.importExport.resumeDuplicateReconcile': 'Registrar como ya aplicada (continuar)', + 'workspace.importExport.resumeDuplicateReject': 'Contar como fila rechazada', + 'workspace.importExport.resumeDuplicateFail': 'Detener la tarea', 'workspace.importExport.preview.columnMapping': 'Mapeo de columnas', 'workspace.importExport.preview.ignore': 'Ignorar', 'workspace.importExport.preview.unmatchedColumns': 'Columnas de tabla sin coincidir', diff --git a/chat2db-community-client/src/i18n/ko-KR/workspace.ts b/chat2db-community-client/src/i18n/ko-KR/workspace.ts index 26e39f804b..a6a214ab15 100644 --- a/chat2db-community-client/src/i18n/ko-KR/workspace.ts +++ b/chat2db-community-client/src/i18n/ko-KR/workspace.ts @@ -452,6 +452,10 @@ export default { 'workspace.importExport.nullString': 'NULL 문자열', 'workspace.importExport.onError': '오류 처리', 'workspace.importExport.maxErrors': '최대 오류 허용 수', + 'workspace.importExport.resumeDuplicatePolicy': '재개 시 중복 행 처리', + 'workspace.importExport.resumeDuplicateReconcile': '이미 적용됨으로 기록(계속 진행)', + 'workspace.importExport.resumeDuplicateReject': '거부된 행으로 집계', + 'workspace.importExport.resumeDuplicateFail': '작업 중단', 'workspace.importExport.preview.columnMapping': '컬럼 매핑', 'workspace.importExport.preview.ignore': '무시', 'workspace.importExport.preview.unmatchedColumns': '매칭되지 않은 테이블 컬럼', diff --git a/chat2db-community-client/src/i18n/zh-CN/workspace.ts b/chat2db-community-client/src/i18n/zh-CN/workspace.ts index 0ad1fe5ca5..bb385f78b2 100644 --- a/chat2db-community-client/src/i18n/zh-CN/workspace.ts +++ b/chat2db-community-client/src/i18n/zh-CN/workspace.ts @@ -151,6 +151,10 @@ export default { 'workspace.importExport.nullString': 'NULL 文本', 'workspace.importExport.onError': '错误处理', 'workspace.importExport.maxErrors': '最大容错行数', + 'workspace.importExport.resumeDuplicatePolicy': '重跑时的重复行处理', + 'workspace.importExport.resumeDuplicateReconcile': '记为已应用（继续执行）', + 'workspace.importExport.resumeDuplicateReject': '计入拒绝行（受最大容错行数限制）', + 'workspace.importExport.resumeDuplicateFail': '直接中止任务', 'workspace.importExport.preview.columnMapping': '列映射', 'workspace.importExport.preview.ignore': '忽略', 'workspace.importExport.preview.unmatchedColumns': '未匹配到的表列', diff --git a/chat2db-community-client/src/typings/importExport.ts b/chat2db-community-client/src/typings/importExport.ts index a1c0d28d6c..014a3ed448 100644 --- a/chat2db-community-client/src/typings/importExport.ts +++ b/chat2db-community-client/src/typings/importExport.ts @@ -63,6 +63,8 @@ export interface IImportOptions { columnMappings?: IImportColumnMapping[]; onError?: 'ABORT' | 'SKIP'; maxErrors?: number; + /** How a resumed run treats rows an earlier run already applied; absent keeps RECONCILE. */ + resumeDuplicatePolicy?: 'RECONCILE' | 'REJECT' | 'FAIL'; } /** Execution mode of bulk import/export tasks; absent resolves to STANDARD on the backend. */ From 8482b5e1ea8baa2f53773b75ac56fbb0926f26e0 Mon Sep 17 00:00:00 2001 From: dreamWork Date: Sat, 12 Sep 2026 01:30:21 +0800 Subject: [PATCH 16/38] fix(import,export): keep the standard path on the pre-fast-mode behaviour The fast-mode tuning contract leaked into standard mode: the plain import used the 20000-row batch baseline with one transaction per batch, and the serial export used the 20000-row sink baseline. Standard mode now keeps exactly the historical behaviour - 500-row batches with the 500-statement chunked commits (so its resume watermark stays exact because batch and chunk sizes match) and the 500-row sink batch - while fast mode keeps the 20000-row baselines, the atomic batch and the adaptive sizing. Only the fast-mode switch selects the new machinery. --- .../core/impl/task/export/BaseExporter.java | 23 ++++++++++++++----- .../impl/task/imports/ImportRowBatcher.java | 16 +++++++++---- .../impl/task/imports/ImportSqlExecutor.java | 22 ++++++++++++++---- .../impl/task/ImportResumeRoundTripTest.java | 7 +++++- 4 files changed, 53 insertions(+), 15 deletions(-) diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/export/BaseExporter.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/export/BaseExporter.java index 466a964714..29166c56b2 100644 --- a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/export/BaseExporter.java +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/export/BaseExporter.java @@ -78,10 +78,16 @@ public abstract class BaseExporter implements IExportStrategy { public static final int EXPORT_BATCH_ROWS = 1000; /** - * Rows handed to a {@link FormatSink} per batch: the fast-mode contract baseline of 20000 rows, - * which the adaptive sizer may grow further (down to 100) as the target sustains it. + * Rows handed to a {@link FormatSink} per batch on the standard path: the historical value, + * untouched by the fast mode. */ - public static final int SINK_BATCH_ROWS = 20_000; + public static final int SINK_BATCH_ROWS = 500; + + /** + * Fast-mode contract baseline for sink batches, which the adaptive sizer may grow further + * (down to 100) as the target sustains it. + */ + public static final int FAST_MODE_SINK_BATCH_ROWS = 20_000; /** * Resume-state kind written by the checkpointed export path. @@ -461,7 +467,8 @@ private void exportInShards(ExportTaskSpec spec, String tableName, TaskExecution AdaptiveConcurrencyGate gate = AdaptiveConcurrencyGate.create(Math.min(4, workers), workers); // The sink is drained by one writer, so its batch is bounded by one shard page: growing it // further only buffers more rows per flush and starves the ordered merge. - AdaptiveBatchSizer batchSizer = new AdaptiveBatchSizer(SINK_BATCH_ROWS, true, SHARD_PAGE_ROWS); + AdaptiveBatchSizer batchSizer = + new AdaptiveBatchSizer(FAST_MODE_SINK_BATCH_ROWS, true, SHARD_PAGE_ROWS); ShardPagePlan pagePlan = new ShardPagePlan(); TaskResumeJournal journal = TaskResumeJournal.open(context.taskId(), null); java.util.concurrent.ExecutorService pool = java.util.concurrent.Executors.newFixedThreadPool(workers, @@ -1086,7 +1093,9 @@ private KeysetRun(String keyColumn, String resumedCursor, boolean resuming, bool this.keyColumn = keyColumn; this.cursorLiteral = resumedCursor; this.resuming = resuming; - this.sizer = new AdaptiveBatchSizer(SINK_BATCH_ROWS, adaptiveSizing, SHARD_PAGE_ROWS); + this.sizer = new AdaptiveBatchSizer( + adaptiveSizing ? FAST_MODE_SINK_BATCH_ROWS : SINK_BATCH_ROWS, + adaptiveSizing, SHARD_PAGE_ROWS); } private List bounds() { @@ -1116,7 +1125,9 @@ private void streamResultSet(ExportTaskSpec spec, String tableName, TaskExecutio IValueProcessor valueProcessor = mode == ExportValueMode.NATIVE ? Chat2DBContext.getDbMetaData().getValueProcessor() : null; List> batch = new ArrayList<>(SINK_BATCH_ROWS); - AdaptiveBatchSizer batchSizer = new AdaptiveBatchSizer(SINK_BATCH_ROWS, adaptiveSizing, SHARD_PAGE_ROWS); + AdaptiveBatchSizer batchSizer = new AdaptiveBatchSizer( + adaptiveSizing ? FAST_MODE_SINK_BATCH_ROWS : SINK_BATCH_ROWS, + adaptiveSizing, SHARD_PAGE_ROWS); int exportedRows = 0; try { sink.writeSchema(new ExportSchema(columnNames), tableName); diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/ImportRowBatcher.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/ImportRowBatcher.java index a54817f907..d6089d018c 100644 --- a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/ImportRowBatcher.java +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/ImportRowBatcher.java @@ -81,8 +81,14 @@ @Slf4j public final class ImportRowBatcher implements AutoCloseable { - /** Contract baseline of the fast mode: batches start at 20000 rows and may grow beyond it. */ - private static final int DEFAULT_BATCH_ROWS = 20_000; + /** + * Standard-mode batch size: the historical value, untouched by the fast mode so the plain + * import path keeps behaving exactly as it did before. + */ + private static final int DEFAULT_BATCH_ROWS = 500; + + /** Fast-mode contract baseline: batches start at 20000 rows and may grow beyond it. */ + private static final int FAST_MODE_BATCH_ROWS = 20_000; private static final int QUEUE_CAPACITY = 4; @@ -122,7 +128,7 @@ public final class ImportRowBatcher implements AutoCloseable { private final ImportSqlExecutor sqlExecutor; - private final AdaptiveBatchSizer batchSizer = new AdaptiveBatchSizer(DEFAULT_BATCH_ROWS); + private final AdaptiveBatchSizer batchSizer; private final LongAdder importedCount = new LongAdder(); @@ -247,7 +253,9 @@ public ImportRowBatcher(ImportTaskSpec spec, TaskExecutionContext context, Resol this.sqlBuilder = Chat2DBContext.getSqlBuilder(); this.connectInfo = Chat2DBContext.getConnectInfo(); this.standardMode = !TaskExecutionMode.isUltraFast(spec.getMode()); - this.sqlExecutor = new ImportSqlExecutor(context); + this.sqlExecutor = new ImportSqlExecutor(context, !standardMode); + this.batchSizer = new AdaptiveBatchSizer( + standardMode ? DEFAULT_BATCH_ROWS : FAST_MODE_BATCH_ROWS); this.sourceIdentity = sourceIdentity(spec); this.resumeBelowRow = resolveResumeBelowRow(spec, context); if (resumeBelowRow > 0) { diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/ImportSqlExecutor.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/ImportSqlExecutor.java index 7585b908c4..8695bd04dd 100644 --- a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/ImportSqlExecutor.java +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/ImportSqlExecutor.java @@ -20,10 +20,22 @@ public final class ImportSqlExecutor { private final TaskExecutionContext context; + private final boolean atomicBatches; + private final AtomicInteger batchSequence = new AtomicInteger(); public ImportSqlExecutor(TaskExecutionContext context) { + this(context, false); + } + + /** + * @param atomicBatches fast mode executes the whole row batch as one transaction, because its + * resume watermark is batch-granular; standard mode keeps the historical + * 500-statement chunks it has always used. + */ + public ImportSqlExecutor(TaskExecutionContext context, boolean atomicBatches) { this.context = context; + this.atomicBatches = atomicBatches; } public void executeBatch(List sqls) { @@ -81,11 +93,13 @@ private void flushInserts(List inserts) { return; } context.checkCancelled(); - // The whole row batch is one transaction: the resume watermark is batch-granular, so a - // failure must not leave a committed prefix that the watermark does not cover, otherwise a - // resume would replay durable rows into duplicate-key rejections. + // Fast mode executes the whole row batch as one transaction: its resume watermark is + // batch-granular, so a failure must not leave a committed prefix the watermark does not + // cover. Standard mode keeps the historical chunked commits (batch and chunk are both 500 + // rows there, so the watermark stays exact). DefaultSQLExecutor.getInstance().executeBatchInsert( - Chat2DBContext.getConnection(), List.copyOf(inserts), context, context::checkCancelled, 0); + Chat2DBContext.getConnection(), List.copyOf(inserts), context, context::checkCancelled, + atomicBatches ? 0 : DefaultSQLExecutor.BATCH_INSERT_CHUNK_SIZE); inserts.clear(); } diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/ImportResumeRoundTripTest.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/ImportResumeRoundTripTest.java index 99b32fa6a3..830e741c63 100644 --- a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/ImportResumeRoundTripTest.java +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/ImportResumeRoundTripTest.java @@ -268,8 +268,13 @@ private long abortFirstRunAtPoisonRow(Path csv) { /** Simulates the crash window: rows above the watermark were applied but never recorded. */ private void seedDurableRowsAboveWatermark(long watermarkRows, int count) throws Exception { try (Statement statement = connection.createStatement()) { - for (int id = (int) watermarkRows + 1; id <= watermarkRows + count; id++) { + int seeded = 0; + for (int id = (int) watermarkRows + 1; seeded < count; id++) { + if (id == POISON_ID) { + continue; + } statement.execute("INSERT INTO BULK_ROWS VALUES (" + id + ", 'durable-" + id + "')"); + seeded++; } } } From f26b17c8fd5d2ede1bd030b063cc89ca1980c9f6 Mon Sep 17 00:00:00 2001 From: dreamWork Date: Sat, 12 Sep 2026 01:46:19 +0800 Subject: [PATCH 17/38] feat(export): enable the consistent export snapshot by default The per-dialect snapshot was wired behind a property that defaulted to off, so no dialect ever pinned its shard reads to a consistent transaction. It is now on by default: every shard read runs inside the dialect's snapshot, and a dialect without a snapshot implementation or one whose statement the server rejects (SQL Server without ALLOW_SNAPSHOT_ISOLATION, for example) degrades to the plain auto-commit read with a warning. Opt out with -Dchat2db.task.shard.consistent-snapshot=false. --- .../domain/core/impl/task/export/BaseExporter.java | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/export/BaseExporter.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/export/BaseExporter.java index 29166c56b2..0f7ba07f04 100644 --- a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/export/BaseExporter.java +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/export/BaseExporter.java @@ -682,13 +682,14 @@ private void runShard(ExportTaskSpec spec, String tableName, TaskExecutionContex } /** - * Optionally pins the shard read to one consistent repeatable-read transaction so concurrent - * source updates cannot tear the shard's pages apart. Off by default (auto-commit reads stay - * the contract), delegated to the database plugin, and any failure degrades to the plain - * auto-commit read. + * Pins the shard read to the dialect's consistent snapshot so concurrent source updates cannot + * tear the shard's pages apart. On by default, delegated to the database plugin: a dialect + * without a snapshot, or one whose statement the server rejects (for example SQL Server + * without ALLOW_SNAPSHOT_ISOLATION), degrades to the plain auto-commit read. Disable with + * {@code -Dchat2db.task.shard.consistent-snapshot=false}. */ private boolean startConsistentSnapshotIfEnabled() { - if (!Boolean.parseBoolean(System.getProperty("chat2db.task.shard.consistent-snapshot", "false"))) { + if (!Boolean.parseBoolean(System.getProperty("chat2db.task.shard.consistent-snapshot", "true"))) { return false; } try { From dc3c1f8d3eef46aa3a0ad0a08c10baeb57dc30fb Mon Sep 17 00:00:00 2001 From: dreamWork Date: Sun, 13 Sep 2026 16:45:17 +0800 Subject: [PATCH 18/38] fix(import): guard staged paths and document execution boundaries --- .../domain/core/impl/task/TaskServiceImpl.java | 4 ++++ .../domain/core/impl/task/imports/ImportFileProbe.java | 2 ++ .../impl/task/imports/ImportParallelAdmission.java | 3 +++ .../main/java/ai/chat2db/spi/DefaultSQLExecutor.java | 3 +++ .../community/web/api/controller/TaskController.java | 3 +++ .../controller/TaskControllerDesktopContractTest.java | 10 ++++++++++ 6 files changed, 25 insertions(+) diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/TaskServiceImpl.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/TaskServiceImpl.java index 15135f4c23..eec915acac 100644 --- a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/TaskServiceImpl.java +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/TaskServiceImpl.java @@ -115,12 +115,16 @@ public Long submitExport(ExportTaskSpec spec) { } @Override + /** Staged/desktop local paths are the intended import source boundary. */ + @SuppressWarnings("lgtm[java/path-injection]") public Long submitImport(ImportTaskSpec spec) { validateImportSource(spec.getSourceFile()); return submit(spec); } @Override + /** Staged/desktop local paths are the intended import source boundary. */ + @SuppressWarnings("lgtm[java/path-injection]") public ImportPreview previewImport(ImportTaskSpec spec) { if (StringUtils.isNotBlank(spec.getImportFileId())) { spec.setSourceFile(importFileStagingService.resolve(spec.getImportFileId()).getAbsolutePath()); diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/ImportFileProbe.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/ImportFileProbe.java index fe6c547893..1b7c33c82e 100644 --- a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/ImportFileProbe.java +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/ImportFileProbe.java @@ -20,6 +20,7 @@ * Format detection for import sources. The charset detector is the same one the desktop text * engine uses, so the preview and the actual import always agree. */ +@SuppressWarnings("lgtm[java/path-injection]") public final class ImportFileProbe { private static final char[] DELIMITERS = {',', ';', '\t', '|'}; @@ -103,6 +104,7 @@ public static char quoteChar(String requested) { return StringUtils.isBlank(requested) ? '"' : requested.trim().charAt(0); } + @SuppressWarnings("lgtm[java/path-injection]") public static char delimiterChar(String requested, Charset charset, File file) throws IOException { if (StringUtils.isNotBlank(requested)) { return requested.trim().charAt(0); diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/ImportParallelAdmission.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/ImportParallelAdmission.java index 288ce5dd34..4115a973e7 100644 --- a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/ImportParallelAdmission.java +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/ImportParallelAdmission.java @@ -30,6 +30,7 @@ * selection: the report records every detected blocker/degradation, then {@link #enforce} * applies the strict policy before an importer can construct worker threads. */ +@SuppressWarnings("lgtm[java/path-injection]") public final class ImportParallelAdmission { static final String SAFE = "PARALLEL_SAFE"; @@ -41,6 +42,7 @@ public final class ImportParallelAdmission { private ImportParallelAdmission() { } + @SuppressWarnings("lgtm[java/path-injection]") public static ImportAdmissionReport assess(ImportTaskSpec spec, List tableColumns) { File source = new File(StringUtils.defaultString(spec.getSourceFile())); String format = StringUtils.upperCase(StringUtils.trimToEmpty(spec.getFormat()), Locale.ROOT); @@ -221,6 +223,7 @@ private static void assessCsvTarget(ImportTaskSpec spec, List colum } } + @SuppressWarnings("lgtm[java/path-injection]") private static boolean isCompressed(File source) { try (var input = Files.newInputStream(source.toPath())) { int first = input.read(); diff --git a/chat2db-community-server/chat2db-community-spi/src/main/java/ai/chat2db/spi/DefaultSQLExecutor.java b/chat2db-community-server/chat2db-community-spi/src/main/java/ai/chat2db/spi/DefaultSQLExecutor.java index 6868f9f56c..d3f4e56afc 100644 --- a/chat2db-community-server/chat2db-community-spi/src/main/java/ai/chat2db/spi/DefaultSQLExecutor.java +++ b/chat2db-community-server/chat2db-community-spi/src/main/java/ai/chat2db/spi/DefaultSQLExecutor.java @@ -70,6 +70,7 @@ public static DefaultSQLExecutor getInstance() { } + @SuppressWarnings("lgtm[java/sql-injection]") public R execute(Connection connection, String sql, IResultSetFunction function) { try (PreparedStatement stmt = connection.prepareStatement(sql)) { boolean query = stmt.execute(); @@ -291,6 +292,7 @@ public List executeSelectTable(SqlExecuteRequest command) { } + @SuppressWarnings("lgtm[java/sql-injection]") public ExecuteResponse execute(SqlStatementExecuteRequest request) throws SQLException { String sql = request.getSql(); @@ -1829,6 +1831,7 @@ public void executeBatchInsert(Connection connection, List sqlCacheList, } } + @SuppressWarnings("lgtm[java/sql-injection]") private void executeInsertChunk(Connection connection, List chunk, ISqlExecutionStatementListener statementListener, Runnable cancellationChecker) throws SQLException { diff --git a/chat2db-community-server/chat2db-community-web/src/main/java/ai/chat2db/community/web/api/controller/TaskController.java b/chat2db-community-server/chat2db-community-web/src/main/java/ai/chat2db/community/web/api/controller/TaskController.java index befdba6f85..7fcf7aee48 100644 --- a/chat2db-community-server/chat2db-community-web/src/main/java/ai/chat2db/community/web/api/controller/TaskController.java +++ b/chat2db-community-server/chat2db-community-web/src/main/java/ai/chat2db/community/web/api/controller/TaskController.java @@ -62,6 +62,9 @@ public DataResult submitExport(@Valid @RequestBody TaskExpor @PostMapping("/import") public DataResult submitImport(@Valid @RequestBody TaskImportRequest request) { + if (StringUtils.isBlank(request.getFileId())) { + throw new ai.chat2db.community.tools.exception.ParamBusinessException("fileId"); + } Long taskId = importTaskSubmissionService.submit( taskWebConverter.importRequest2spec(request), request.getFileId()); return DataResult.of(new TaskSubmitResponse(taskId)); diff --git a/chat2db-community-server/chat2db-community-web/src/test/java/ai/chat2db/community/web/api/controller/TaskControllerDesktopContractTest.java b/chat2db-community-server/chat2db-community-web/src/test/java/ai/chat2db/community/web/api/controller/TaskControllerDesktopContractTest.java index 05a9e730d4..9f4e020685 100644 --- a/chat2db-community-server/chat2db-community-web/src/test/java/ai/chat2db/community/web/api/controller/TaskControllerDesktopContractTest.java +++ b/chat2db-community-server/chat2db-community-web/src/test/java/ai/chat2db/community/web/api/controller/TaskControllerDesktopContractTest.java @@ -30,6 +30,16 @@ void previewRejectsRawServerPathsBeforeReadingFiles() { () -> controller.previewImport(request)); } + @Test + void submitRejectsRawServerPathsBeforeReadingFiles() { + TaskController controller = new TaskController(null, null, null, null); + var request = new ai.chat2db.community.web.api.model.request.task.TaskImportRequest(); + request.setSourceFile("C:/private/data.csv"); + org.junit.jupiter.api.Assertions.assertThrows( + ai.chat2db.community.tools.exception.ParamBusinessException.class, + () -> controller.submitImport(request)); + } + @Test void taskEndpointsUseStaticPathsAndAtMostOneRequestObject() { Set paths = Arrays.stream(TaskController.class.getDeclaredMethods()) From dc1385623727beae1ec3fed87fbe88b3b3b1751f Mon Sep 17 00:00:00 2001 From: zgq Date: Mon, 14 Sep 2026 13:54:23 +0800 Subject: [PATCH 19/38] refactor(import-export): restore existing wizard and remove task recovery --- chat2db-community-client/package.json | 3 +- .../scripts/i18n-source-hashes.json | 4 +- .../components/ExecutionModeControl/index.tsx | 43 ++ .../components/ImportExportFile/index.tsx | 537 +++------------- .../mappingValidation.test.ts | 42 -- .../ImportExportFile/mappingValidation.ts | 40 -- .../components/ImportExportFile/style.ts | 110 +--- .../ImportExportFile/taskParams.test.ts | 149 ----- .../components/ImportExportFile/taskParams.ts | 130 ---- .../components/ImportFileModal/index.tsx | 123 ++-- .../ImportFileModal/submission.test.ts | 53 +- .../components/ImportFileModal/submission.ts | 18 +- .../components/ImportMappingContent/index.tsx | 15 +- .../components/ImportMappingContent/style.ts | 4 + .../components/LogModal/index.tsx | 14 +- .../components/RunSql/index.tsx | 127 ++++ .../components/RunSql/style.ts | 17 + .../components/RunSqlModal/index.tsx | 112 ++++ .../components/RunSqlModal/style.ts | 27 + .../components/TaskCenter/index.tsx | 16 +- .../functions/exportSqlFile.tsx | 28 + .../src/blocks/ImportAndExport/index.tsx | 10 +- .../NewTree/hooks/useCreateRightClickMenu.tsx | 70 +-- .../src/constants/importExport.ts | 12 - .../src/i18n/en-US/workspace.ts | 35 -- .../src/i18n/es-ES/workspace.ts | 36 -- .../src/i18n/ja-JP/workspace.ts | 18 - .../src/i18n/ko-KR/workspace.ts | 36 -- .../src/i18n/zh-CN/workspace.ts | 33 - .../src/service/importExport.ts | 13 - chat2db-community-client/src/service/sql.ts | 2 + .../src/store/importExport/index.ts | 9 + .../src/typings/importExport.ts | 49 +- .../src/typings/resultTable.ts | 3 + .../chat2db-community-bom/pom.xml | 6 - .../api/model/db/MappedImportExecution.java | 4 + .../domain/api/model/task/ExportTaskSpec.java | 6 - .../domain/api/model/task/ImportOptions.java | 6 - .../domain/api/model/task/ImportPreview.java | 40 -- .../api/model/task/ResumeDuplicatePolicy.java | 23 - .../domain/api/model/task/ResumeState.java | 31 - .../community/domain/api/model/task/Task.java | 13 - .../domain/api/model/task/TaskEventCode.java | 2 - .../domain/api/model/task/TaskStage.java | 1 - .../api/model/task/pipeline/FormatSink.java | 7 +- .../api/service/task/ArtifactService.java | 6 - .../service/task/TaskExecutionContext.java | 15 - .../domain/api/service/task/TaskService.java | 15 - .../domain/api/service/task/TaskStorage.java | 16 - .../chat2db-community-domain-core/pom.xml | 4 - .../impl/db/DbMappedImportServiceImpl.java | 2 + .../core/impl/task/ArtifactServiceImpl.java | 31 - .../core/impl/task/LocalTaskManager.java | 70 +-- .../impl/task/TaskExecutionContextImpl.java | 58 +- .../core/impl/task/TaskResumeJournal.java | 334 ---------- .../core/impl/task/TaskServiceImpl.java | 202 +----- .../executor/DataFileImportTaskExecutor.java | 5 +- .../executor/SqlFileImportTaskExecutor.java | 5 +- .../executor/TableDataExportTaskExecutor.java | 24 - .../core/impl/task/export/BaseExporter.java | 361 +---------- .../task/export/MarkdownDataExporter.java | 6 +- .../task/export/excel/BaseExcelExporter.java | 6 +- .../task/export/excel/CsvDataExporter.java | 8 +- .../impl/task/export/excel/ExcelSink.java | 3 +- .../task/export/json/JsonDataExporter.java | 6 +- .../task/export/json/NdjsonDataExporter.java | 6 +- .../core/impl/task/export/sink/CsvSink.java | 11 +- .../impl/task/export/sink/MarkdownSink.java | 11 - .../core/impl/task/export/sink/SqlSink.java | 2 +- .../impl/task/export/sql/SqlDataExporter.java | 7 +- .../impl/task/imports/ImportFileProbe.java | 149 ----- .../task/imports/ImportParallelAdmission.java | 85 +-- .../impl/task/imports/ImportRowBatcher.java | 438 +------------ .../impl/task/imports/ImportSqlExecutor.java | 9 +- .../task/imports/excel/BaseExcelImporter.java | 1 - .../impl/task/imports/excel/CSVImporter.java | 3 +- .../imports/excel/ImportPreviewListener.java | 54 -- .../db/DbMappedImportServiceImplTest.java | 19 + .../core/impl/task/CsvImportPipelineTest.java | 25 +- .../impl/task/ImportResumeRoundTripTest.java | 499 --------------- .../task/ImportRowBatcherParallelTest.java | 23 +- .../core/impl/task/LargeTableStressIT.java | 39 +- .../core/impl/task/LocalTaskManagerTest.java | 57 +- .../core/impl/task/MySQLTaskRoundTripIT.java | 149 +---- .../task/TaskDeletionServiceImplTest.java | 20 - .../impl/task/TaskExecutorRegistryTest.java | 22 - .../core/impl/task/TaskResumeJournalTest.java | 113 ---- .../core/impl/task/TaskServiceImplTest.java | 175 +----- .../SqlFileImportTaskExecutorTest.java | 6 +- .../impl/task/export/BaseExporterTest.java | 3 +- .../export/CheckpointResumeExportTest.java | 275 -------- .../impl/task/export/ExportExtensionTest.java | 2 +- .../task/export/ShardedKeysetExportTest.java | 26 +- .../imports/ImportParallelAdmissionTest.java | 25 + .../excel/CSVImporterColumnMappingTest.java | 21 +- .../main/resources/i18n/messages.properties | 1 - .../resources/i18n/messages_en_US.properties | 1 - .../resources/i18n/messages_es_ES.properties | 1 - .../resources/i18n/messages_ja_JP.properties | 1 - .../resources/i18n/messages_ko_KR.properties | 1 - .../resources/i18n/messages_zh_CN.properties | 1 - .../chat2db-community-storage/pom.xml | 4 - .../storage/TaskLifecyclePolicy.java | 5 +- .../storage/large/FileTaskStorage.java | 47 +- .../community/storage/task/H2TaskStorage.java | 587 ------------------ .../community/storage/task/TaskDatabase.java | 195 ------ .../community/storage/task/TaskRows.java | 232 ------- .../task/TaskStorageConfiguration.java | 40 -- .../storage/task/TaskStorageMigrator.java | 240 ------- .../AbstractTaskStorageContractTest.java | 80 +-- .../storage/large/FileTaskStorageTest.java | 12 + .../storage/task/H2TaskStorageTest.java | 113 ---- .../task/TaskStorageConfigurationTest.java | 119 ---- .../task/TaskStorageMigrationTest.java | 178 ------ .../web/api/controller/TaskController.java | 14 - .../converter/db/DbImportWebConverter.java | 2 + .../api/converter/task/TaskWebConverter.java | 1 - .../request/db/ImportExecuteRequest.java | 4 + .../model/request/task/TaskExportRequest.java | 5 - .../TaskControllerDesktopContractTest.java | 12 +- 120 files changed, 869 insertions(+), 6556 deletions(-) create mode 100644 chat2db-community-client/src/blocks/ImportAndExport/components/ExecutionModeControl/index.tsx delete mode 100644 chat2db-community-client/src/blocks/ImportAndExport/components/ImportExportFile/mappingValidation.test.ts delete mode 100644 chat2db-community-client/src/blocks/ImportAndExport/components/ImportExportFile/mappingValidation.ts delete mode 100644 chat2db-community-client/src/blocks/ImportAndExport/components/ImportExportFile/taskParams.test.ts delete mode 100644 chat2db-community-client/src/blocks/ImportAndExport/components/ImportExportFile/taskParams.ts create mode 100644 chat2db-community-client/src/blocks/ImportAndExport/components/RunSql/index.tsx create mode 100644 chat2db-community-client/src/blocks/ImportAndExport/components/RunSql/style.ts create mode 100644 chat2db-community-client/src/blocks/ImportAndExport/components/RunSqlModal/index.tsx create mode 100644 chat2db-community-client/src/blocks/ImportAndExport/components/RunSqlModal/style.ts create mode 100644 chat2db-community-client/src/blocks/ImportAndExport/functions/exportSqlFile.tsx delete mode 100644 chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/task/ImportPreview.java delete mode 100644 chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/task/ResumeDuplicatePolicy.java delete mode 100644 chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/task/ResumeState.java delete mode 100644 chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/TaskResumeJournal.java delete mode 100644 chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/ImportFileProbe.java delete mode 100644 chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/excel/ImportPreviewListener.java delete mode 100644 chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/ImportResumeRoundTripTest.java delete mode 100644 chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/TaskResumeJournalTest.java delete mode 100644 chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/export/CheckpointResumeExportTest.java delete mode 100644 chat2db-community-server/chat2db-community-storage/src/main/java/ai/chat2db/community/storage/task/H2TaskStorage.java delete mode 100644 chat2db-community-server/chat2db-community-storage/src/main/java/ai/chat2db/community/storage/task/TaskDatabase.java delete mode 100644 chat2db-community-server/chat2db-community-storage/src/main/java/ai/chat2db/community/storage/task/TaskRows.java delete mode 100644 chat2db-community-server/chat2db-community-storage/src/main/java/ai/chat2db/community/storage/task/TaskStorageConfiguration.java delete mode 100644 chat2db-community-server/chat2db-community-storage/src/main/java/ai/chat2db/community/storage/task/TaskStorageMigrator.java delete mode 100644 chat2db-community-server/chat2db-community-storage/src/test/java/ai/chat2db/community/storage/task/H2TaskStorageTest.java delete mode 100644 chat2db-community-server/chat2db-community-storage/src/test/java/ai/chat2db/community/storage/task/TaskStorageConfigurationTest.java delete mode 100644 chat2db-community-server/chat2db-community-storage/src/test/java/ai/chat2db/community/storage/task/TaskStorageMigrationTest.java diff --git a/chat2db-community-client/package.json b/chat2db-community-client/package.json index 557284f5d4..0d159049e5 100644 --- a/chat2db-community-client/package.json +++ b/chat2db-community-client/package.json @@ -12,7 +12,7 @@ "build": "yarn run build:web:community", "build:prod": "yarn run build:web:community", "build:web": "umi build", - "prebuild:web:community": "yarn test:community-boundary && yarn test:database-capabilities && yarn test:table-editor && yarn test:retired-ai && yarn test:tree-title-highlight && yarn test:tree-search-lifecycle && yarn test:driver-upload && yarn test:tree-loading && yarn test:tree-node-lookup && yarn test:tree-node-path && yarn test:data-source-authorization && yarn test:active-transactions && yarn test:data-source-mutation-refresh && yarn test:ai-model-config && yarn test:ai-model-select && yarn test:export-connections && yarn test:main-page-navigation && yarn test:shortcut && yarn test:console-tab-name && yarn test:workspace-tab-scroll && yarn test:workspace-split-lifecycle && yarn test:sql-completion-context && yarn test:file-manager-label && yarn test:local-file-encoding && yarn test:local-file-save && yarn test:local-file-tree-path && yarn test:saved-console-dirty && yarn test:editor-close && yarn test:result-set-ui && yarn test:result-pagination && yarn test:sql-execution-stream && yarn test:result-table-layout && yarn test:result-status && yarn test:data-source-identity && yarn test:data-source-execution-snapshot && yarn test:data-source-watermark && yarn test:hot-update && yarn test:task-center && yarn test:import-export-wizard && yarn test:import-preview && yarn test:application-exit && yarn test:monaco-lifecycle && yarn test:result-set-editor && yarn test:account-grants-request && yarn test:connection-close-request && yarn test:ssh-connection && yarn test:operation-log-filters && yarn test:dashboard-store", + "prebuild:web:community": "yarn test:community-boundary && yarn test:database-capabilities && yarn test:table-editor && yarn test:retired-ai && yarn test:tree-title-highlight && yarn test:tree-search-lifecycle && yarn test:driver-upload && yarn test:tree-loading && yarn test:tree-node-lookup && yarn test:tree-node-path && yarn test:data-source-authorization && yarn test:active-transactions && yarn test:data-source-mutation-refresh && yarn test:ai-model-config && yarn test:ai-model-select && yarn test:export-connections && yarn test:main-page-navigation && yarn test:shortcut && yarn test:console-tab-name && yarn test:workspace-tab-scroll && yarn test:workspace-split-lifecycle && yarn test:sql-completion-context && yarn test:file-manager-label && yarn test:local-file-encoding && yarn test:local-file-save && yarn test:local-file-tree-path && yarn test:saved-console-dirty && yarn test:editor-close && yarn test:result-set-ui && yarn test:result-pagination && yarn test:sql-execution-stream && yarn test:result-table-layout && yarn test:result-status && yarn test:data-source-identity && yarn test:data-source-execution-snapshot && yarn test:data-source-watermark && yarn test:hot-update && yarn test:task-center && yarn test:import-preview && yarn test:application-exit && yarn test:monaco-lifecycle && yarn test:result-set-editor && yarn test:account-grants-request && yarn test:connection-close-request && yarn test:ssh-connection && yarn test:operation-log-filters && yarn test:dashboard-store", "postbuild:web:community": "node ./scripts/verify-production-bundles.cjs", "build:web:2java": "yarn run build:web:community && rm -rf ../chat2db-community-server/chat2db-community-start/src/main/resources/thymeleaf/* && cp -r dist/index.html ../chat2db-community-server/chat2db-community-start/src/main/resources/thymeleaf/", "build:web:community": "cross-env UMI_ENV=community cross-env APP_NAME=chat2db-community cross-env APP_VERSION=${npm_config_app_version} cross-env PRINT_LOGS=${npm_config_print_logs} cross-env APP_PORT=${npm_config_app_port} umi build", @@ -24,7 +24,6 @@ "lint:eslint": "eslint \"src/**/*.{js,jsx,ts,tsx}\" --max-warnings=0", "lint:style": "stylelint \"src/**/*.{css,less}\" --max-warnings=0", "test:i18n": "node ./scripts/validate-i18n.cjs", - "test:import-export-wizard": "tsx src/blocks/ImportAndExport/components/ImportExportFile/taskParams.test.ts && tsx src/blocks/ImportAndExport/components/ImportExportFile/mappingValidation.test.ts", "test:community-boundary": "node ./scripts/verify-community-boundary.cjs", "test:database-capabilities": "tsx src/utils/databaseJudgments.test.ts", "test:table-editor": "tsx src/blocks/DatabaseTableEditor/ColumnList/normalizeColumn.test.ts", diff --git a/chat2db-community-client/scripts/i18n-source-hashes.json b/chat2db-community-client/scripts/i18n-source-hashes.json index 6aa2d937f1..096121a495 100644 --- a/chat2db-community-client/scripts/i18n-source-hashes.json +++ b/chat2db-community-client/scripts/i18n-source-hashes.json @@ -21,7 +21,7 @@ "spaceSetting.ts": "defdf3138e5e1782f0282760bcef4ce1b8550cc80b03052fea468d97165cb980", "sqlEditor.ts": "5c41e5f24d5ba78909795bcc681cabb8460cca075233b665dc4940afe82ce0c9", "stream.ts": "2e2fca0b1ba1ecd5a1501b767be8857cf0b72d800d69a4acf9cd62a61d5ceaf5", - "workspace.ts": "c24d9ace26288c5141b1bc1c1db82ba8d756ba0350864eddbe8864d753e344ea" + "workspace.ts": "f763878cad4b75850b139c8713fa9e8983acfe55f89cd346080439ea00276d3a" }, "ko-KR": { "ai.ts": "e7a2fcbfb3ed647bd14f63aca4b6c35c8fb402c49a6a1f36bb04f76f4e8415af", @@ -42,7 +42,7 @@ "spaceSetting.ts": "defdf3138e5e1782f0282760bcef4ce1b8550cc80b03052fea468d97165cb980", "sqlEditor.ts": "5c41e5f24d5ba78909795bcc681cabb8460cca075233b665dc4940afe82ce0c9", "stream.ts": "2e2fca0b1ba1ecd5a1501b767be8857cf0b72d800d69a4acf9cd62a61d5ceaf5", - "workspace.ts": "c24d9ace26288c5141b1bc1c1db82ba8d756ba0350864eddbe8864d753e344ea" + "workspace.ts": "f763878cad4b75850b139c8713fa9e8983acfe55f89cd346080439ea00276d3a" } } } diff --git a/chat2db-community-client/src/blocks/ImportAndExport/components/ExecutionModeControl/index.tsx b/chat2db-community-client/src/blocks/ImportAndExport/components/ExecutionModeControl/index.tsx new file mode 100644 index 0000000000..5fdbf535e8 --- /dev/null +++ b/chat2db-community-client/src/blocks/ImportAndExport/components/ExecutionModeControl/index.tsx @@ -0,0 +1,43 @@ +import { Modal, Space, Switch, Tooltip } from 'antd'; +import i18n from '@/i18n'; +import type { ImportExecutionMode } from '@/typings/importExport'; + +interface Props { + value: ImportExecutionMode; + onChange: (value: ImportExecutionMode) => void; + disabled?: boolean; + confirmImport?: boolean; +} + +export default function ExecutionModeControl({ value, onChange, disabled, confirmImport }: Props) { + const [modal, contextHolder] = Modal.useModal(); + + const toggle = (checked: boolean) => { + if (checked && confirmImport) { + modal.confirm({ + title: i18n('workspace.importExport.ultraModeConfirmTitle'), + content: i18n('workspace.importExport.ultraModeAcknowledge'), + okText: i18n('workspace.importExport.ultraModeConfirm'), + cancelText: i18n('common.button.cancel'), + onOk: () => onChange('ULTRA_FAST'), + }); + return; + } + onChange(checked ? 'ULTRA_FAST' : 'STANDARD'); + }; + + return ( + + {contextHolder} + + {i18n('workspace.importExport.ultraMode')} + + + + ); +} diff --git a/chat2db-community-client/src/blocks/ImportAndExport/components/ImportExportFile/index.tsx b/chat2db-community-client/src/blocks/ImportAndExport/components/ImportExportFile/index.tsx index eac5b790fe..dc47479750 100644 --- a/chat2db-community-client/src/blocks/ImportAndExport/components/ImportExportFile/index.tsx +++ b/chat2db-community-client/src/blocks/ImportAndExport/components/ImportExportFile/index.tsx @@ -1,296 +1,129 @@ import { memo, useMemo, useState, forwardRef, ForwardedRef, useImperativeHandle, useEffect } from 'react'; import { useStyles } from './style'; import UploadLocalFile, { type FileUrl } from '@/components/UploadLocalFile'; -import { Alert, Checkbox, Form, Input, Select, InputNumber, Switch, Table, Tooltip } from 'antd'; +import { Form, Input, Select } from 'antd'; import i18n from '@/i18n'; import { useImportExportStore } from '@/store/importExport'; import { IconButton } from '@chat2db/ui'; import { ImportExportType, ImportExportFileType, ImportExportTaskType } from '@/constants/importExport'; -import importExportServices, { ExportTaskParams, ImportTaskParams } from '@/service/importExport'; -import { IImportPreview, ImportExecutionMode } from '@/typings/importExport'; -import { isDesktop } from '@/utils/env'; +import { ExportTaskParams, ImportTaskParams } from '@/service/importExport'; +import { isDesktop, isDevelopment } from '@/utils/env'; import jcefApi from '@/jcef'; -import { CircleHelp } from 'lucide-react'; -import { buildTaskParams, initialFileType, type ImportExportFormValue } from './taskParams'; -import sqlService from '@/service/sql'; -import { stageSelectedImportFile } from '../ImportMappingContent/fileStaging'; -import { getImportPreviewErrorMessage } from '../ImportMappingContent/mapping'; -import { getImportMappingIssues, mergeImportColumnMappings } from './mappingValidation'; +import { hasSelectedImportFile } from './selection'; +import type { ImportExecutionMode } from '@/typings/importExport'; +import ExecutionModeControl from '../ExecutionModeControl'; interface IProps { className?: string; setIsReady?: (p: boolean) => void; + onImportFileChange?: (file?: FileUrl) => void; } export interface ImportExportFileRef { getValues: () => ExportTaskParams | ImportTaskParams | null; } +interface ImportExportFormValue { + exportType: ImportExportFileType; + containsHeader: boolean; + fileUrl?: string; +} + const exportTypeOptions = [ { label: 'CSV', value: ImportExportFileType.CSV, accept: '.csv' }, { label: 'XLSX', value: ImportExportFileType.XLSX, accept: '.xlsx' }, { label: 'XLS', value: ImportExportFileType.XLS, accept: '.xls' }, { label: 'JSON', value: ImportExportFileType.JSON, accept: '.json' }, - { label: 'NDJSON', value: ImportExportFileType.NDJSON, accept: '.ndjson' }, - { label: 'Markdown', value: ImportExportFileType.MARKDOWN, accept: '.md' }, { label: 'SQL', value: ImportExportFileType.SQL, accept: '.sql' }, ]; -// The import backend parses these formats; NDJSON/Markdown are export-only. -const importTypeOptions = exportTypeOptions.filter( - (option) => option.value !== ImportExportFileType.NDJSON && option.value !== ImportExportFileType.MARKDOWN, -); - -// Formats that can be checkpointed for resumable export. -const checkpointableFormats = [ - ImportExportFileType.CSV, - ImportExportFileType.NDJSON, - ImportExportFileType.MARKDOWN, - ImportExportFileType.SQL, -]; - const ImportExportFile = forwardRef((props: IProps, ref: ForwardedRef) => { - const { setIsReady } = props; + const { setIsReady, onImportFileChange } = props; const { styles } = useStyles(); const [form] = Form.useForm(); + const [mode, setMode] = useState('STANDARD'); + const [selectedFilePaths, setSelectedFilePaths] = useState([]); + const [exportLocation, setExportLocation] = useState(''); + const [formValue, setFormValue] = useState({ + exportType: ImportExportFileType.CSV, + containsHeader: true, + }); + const { importExportDataBoundInfo } = useImportExportStore((state) => { return { importExportDataBoundInfo: state.importExportDataBoundInfo, }; }); - const defaultFileType = importExportDataBoundInfo - ? initialFileType(importExportDataBoundInfo) - : ImportExportFileType.CSV; - const [selectedFile, setSelectedFile] = useState(); - const [stagedFile, setStagedFile] = useState<{ selection: FileUrl; id: string }>(); - const [previewLoading, setPreviewLoading] = useState(false); - const [importError, setImportError] = useState(); - const [exportLocation, setExportLocation] = useState(''); - const [importPreview, setImportPreview] = useState(null); - const [columnMappings, setColumnMappings] = useState>({}); - const [formValue, setFormValue] = useState({ - exportType: defaultFileType, - containsHeader: true, - }); - const [mode, setMode] = useState('STANDARD'); - const [confirmedNoStrongRelations, setConfirmedNoStrongRelations] = useState(false); const isImport = importExportDataBoundInfo?.type === ImportExportType.IMPORT; const isExport = importExportDataBoundInfo?.type === ImportExportType.EXPORT; - const isTableTarget = importExportDataBoundInfo?.targetScope === 'TABLE'; - const isSqlExport = isExport && !!importExportDataBoundInfo?.sqlExportScope; - const fileTypeOptions = importExportDataBoundInfo?.fileType - ? exportTypeOptions.filter((option) => option.value === importExportDataBoundInfo.fileType) - : isImport - ? importTypeOptions - : exportTypeOptions; useEffect(() => { if (importExportDataBoundInfo) { const { dataSourceName, databaseName, schemaName, tableName } = importExportDataBoundInfo; const tableNameDisplay = [dataSourceName, databaseName, schemaName, tableName].filter(Boolean).join('/'); - const exportType = initialFileType(importExportDataBoundInfo); - const initialValues: ImportExportFormValue = { - exportType, - containsHeader: true, - }; - setFormValue(initialValues); - setSelectedFile(undefined); - setStagedFile(undefined); - setImportError(undefined); - setExportLocation(''); - setImportPreview(null); - setColumnMappings({}); - setMode('STANDARD'); - setConfirmedNoStrongRelations(false); - form.resetFields(); form.setFieldsValue({ tableNameDisplay: tableNameDisplay, - ...initialValues, }); } - }, [form, importExportDataBoundInfo]); + }, [importExportDataBoundInfo]); // Gets the corresponding file type based on the export type const uploadLocalFileAccept = useMemo(() => { return formValue.exportType ? exportTypeOptions.find((item) => item.value === formValue.exportType)?.accept : ''; }, [formValue.exportType]); - // Both browser and desktop selections use the server's opaque staged-file contract. - useEffect(() => { - setStagedFile(undefined); - setImportPreview(null); - setImportError(undefined); - if (!selectedFile) return; - let cancelled = false; - stageSelectedImportFile(selectedFile, sqlService.uploadImportFile, sqlService.stageDesktopImportFile) - .then((id) => { - if (!cancelled) setStagedFile({ selection: selectedFile, id }); - }) - .catch((error) => { - if (!cancelled) setImportError(getImportPreviewErrorMessage(error, i18n('common.text.failure'))); - }); - return () => { - cancelled = true; - }; - }, [selectedFile]); - - // Previews the selected import file once both the file and the format are known, so the - // column mapping panel below reflects what the backend will actually import. - const fileId = stagedFile?.selection === selectedFile ? stagedFile?.id : undefined; - const previewableFormat = - isImport && - isTableTarget && - [ImportExportFileType.CSV, ImportExportFileType.XLS, ImportExportFileType.XLSX].includes(formValue.exportType); - useEffect(() => { - setImportPreview(null); - setPreviewLoading(false); - if (!previewableFormat || !fileId || !importExportDataBoundInfo) { - setImportPreview(null); - return () => {}; - } - let cancelled = false; - setPreviewLoading(true); - setImportError(undefined); - const { dataSourceId, databaseName, schemaName, tableName } = importExportDataBoundInfo; - importExportServices - .previewImport({ - dataSourceId, - databaseName, - schemaName, - taskType: ImportExportTaskType.DATA_FILE_IMPORT, - format: formValue.exportType, - tableName, - fileId, - mode, - confirmedNoStrongRelations, - options: - formValue.exportType === ImportExportFileType.CSV - ? { - charset: formValue.charset || undefined, - delimiter: formValue.delimiter || undefined, - quoteChar: formValue.quoteChar || undefined, - } - : undefined, - }) - .then((preview) => { - if (cancelled) return; - setImportPreview(preview); - setColumnMappings((current) => mergeImportColumnMappings(preview, current)); - }) - .catch((error) => { - if (!cancelled) setImportError(getImportPreviewErrorMessage(error, i18n('common.text.failure'))); - }) - .finally(() => { - if (!cancelled) setPreviewLoading(false); - }); - return () => { - cancelled = true; - }; - }, [ - previewableFormat, - fileId, - formValue.exportType, - formValue.charset, - formValue.delimiter, - formValue.quoteChar, - mode, - confirmedNoStrongRelations, - importExportDataBoundInfo, - ]); - - const targetColumnOptions = useMemo(() => { - if (!importPreview) return []; - const names = new Set(); - importPreview.columnMatches.forEach((match) => { - if (match.tableColumn) names.add(match.tableColumn); - }); - importPreview.missingTableColumns.forEach((name) => names.add(name)); - return Array.from(names) - .sort() - .map((name) => ({ label: name, value: name })); - }, [importPreview]); - - const mappingIssues = getImportMappingIssues(importPreview, columnMappings, formValue.unmappedTarget); - const admission = importPreview?.parallelAdmission; - const unsupportedParallelFormat = - isImport && mode === 'ULTRA_FAST' && formValue.exportType !== ImportExportFileType.CSV; - const parallelForbidden = - isImport && - mode === 'ULTRA_FAST' && - (unsupportedParallelFormat || !confirmedNoStrongRelations || admission?.verdict === 'PARALLEL_FORBIDDEN'); - const importReady = - !!fileId && - !importError && - !previewLoading && - !parallelForbidden && - (!previewableFormat || - (!!importPreview && !mappingIssues.duplicate && !mappingIssues.empty && !mappingIssues.required.length)); - useEffect(() => { - if (isImport) setIsReady?.(importReady); - }, [isImport, importReady, setIsReady]); - useEffect(() => { if (isExport) { - setIsReady?.(!isDesktop || !!exportLocation); + setIsReady?.(!isDesktop || !!exportLocation || !!formValue.fileUrl); } - }, [exportLocation, isExport, setIsReady, importExportDataBoundInfo]); + }, [exportLocation, formValue]); - const handleFileUrlListChange = (files: FileUrl[]) => { - setIsReady?.(false); - setColumnMappings({}); - setSelectedFile(files[0]); + const handleSelectedFilesChange = (files: FileUrl[]) => { + setSelectedFilePaths(files.map((item) => item.filePath).filter((path): path is string => !!path)); + if (isImport) { + setIsReady?.(hasSelectedImportFile(files)); + onImportFileChange?.(files[0]); + } }; useImperativeHandle(ref, () => ({ getValues: () => { if (!importExportDataBoundInfo) return null; - if (isImport && !importReady) return null; - const params = buildTaskParams({ - boundInfo: importExportDataBoundInfo, - formValue, - mode, - sourceFile: '', - exportLocation, - desktop: isDesktop, - importPreview, - columnMappings, - checkpointableFormats, - confirmedNoStrongRelations, - }); - return isImport - ? ({ - ...params, - sourceFile: undefined, - fileId, - displayFileName: selectedFile?.fileName || selectedFile?.file?.name, - } as ImportTaskParams) - : params; + const { dataSourceId, databaseName, schemaName, tableName } = importExportDataBoundInfo; + const commonValues = { + dataSourceId, + databaseName, + schemaName, + format: formValue.exportType, + }; + if (isExport) { + return { + ...commonValues, + taskType: ImportExportTaskType.TABLE_DATA_EXPORT, + mode, + tableNames: [tableName], + containsHeader: formValue.containsHeader, + exportPath: exportLocation || formValue.fileUrl, + }; + } + return { + ...commonValues, + taskType: + formValue.exportType === ImportExportFileType.SQL + ? ImportExportTaskType.SQL_FILE_IMPORT + : ImportExportTaskType.DATA_FILE_IMPORT, + tableName, + sourceFile: selectedFilePaths[0] || '', + }; }, })); const handleFormChange = (changedValues, allValues) => { - if (changedValues.exportType) { - setSelectedFile(undefined); - setColumnMappings({}); - setIsReady?.(false); - } - if (['charset', 'delimiter', 'quoteChar'].some((key) => key in changedValues)) { - setImportPreview(null); - setIsReady?.(false); - } - const nextValue: ImportExportFormValue = { + setFormValue({ ...formValue, ...allValues, - }; - if ( - changedValues.compression || - (changedValues.exportType && !checkpointableFormats.includes(changedValues.exportType)) - ) { - form.setFieldValue('checkpointRows', undefined); - nextValue.checkpointRows = undefined; - } - setFormValue(nextValue); + }); }; const handleSelectExportLocation = async () => { @@ -299,26 +132,6 @@ const ImportExportFile = forwardRef((props: IProps, ref: ForwardedRef { - setMode(checked ? 'ULTRA_FAST' : 'STANDARD'); - if (!checked) setConfirmedNoStrongRelations(false); - }; - - const ultraModeTooltip = ( -

-
{i18n('workspace.importExport.ultraModeHint')}
-
{i18n('workspace.importExport.ultraModeConfirmIntro')}
-
    -
  • {i18n('workspace.importExport.ultraModeBenefit1')}
    • -
  • {i18n('workspace.importExport.ultraModeBenefit2')}
    • -
  • {i18n('workspace.importExport.ultraModeBenefit3')}
    • -
  • {i18n('workspace.importExport.ultraModeRisk1')}
    • -
  • {i18n('workspace.importExport.ultraModeRisk2')}
    • -
  • {i18n('workspace.importExport.ultraModeRisk3')}
    • -
-
- ); - return ( + {isExport && ( + + + + )} - {isExport && isDesktop && (
- +
)} - {isExport && !isSqlExport && ( - <> - - - - )} - - )} {isImport && ( - - + + )} - {isImport && formValue.exportType === ImportExportFileType.CSV && ( - <> - - - - - - - - - - - - - - - - - - - - )} - {isImport && importError && } - {isImport && importPreview && ( -
-
{i18n('workspace.importExport.preview.columnMapping')}
- setColumnMappings((previous) => ({ ...previous, [match.fileColumn]: value }))} - /> -
- ))} - {mappingIssues.duplicate && ( - - )} - {mappingIssues.required.length > 0 && ( - column.name) - .join(', ')}`} - /> - )} - {mappingIssues.unmapped.length > 0 && ( -
- {i18n('workspace.importExport.preview.unmatchedColumns')}:{' '} - {mappingIssues.unmapped.map((column) => column.name).join(', ')} -
- )} -
- {i18n('workspace.importExport.dataPreview', importPreview.sampleRows.length)} -
- - size="small" - pagination={false} - scroll={{ x: 'max-content', y: 200 }} - rowKey="key" - dataSource={importPreview.sampleRows.map((values, key) => ({ key, values }))} - columns={importPreview.fileColumns.map((title, index) => ({ - title, - width: 160, - render: (_, row) => row.values[index], - }))} - /> -
- )} - - - - -
- - - - -
- - {isImport && mode === 'ULTRA_FAST' && ( -
- `[${finding.code}]`) - .join(', ')}. ${i18n('workspace.importExport.parallelAdmissionEffectiveMode')}: ${ - admission.effectiveMode - }` - : undefined - } - /> - setConfirmedNoStrongRelations(event.target.checked)} - > - {i18n('workspace.importExport.ultraModeAcknowledge')} - -
+ {isDevelopment && isExport && ( + + + )} -
- {mode === 'ULTRA_FAST' - ? i18n('workspace.importExport.modeBadgeUltra') - : i18n('workspace.importExport.modeBadgeStandard')} -
+ {isExport && } {/* {i18n('workspace.importExport.containsHeader')} */} diff --git a/chat2db-community-client/src/blocks/ImportAndExport/components/ImportExportFile/mappingValidation.test.ts b/chat2db-community-client/src/blocks/ImportAndExport/components/ImportExportFile/mappingValidation.test.ts deleted file mode 100644 index 0ac4ae22b0..0000000000 --- a/chat2db-community-client/src/blocks/ImportAndExport/components/ImportExportFile/mappingValidation.test.ts +++ /dev/null @@ -1,42 +0,0 @@ -import assert from 'node:assert/strict'; -import { getImportMappingIssues, mergeImportColumnMappings } from './mappingValidation'; -import type { IImportPreview } from '@/typings/importExport'; - -const preview: IImportPreview = { - fileColumns: ['Full Name', 'status'], - columnMatches: [], - sampleRows: [], - missingTableColumns: [], - targetColumns: [ - { name: 'id', dataType: 'INT', nullable: false, autoIncrement: true, comment: null, defaultValue: null }, - { name: 'name', dataType: 'VARCHAR', nullable: false, autoIncrement: false, comment: null, defaultValue: null }, - { name: 'status', dataType: 'VARCHAR', nullable: false, autoIncrement: false, comment: null, defaultValue: "'NEW'" }, - ], -}; -assert.deepEqual(getImportMappingIssues(preview, { 'Full Name': 'name' }).required, []); -assert.deepEqual( - getImportMappingIssues(preview, { 'Full Name': 'name' }, 'NULL').required.map((column) => column.name), - ['status'], -); -assert.equal(getImportMappingIssues(preview, { 'Full Name': 'name', status: 'NAME' }).duplicate, true); -assert.equal(getImportMappingIssues(preview, { 'Full Name': undefined }).empty, true); -assert.deepEqual( - getImportMappingIssues(preview, { status: 'status' }).required.map((column) => column.name), - ['name'], -); -const refreshedPreview: IImportPreview = { - ...preview, - columnMatches: [ - { fileColumn: 'Full Name', matched: false }, - { fileColumn: 'status', tableColumn: 'status', matched: true }, - ], -}; -assert.deepEqual(mergeImportColumnMappings(refreshedPreview, { 'Full Name': 'name', status: undefined }), { - 'Full Name': 'name', - status: undefined, -}); -assert.deepEqual(mergeImportColumnMappings(refreshedPreview, { 'Full Name': 'removed_column' }), { - 'Full Name': undefined, - status: 'status', -}); -console.log('Import mapping validation and refresh-preservation checks passed'); diff --git a/chat2db-community-client/src/blocks/ImportAndExport/components/ImportExportFile/mappingValidation.ts b/chat2db-community-client/src/blocks/ImportAndExport/components/ImportExportFile/mappingValidation.ts deleted file mode 100644 index 400ff35722..0000000000 --- a/chat2db-community-client/src/blocks/ImportAndExport/components/ImportExportFile/mappingValidation.ts +++ /dev/null @@ -1,40 +0,0 @@ -import type { IImportPreview } from '@/typings/importExport'; - -export const mergeImportColumnMappings = ( - preview: IImportPreview, - previous: Record, -) => { - const validTargets = new Set(preview.targetColumns.map((column) => column.name)); - return Object.fromEntries( - preview.columnMatches.map((match) => { - if (Object.prototype.hasOwnProperty.call(previous, match.fileColumn)) { - const previousTarget = previous[match.fileColumn]; - if (!previousTarget || validTargets.has(previousTarget)) { - return [match.fileColumn, previousTarget]; - } - } - return [match.fileColumn, match.matched ? match.tableColumn : undefined]; - }), - ); -}; - -export const getImportMappingIssues = ( - preview: IImportPreview | null, - mapping: Record, - strategy: 'DEFAULT' | 'NULL' = 'DEFAULT', -) => { - const targets = Object.values(mapping).filter((target): target is string => !!target); - const normalizedTargets = targets.map((target) => target.trim().toLowerCase()); - return { - duplicate: new Set(normalizedTargets).size !== targets.length, - empty: targets.length === 0, - required: (preview?.targetColumns || []).filter( - (column) => - !column.nullable && - !column.autoIncrement && - !targets.includes(column.name) && - (strategy === 'NULL' || column.defaultValue == null), - ), - unmapped: (preview?.targetColumns || []).filter((column) => !targets.includes(column.name)), - }; -}; diff --git a/chat2db-community-client/src/blocks/ImportAndExport/components/ImportExportFile/style.ts b/chat2db-community-client/src/blocks/ImportAndExport/components/ImportExportFile/style.ts index 5b6787a5d9..d6c5cd8a46 100644 --- a/chat2db-community-client/src/blocks/ImportAndExport/components/ImportExportFile/style.ts +++ b/chat2db-community-client/src/blocks/ImportAndExport/components/ImportExportFile/style.ts @@ -1,6 +1,6 @@ import { createStyles } from 'antd-style'; -export const useStyles = createStyles(({ css, token }) => { +export const useStyles = createStyles(({ css }) => { return { checkboxBody: css` .ant-form-item { @@ -20,115 +20,7 @@ export const useStyles = createStyles(({ css, token }) => { border-radius: 6px !important; `, form: css` - display: grid; - grid-template-columns: repeat(2, minmax(0, 1fr)); - column-gap: 16px; padding-top: 20px; - - @media (max-width: 640px) { - grid-template-columns: minmax(0, 1fr); - } - `, - fullWidth: css` - grid-column: 1 / -1; - `, - previewPanel: css` - grid-column: 1 / -1; - min-width: 0; - display: flex; - flex-direction: column; - gap: 8px; - padding: 12px; - border: 1px solid rgba(128, 128, 128, 0.25); - border-radius: 8px; - margin-bottom: 16px; - `, - previewTitle: css` - font-weight: 600; - `, - previewMeta: css` - display: flex; - gap: 16px; - color: rgba(128, 128, 128, 1); - font-size: 12px; - `, - previewRow: css` - display: flex; - align-items: center; - gap: 8px; - .ant-select { - flex: 1; - } - `, - previewFileColumn: css` - width: 120px; - overflow: hidden; - text-overflow: ellipsis; - white-space: nowrap; - `, - previewWarning: css` - color: rgba(230, 162, 60, 1); - font-size: 12px; - `, - modeControl: css` - display: flex; - align-items: center; - gap: 8px; - min-height: 22px; - `, - modeHelpButton: css` - display: inline-flex; - align-items: center; - justify-content: center; - width: 20px; - height: 20px; - padding: 0; - color: ${token.colorTextTertiary}; - cursor: help; - background: transparent; - border: 0; - - &:hover { - color: ${token.colorPrimary}; - } - - &:focus-visible { - outline: 2px solid currentColor; - outline-offset: 2px; - } - `, - modeTooltip: css` - width: min(400px, calc(100vw - 48px)); - - ul { - padding-left: 18px; - margin: 6px 0 0; - } - - li + li { - margin-top: 4px; - } - `, - modeTooltipTitle: css` - margin-top: 8px; - font-weight: 600; - `, - modeIndicator: css` - grid-column: 1 / -1; - margin: -8px 0 12px; - color: rgba(128, 128, 128, 0.75); - font-size: 12px; - `, - admissionPanel: css` - grid-column: 1 / -1; - display: flex; - flex-direction: column; - gap: 10px; - margin: -6px 0 12px; - - .ant-alert-description { - white-space: pre-line; - } `, }; }); diff --git a/chat2db-community-client/src/blocks/ImportAndExport/components/ImportExportFile/taskParams.test.ts b/chat2db-community-client/src/blocks/ImportAndExport/components/ImportExportFile/taskParams.test.ts deleted file mode 100644 index 002f486d71..0000000000 --- a/chat2db-community-client/src/blocks/ImportAndExport/components/ImportExportFile/taskParams.test.ts +++ /dev/null @@ -1,149 +0,0 @@ -import assert from 'node:assert/strict'; -import { ImportExportFileType, ImportExportTaskType, ImportExportType } from '@/constants/importExport'; -import type { ImportExportDataBoundInfo } from '@/typings/importExport'; -import { buildTaskParams, initialFileType, type ImportExportFormValue } from './taskParams'; - -const checkpointableFormats = [ - ImportExportFileType.CSV, - ImportExportFileType.NDJSON, - ImportExportFileType.MARKDOWN, - ImportExportFileType.SQL, -]; -const baseForm: ImportExportFormValue = { - exportType: ImportExportFileType.CSV, - containsHeader: true, -}; - -function build(boundInfo: ImportExportDataBoundInfo, overrides: Partial = {}) { - return buildTaskParams({ - boundInfo, - formValue: { ...baseForm, ...overrides }, - mode: 'ULTRA_FAST', - sourceFile: 'C:\\imports\\dump.sql', - exportLocation: 'C:\\exports', - desktop: true, - importPreview: null, - columnMappings: {}, - checkpointableFormats, - }); -} - -const tableExport: ImportExportDataBoundInfo = { - dataSourceId: 1, - databaseName: 'app', - schemaName: 'public', - tableName: 'orders', - targetScope: 'TABLE', - type: ImportExportType.EXPORT, -}; -assert.equal(initialFileType(tableExport), ImportExportFileType.CSV); -assert.deepEqual(build(tableExport, { checkpointRows: 10000 }), { - dataSourceId: 1, - databaseName: 'app', - schemaName: 'public', - format: ImportExportFileType.CSV, - mode: 'ULTRA_FAST', - taskType: ImportExportTaskType.TABLE_DATA_EXPORT, - tableNames: ['orders'], - containsHeader: true, - exportPath: 'C:\\exports', - compression: undefined, - checkpointRows: 10000, -}); - -const schemaExport: ImportExportDataBoundInfo = { - dataSourceId: 1, - databaseName: 'app', - schemaName: 'reporting', - targetScope: 'SCHEMA', - type: ImportExportType.EXPORT, - fileType: ImportExportFileType.SQL, - sqlExportScope: 'ALL', -}; -assert.equal(initialFileType(schemaExport), ImportExportFileType.SQL); -assert.deepEqual(build(schemaExport, { exportType: ImportExportFileType.SQL, compression: 'GZIP' }), { - dataSourceId: 1, - databaseName: 'app', - schemaName: 'reporting', - format: ImportExportFileType.SQL, - mode: 'ULTRA_FAST', - taskType: ImportExportTaskType.SQL_EXPORT, - tableNames: undefined, - scope: 'ALL', - containData: true, - containsHeader: true, - exportPath: 'C:\\exports', -}); - -const databaseImport: ImportExportDataBoundInfo = { - dataSourceId: 1, - databaseName: 'app', - targetScope: 'DATABASE', - type: ImportExportType.IMPORT, - fileType: ImportExportFileType.SQL, -}; -assert.deepEqual(build(databaseImport, { exportType: ImportExportFileType.SQL }), { - dataSourceId: 1, - databaseName: 'app', - schemaName: undefined, - format: ImportExportFileType.SQL, - mode: 'ULTRA_FAST', - taskType: ImportExportTaskType.SQL_FILE_IMPORT, - tableName: undefined, - sourceFile: 'C:\\imports\\dump.sql', - options: undefined, -}); - -const tableImport: ImportExportDataBoundInfo = { - ...tableExport, - type: ImportExportType.IMPORT, -}; -const mappedImport = buildTaskParams({ - boundInfo: tableImport, - formValue: { ...baseForm, delimiter: ';', onError: 'SKIP', maxErrors: 5 }, - mode: 'STANDARD', - sourceFile: 'C:\\imports\\orders.csv', - exportLocation: '', - desktop: true, - importPreview: { - fileColumns: ['order_id'], - columnMatches: [], - missingTableColumns: [], - sampleRows: [], - }, - columnMappings: { order_id: 'id' }, - checkpointableFormats, -}); -assert.equal(mappedImport.taskType, ImportExportTaskType.DATA_FILE_IMPORT); -assert.equal('tableName' in mappedImport ? mappedImport.tableName : undefined, 'orders'); -assert.deepEqual('options' in mappedImport ? mappedImport.options : undefined, { - charset: undefined, - delimiter: ';', - quoteChar: undefined, - skipRows: undefined, - nullString: undefined, - onError: 'SKIP', - maxErrors: 5, - columnMappings: [{ sourceColumn: 'order_id', targetColumn: 'id' }], -}); - -const confirmedParallelImport = buildTaskParams({ - boundInfo: tableImport, - formValue: baseForm, - mode: 'ULTRA_FAST', - sourceFile: 'C:\\imports\\orders.csv', - exportLocation: '', - desktop: true, - importPreview: null, - columnMappings: {}, - checkpointableFormats, - confirmedNoStrongRelations: true, -}); -assert.equal( - 'confirmedNoStrongRelations' in confirmedParallelImport - ? confirmedParallelImport.confirmedNoStrongRelations - : undefined, - true, -); - -console.log('Import/export wizard task parameter tests passed'); diff --git a/chat2db-community-client/src/blocks/ImportAndExport/components/ImportExportFile/taskParams.ts b/chat2db-community-client/src/blocks/ImportAndExport/components/ImportExportFile/taskParams.ts deleted file mode 100644 index 271a9f7330..0000000000 --- a/chat2db-community-client/src/blocks/ImportAndExport/components/ImportExportFile/taskParams.ts +++ /dev/null @@ -1,130 +0,0 @@ -import { ImportExportFileType, ImportExportTaskType, ImportExportType } from '@/constants/importExport'; -import type { ExportTaskParams, ImportTaskParams } from '@/service/importExport'; -import type { - IImportColumnMapping, - IImportOptions, - IImportPreview, - ImportExecutionMode, - ImportExportDataBoundInfo, -} from '@/typings/importExport'; - -export interface ImportExportFormValue { - exportType: ImportExportFileType; - containsHeader: boolean; - compression?: string; - checkpointRows?: number; - charset?: string; - delimiter?: string; - quoteChar?: string; - skipRows?: number; - nullString?: string; - onError?: 'ABORT' | 'SKIP'; - maxErrors?: number; - resumeDuplicatePolicy?: 'RECONCILE' | 'REJECT' | 'FAIL'; - unmappedTarget?: 'DEFAULT' | 'NULL'; -} - -interface BuildTaskParamsInput { - boundInfo: ImportExportDataBoundInfo; - formValue: ImportExportFormValue; - mode: ImportExecutionMode; - sourceFile: string; - exportLocation: string; - desktop: boolean; - importPreview: IImportPreview | null; - columnMappings: Record; - checkpointableFormats: ImportExportFileType[]; - confirmedNoStrongRelations?: boolean; -} - -export function initialFileType(boundInfo: ImportExportDataBoundInfo): ImportExportFileType { - if (boundInfo.fileType) return boundInfo.fileType; - return boundInfo.targetScope === 'TABLE' ? ImportExportFileType.CSV : ImportExportFileType.SQL; -} - -export function buildTaskParams({ - boundInfo, - formValue, - mode, - sourceFile, - exportLocation, - desktop, - importPreview, - columnMappings, - checkpointableFormats, - confirmedNoStrongRelations = false, -}: BuildTaskParamsInput): ExportTaskParams | ImportTaskParams { - const { dataSourceId, databaseName, schemaName, tableName } = boundInfo; - const commonValues = { - dataSourceId, - databaseName, - schemaName, - format: formValue.exportType, - mode, - }; - - if (boundInfo.type === ImportExportType.EXPORT) { - if (boundInfo.sqlExportScope) { - return { - ...commonValues, - taskType: ImportExportTaskType.SQL_EXPORT, - format: ImportExportFileType.SQL, - tableNames: tableName ? [tableName] : undefined, - scope: boundInfo.sqlExportScope, - containData: boundInfo.sqlExportScope === 'ALL', - containsHeader: formValue.containsHeader, - exportPath: desktop ? exportLocation : undefined, - }; - } - return { - ...commonValues, - taskType: ImportExportTaskType.TABLE_DATA_EXPORT, - tableNames: tableName ? [tableName] : undefined, - containsHeader: formValue.containsHeader, - exportPath: desktop ? exportLocation : undefined, - compression: formValue.compression || undefined, - checkpointRows: - !formValue.compression && checkpointableFormats.includes(formValue.exportType) - ? formValue.checkpointRows || undefined - : undefined, - }; - } - - const mappingList: IImportColumnMapping[] | undefined = importPreview - ? Object.entries(columnMappings) - .filter((entry) => !!entry[1]) - .map(([sourceColumn, targetColumn]) => ({ sourceColumn, targetColumn: targetColumn as string })) - : undefined; - const options: IImportOptions | undefined = - formValue.exportType === ImportExportFileType.SQL - ? undefined - : { - ...(formValue.exportType === ImportExportFileType.CSV - ? { - charset: formValue.charset || undefined, - delimiter: formValue.delimiter || undefined, - quoteChar: formValue.quoteChar || undefined, - skipRows: formValue.skipRows || undefined, - nullString: formValue.nullString || undefined, - onError: formValue.onError || undefined, - maxErrors: formValue.onError === 'SKIP' ? formValue.maxErrors || undefined : undefined, - } - : {}), - ...(formValue.resumeDuplicatePolicy - ? { resumeDuplicatePolicy: formValue.resumeDuplicatePolicy } - : {}), - ...(mappingList ? { columnMappings: mappingList } : {}), - }; - return { - ...commonValues, - taskType: - formValue.exportType === ImportExportFileType.SQL - ? ImportExportTaskType.SQL_FILE_IMPORT - : ImportExportTaskType.DATA_FILE_IMPORT, - tableName: boundInfo.targetScope === 'TABLE' ? tableName : undefined, - sourceFile, - ...(mode === 'ULTRA_FAST' && confirmedNoStrongRelations ? { confirmedNoStrongRelations: true } : {}), - options, - ...(importPreview ? { unmappedTarget: formValue.unmappedTarget || 'DEFAULT' } : {}), - }; -} diff --git a/chat2db-community-client/src/blocks/ImportAndExport/components/ImportFileModal/index.tsx b/chat2db-community-client/src/blocks/ImportAndExport/components/ImportFileModal/index.tsx index 843d311ca5..0cc865f1ad 100644 --- a/chat2db-community-client/src/blocks/ImportAndExport/components/ImportFileModal/index.tsx +++ b/chat2db-community-client/src/blocks/ImportAndExport/components/ImportFileModal/index.tsx @@ -5,28 +5,38 @@ import i18n from '@/i18n'; import ImportExportFile, { ImportExportFileRef } from '../ImportExportFile'; import { useImportExportStore } from '@/store/importExport'; import ModalFooterButton from '@/components/Modal/ModalFooterButton'; -import importExportServices, { type ExportTaskParams, type ImportTaskParams } from '@/service/importExport'; +import importExportServices from '@/service/importExport'; import { ImportExportTaskStatus, ImportExportType } from '@/constants/importExport'; import Log from '@/blocks/ImportAndExport/components/Log'; import { ImportExportTaskDetails } from '@/typings/importExport'; +import ImportMappingContent from '@/blocks/ImportAndExport/components/ImportMappingContent'; import jcefApi from '@/jcef'; import { isDesktop } from '@/utils/env'; +import sqlService from '@/service/sql'; +import { prepareImportParams } from './submission'; import { IMPORT_TARGET_TABLE_REFRESH_EVENT, shouldRefreshImportTargetTable, } from '@/store/importExport/taskCenterUtils'; +import type { FileUrl } from '@/components/UploadLocalFile'; interface IProps { className?: string; } +const isPreviewFile = (file?: FileUrl) => { + const name = (file?.fileName || file?.file?.name)?.toLowerCase(); + return name?.endsWith('.csv') || name?.endsWith('.xls') || name?.endsWith('.xlsx'); +}; + export default memo((_props) => { const [isReady, setIsReady] = useState(false); + const [submitting, setSubmitting] = useState(false); const importExportFileRef = useRef(null); + const previousTaskDetailsRef = useRef(); const [taskId, setTaskId] = useState(); const [taskDetails, setTaskDetails] = useState(); - const previousTaskDetailsRef = useRef(); - const [submitting, setSubmitting] = useState(false); + const [importFile, setImportFile] = useState(); const { importExportDataBoundInfo, setImportExportDataBoundInfo, getTaskList } = useImportExportStore((state) => { return { @@ -38,29 +48,41 @@ export default memo((_props) => { useEffect(() => { if (!importExportDataBoundInfo) { - setIsReady(false); setTaskId(undefined); setTaskDetails(undefined); previousTaskDetailsRef.current = undefined; + setImportFile(undefined); } }, [importExportDataBoundInfo]); - const handleRunSQl = () => { + const handleRunSQl = async () => { if (submitting) return; const params = importExportFileRef.current?.getValues(); if (!params) return; setSubmitting(true); - const request = - params.taskType === 'DATA_FILE_IMPORT' || params.taskType === 'SQL_FILE_IMPORT' - ? importExportServices.submitImport(params as ImportTaskParams) - : importExportServices.submitExport(params as ExportTaskParams); - request - .then((res) => { - setTaskId(res.taskId); - getTaskList(); - }) - .catch(() => {}) - .finally(() => setSubmitting(false)); + try { + let response; + if ('sourceFile' in params) { + let importParams = params; + if (!importFile) return; + importParams = await prepareImportParams( + importParams, importFile, sqlService.uploadImportFile, sqlService.stageDesktopImportFile, + ); + response = await importExportServices.submitImport(importParams); + } else { + response = await importExportServices.submitExport(params); + } + setTaskId(response.taskId); + getTaskList(); + } catch { + // Request helpers display the submission error. + } finally { + setSubmitting(false); + } + }; + + const handleImportFileChange = (file?: FileUrl) => { + setImportFile(file); }; const renderFooter = () => { @@ -120,41 +142,50 @@ export default memo((_props) => { ); const handleTaskChange = (_taskDetails: ImportExportTaskDetails) => { - const previous = previousTaskDetailsRef.current; + const previousTask = previousTaskDetailsRef.current; previousTaskDetailsRef.current = _taskDetails; setTaskDetails(_taskDetails); - if (shouldRefreshImportTargetTable(previous, _taskDetails)) { - window.dispatchEvent(new CustomEvent(IMPORT_TARGET_TABLE_REFRESH_EVENT, { detail: _taskDetails.target })); + if (shouldRefreshImportTargetTable(previousTask, _taskDetails)) { + window.dispatchEvent( + new CustomEvent(IMPORT_TARGET_TABLE_REFRESH_EVENT, { + detail: _taskDetails.target, + }), + ); void getTaskList(); } }; - const modalTitle = (() => { - if (importExportDataBoundInfo?.type === ImportExportType.IMPORT) { - return importExportDataBoundInfo.targetScope === 'TABLE' - ? i18n('workspace.menu.importData') - : i18n('workspace.menu.runSqlFile'); - } - if (importExportDataBoundInfo?.sqlExportScope === 'SCHEMA') { - return i18n('workspace.menu.exportStructure'); - } - if (importExportDataBoundInfo?.sqlExportScope === 'ALL') { - return i18n('workspace.menu.exportStructureData'); - } - return i18n('workspace.menu.exportData'); - })(); + const importPreviewContext = + importExportDataBoundInfo?.type === ImportExportType.IMPORT && + isPreviewFile(importFile) && + importExportDataBoundInfo.dataSourceId != null && + importExportDataBoundInfo.databaseName != null && + importFile != null + ? { + dataSourceId: importExportDataBoundInfo.dataSourceId, + databaseName: importExportDataBoundInfo.databaseName, + schemaName: importExportDataBoundInfo.schemaName, + tableName: importExportDataBoundInfo.tableName || '', + file: importFile, + } + : null; + const showImportPreview = taskId == null && importPreviewContext != null; return ( { setImportExportDataBoundInfo(null); @@ -162,8 +193,24 @@ export default memo((_props) => { > {taskId ? ( + ) : importPreviewContext ? ( + { + setTaskId(submittedTaskId); + getTaskList(); + }} + /> ) : ( - + )} ); diff --git a/chat2db-community-client/src/blocks/ImportAndExport/components/ImportFileModal/submission.test.ts b/chat2db-community-client/src/blocks/ImportAndExport/components/ImportFileModal/submission.test.ts index 10599d267f..7fe79a19b2 100644 --- a/chat2db-community-client/src/blocks/ImportAndExport/components/ImportFileModal/submission.test.ts +++ b/chat2db-community-client/src/blocks/ImportAndExport/components/ImportFileModal/submission.test.ts @@ -1,25 +1,38 @@ import assert from 'node:assert/strict'; import { ImportExportFileType, ImportExportTaskType } from '@/constants/importExport'; -import { prepareWebImportParams } from './submission'; +import { prepareImportParams } from './submission'; -const file = { name: 'users.json' } as File; -const params: import('@/service/importExport').ImportTaskParams = { - dataSourceId: 1, - databaseName: 'app', - tableName: 'users', - taskType: ImportExportTaskType.DATA_FILE_IMPORT, - format: ImportExportFileType.JSON, - sourceFile: '', -}; +async function run() { + const file = { name: 'users.json' } as File; + const params: import('@/service/importExport').ImportTaskParams = { + dataSourceId: 1, + databaseName: 'app', + tableName: 'users', + taskType: ImportExportTaskType.DATA_FILE_IMPORT, + format: ImportExportFileType.JSON, + sourceFile: 'untrusted-path', + }; + const browser = await prepareImportParams(params, { file }, async ({ file: uploaded }) => { + assert.equal(uploaded, file); + return 'browser-file-id'; + }, async () => { throw new Error('Browser uploads must not stage local paths'); }); + assert.deepEqual(browser, { + ...params, sourceFile: undefined, fileId: 'browser-file-id', displayFileName: 'users.json', + }); -prepareWebImportParams(params, file, async ({ file: uploadedFile }) => { - assert.equal(uploadedFile, file); - return 'staged-file-id'; -}).then((prepared) => { - assert.deepEqual(prepared, { - ...params, - sourceFile: undefined, - fileId: 'staged-file-id', - displayFileName: 'users.json', + const desktop = await prepareImportParams(params, + { filePath: '/imports/users.json', fileName: 'users.json' }, + async () => { throw new Error('Desktop selections must not upload a browser File'); }, + async (request) => { + assert.deepEqual(request, { sourceFile: '/imports/users.json', originalFileName: 'users.json' }); + return 'desktop-file-id'; + }); + assert.deepEqual(desktop, { + ...params, sourceFile: undefined, fileId: 'desktop-file-id', displayFileName: 'users.json', }); -}); + await assert.rejects(prepareImportParams(params, {}, + async () => { throw new Error('Unexpected upload'); }, + async () => { throw new Error('Unexpected local staging'); }), /selection is incomplete/); +} + +run().catch((error) => { console.error(error); process.exitCode = 1; }); diff --git a/chat2db-community-client/src/blocks/ImportAndExport/components/ImportFileModal/submission.ts b/chat2db-community-client/src/blocks/ImportAndExport/components/ImportFileModal/submission.ts index db92755c98..f7eb999ea3 100644 --- a/chat2db-community-client/src/blocks/ImportAndExport/components/ImportFileModal/submission.ts +++ b/chat2db-community-client/src/blocks/ImportAndExport/components/ImportFileModal/submission.ts @@ -1,14 +1,18 @@ -import { ImportTaskParams } from '@/service/importExport'; +import type { FileUrl } from '@/components/UploadLocalFile'; +import type { ImportTaskParams } from '@/service/importExport'; +import { stageSelectedImportFile } from '../ImportMappingContent/fileStaging'; -type UploadImportFile = (params: { file: File }) => Promise; +type UploadBrowserFile = (params: { file: File }) => Promise; +type StageDesktopFile = (params: { sourceFile: string; originalFileName: string }) => Promise; -export const prepareWebImportParams = async ( +export const prepareImportParams = async ( params: ImportTaskParams, - file: File, - uploadImportFile: UploadImportFile, + file: FileUrl, + uploadBrowserFile: UploadBrowserFile, + stageDesktopFile: StageDesktopFile, ): Promise => ({ ...params, sourceFile: undefined, - fileId: await uploadImportFile({ file }), - displayFileName: file.name, + fileId: await stageSelectedImportFile(file, uploadBrowserFile, stageDesktopFile), + displayFileName: file.fileName || file.file?.name, }); diff --git a/chat2db-community-client/src/blocks/ImportAndExport/components/ImportMappingContent/index.tsx b/chat2db-community-client/src/blocks/ImportAndExport/components/ImportMappingContent/index.tsx index 42537405a9..a2d2e1f485 100644 --- a/chat2db-community-client/src/blocks/ImportAndExport/components/ImportMappingContent/index.tsx +++ b/chat2db-community-client/src/blocks/ImportAndExport/components/ImportMappingContent/index.tsx @@ -9,7 +9,7 @@ import { } from '@/constants/importExport'; import i18n from '@/i18n'; import sqlService, { IImportPreview } from '@/service/sql'; -import type { ICsvOptions } from '@/typings/importExport'; +import type { ICsvOptions, ImportExecutionMode } from '@/typings/importExport'; import { buildInitialImportMapping, getDuplicateImportMappings, @@ -19,6 +19,7 @@ import { useStyles } from './style'; import type { FileUrl } from '@/components/UploadLocalFile'; import { stageSelectedImportFile } from './fileStaging'; import CsvOptionsSections from './CsvOptionsSections'; +import ExecutionModeControl from '../ExecutionModeControl'; import useImportDataSections from './ImportDataSections'; import { buildCsvOptionsForTaskSubmit, @@ -51,6 +52,7 @@ const ImportMappingContent = ({ dataSourceId, databaseName, schemaName, tableNam const [mapping, setMapping] = useState>({}); const [unmappedTarget, setUnmappedTarget] = useState(ImportUnmappedTarget.DEFAULT); const [executing, setExecuting] = useState(false); + const [mode, setMode] = useState('STANDARD'); const [activeSections, setActiveSections] = useState(['mapping', 'preview']); const [csvOptions, setCsvOptions] = useState(DEFAULT_CSV_OPTIONS); const selectedFileName = file.fileName || file.file?.name || file.filePath || ''; @@ -76,6 +78,7 @@ const ImportMappingContent = ({ dataSourceId, databaseName, schemaName, tableNam useEffect(() => { let active = true; setFileId(undefined); + setMode('STANDARD'); setLoading(true); setError(null); stageSelectedImportFile(file, sqlService.uploadImportFile, sqlService.stageDesktopImportFile) @@ -227,6 +230,8 @@ const ImportMappingContent = ({ dataSourceId, databaseName, schemaName, tableNam .map(([source, target]) => ({ sourceColumn: source, targetColumn: target })), unmappedTarget, csvOptions: taskCsvOptions, + mode, + confirmedNoStrongRelations: mode === 'ULTRA_FAST', }) .then((result) => onSubmitted(result.taskId)) .catch((e) => setError(resolveErrorMessage(e))) @@ -275,6 +280,14 @@ const ImportMappingContent = ({ dataSourceId, databaseName, schemaName, tableNam
{preview && (
+ {isCsv && ( + + )} - {taskDetails?.status === ImportExportTaskStatus.PENDING && taskDetails?.stage === 'RESUMING' && ( - - )} {taskDetails?.status === ImportExportTaskStatus.SUCCESS && (taskDetails.artifacts?.length ? taskDetails.artifacts.map((artifact) => ( diff --git a/chat2db-community-client/src/blocks/ImportAndExport/components/RunSql/index.tsx b/chat2db-community-client/src/blocks/ImportAndExport/components/RunSql/index.tsx new file mode 100644 index 0000000000..67e9486263 --- /dev/null +++ b/chat2db-community-client/src/blocks/ImportAndExport/components/RunSql/index.tsx @@ -0,0 +1,127 @@ +import { memo, useState, forwardRef, ForwardedRef, useImperativeHandle, useEffect } from 'react'; +import { useStyles } from './style'; +import UploadLocalFile, { type FileUrl } from '@/components/UploadLocalFile'; +import { Form, Input } from 'antd'; +import i18n from '@/i18n'; +import { useImportExportStore } from '@/store/importExport'; +import { isDesktop, isDevelopment } from '@/utils/env'; +import { ImportExportFileType, ImportExportTaskType } from '@/constants/importExport'; +import { ImportTaskParams } from '@/service/importExport'; +import { hasSelectedImportFile } from '../ImportExportFile/selection'; + +interface IProps { + className?: string; + setIsReady?: (p: boolean) => void; +} + +export interface RunSqlRef { + getValues: () => ImportTaskParams | null; + getFile: () => FileUrl | undefined; +} + +// const codeOptions = [ +// { +// label: 'UTF-8', +// value: 'UTF-8', +// }, +// { +// label: 'GB2312', +// value: 'GB2312', +// }, +// ]; + +const RunSql = forwardRef((props: IProps, ref: ForwardedRef) => { + const { setIsReady } = props; + const { styles } = useStyles(); + const [form] = Form.useForm(); + const [fileUrlList, setFileUrlList] = useState([]); + const [formValues, setFormValues] = useState<{ fileUrl?: string }>({}); + + useEffect(() => { + setIsReady && setIsReady(hasSelectedImportFile(fileUrlList) || !!(isDesktop && formValues.fileUrl)); + }, [fileUrlList, formValues]); + + const { runSqlBoundInfo } = useImportExportStore((state) => { + return { + runSqlBoundInfo: state.runSqlBoundInfo, + }; + }); + + useEffect(() => { + if (!runSqlBoundInfo) return; + + const _executionEnvironment = [ + runSqlBoundInfo.dataSourceName, + runSqlBoundInfo.databaseName, + runSqlBoundInfo.schemaName, + ] + .filter(Boolean) + .join('/'); + + form.setFieldsValue({ + executionEnvironment: _executionEnvironment, + }); + }, [runSqlBoundInfo]); + + useImperativeHandle(ref, () => ({ + getFile: () => fileUrlList[0] || (isDesktop && formValues.fileUrl + ? { filePath: formValues.fileUrl, fileName: formValues.fileUrl.split(/[\\/]/).pop() } + : undefined), + getValues: () => { + if (!runSqlBoundInfo) return null; + const { dataSourceId, databaseName, schemaName } = runSqlBoundInfo; + return { + dataSourceId, + databaseName, + schemaName, + taskType: ImportExportTaskType.SQL_FILE_IMPORT, + sourceFile: fileUrlList[0]?.filePath || formValues.fileUrl, + format: ImportExportFileType.SQL, + }; + }, + })); + + const handleFileUrlListChange = (_fileUrlList: FileUrl[]) => { + setFileUrlList(_fileUrlList); + }; + + return ( + { + setFormValues(form.getFieldsValue()); + }} + > + + + + + + + {isDesktop && isDevelopment && ( + + + + )} + {/* + diff --git a/chat2db-community-client/src/blocks/ImportAndExport/components/RunSqlModal/index.tsx b/chat2db-community-client/src/blocks/ImportAndExport/components/RunSqlModal/index.tsx index 76c83acede..a6d6f3c52e 100644 --- a/chat2db-community-client/src/blocks/ImportAndExport/components/RunSqlModal/index.tsx +++ b/chat2db-community-client/src/blocks/ImportAndExport/components/RunSqlModal/index.tsx @@ -7,8 +7,6 @@ import { useImportExportStore } from '@/store/importExport'; import ModalFooterButton from '@/components/Modal/ModalFooterButton'; import importExportServices from '@/service/importExport'; import Log from '@/blocks/ImportAndExport/components/Log'; -import sqlService from '@/service/sql'; -import { prepareImportParams } from '../ImportFileModal/submission'; interface IProps { className?: string; @@ -16,7 +14,6 @@ interface IProps { export default memo((_props) => { const [isReady, setIsReady] = useState(false); - const [submitting, setSubmitting] = useState(false); const runSqlRef = useRef(null); const [taskId, setTaskId] = useState(); @@ -34,24 +31,13 @@ export default memo((_props) => { } }, [runSqlBoundInfo]); - const handleRunSQl = async () => { - if (submitting) return; + const handleRunSQl = () => { const params = runSqlRef.current?.getValues(); - const file = runSqlRef.current?.getFile(); - if (!params || !file) return; - setSubmitting(true); - try { - const prepared = await prepareImportParams( - params, file, sqlService.uploadImportFile, sqlService.stageDesktopImportFile, - ); - const result = await importExportServices.submitImport(prepared); - setTaskId(result.taskId); - void getTaskList(); - } catch { - // Request helpers display the submission error. - } finally { - setSubmitting(false); - } + if (!params) return; + importExportServices.submitImport(params).then((res) => { + setTaskId(res.taskId); + getTaskList(); + }); }; const renderFooter = () => { @@ -66,7 +52,7 @@ export default memo((_props) => { > {i18n('common.button.cancel')} - diff --git a/chat2db-community-server/chat2db-community-web/src/main/java/ai/chat2db/community/web/api/controller/TaskController.java b/chat2db-community-server/chat2db-community-web/src/main/java/ai/chat2db/community/web/api/controller/TaskController.java index 0949857707..a5da0a9c7d 100644 --- a/chat2db-community-server/chat2db-community-web/src/main/java/ai/chat2db/community/web/api/controller/TaskController.java +++ b/chat2db-community-server/chat2db-community-web/src/main/java/ai/chat2db/community/web/api/controller/TaskController.java @@ -18,7 +18,6 @@ import ai.chat2db.community.web.api.model.request.task.TaskImportRequest; import ai.chat2db.community.web.api.model.response.task.TaskSubmitResponse; import jakarta.validation.Valid; -import org.apache.commons.lang3.StringUtils; import org.springframework.core.io.Resource; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.DeleteMapping; @@ -60,9 +59,6 @@ public DataResult submitExport(@Valid @RequestBody TaskExpor @PostMapping("/import") public DataResult submitImport(@Valid @RequestBody TaskImportRequest request) { - if (StringUtils.isBlank(request.getFileId())) { - throw new ai.chat2db.community.tools.exception.ParamBusinessException("fileId"); - } Long taskId = importTaskSubmissionService.submit( taskWebConverter.importRequest2spec(request), request.getFileId()); return DataResult.of(new TaskSubmitResponse(taskId)); diff --git a/chat2db-community-server/chat2db-community-web/src/test/java/ai/chat2db/community/web/api/controller/TaskControllerDesktopContractTest.java b/chat2db-community-server/chat2db-community-web/src/test/java/ai/chat2db/community/web/api/controller/TaskControllerDesktopContractTest.java index df8e2d8e2b..b0cc81d2ac 100644 --- a/chat2db-community-server/chat2db-community-web/src/test/java/ai/chat2db/community/web/api/controller/TaskControllerDesktopContractTest.java +++ b/chat2db-community-server/chat2db-community-web/src/test/java/ai/chat2db/community/web/api/controller/TaskControllerDesktopContractTest.java @@ -20,16 +20,6 @@ class TaskControllerDesktopContractTest { - @Test - void submitRejectsRawServerPathsBeforeReadingFiles() { - TaskController controller = new TaskController(null, null, null, null); - var request = new ai.chat2db.community.web.api.model.request.task.TaskImportRequest(); - request.setSourceFile("C:/private/data.csv"); - org.junit.jupiter.api.Assertions.assertThrows( - ai.chat2db.community.tools.exception.ParamBusinessException.class, - () -> controller.submitImport(request)); - } - @Test void taskEndpointsUseStaticPathsAndAtMostOneRequestObject() { Set paths = Arrays.stream(TaskController.class.getDeclaredMethods()) From f7f3b0685d05e40bbea128f05f8dc5e741b5e067 Mon Sep 17 00:00:00 2001 From: zgq Date: Tue, 15 Sep 2026 11:00:21 +0800 Subject: [PATCH 28/38] fix(import): bound batches and preserve worker lifecycle and context --- .../service/task/TaskExecutionContext.java | 4 + .../core/impl/task/AdaptiveBatchSizer.java | 11 +- .../domain/core/impl/task/RunningTask.java | 22 +- .../impl/task/TaskExecutionContextImpl.java | 5 + .../domain/core/impl/task/TaskRunner.java | 2 +- .../task/extension/TaskExtensionManager.java | 12 +- .../task/imports/ImportColumnResolver.java | 31 +- .../impl/task/imports/ImportRowBatcher.java | 165 +++++---- .../impl/task/AdaptiveBatchSizerTest.java | 15 +- .../task/ParallelImportLifecycleTest.java | 341 ++++++++++++++++++ .../core/impl/task/RunningTaskTest.java | 21 ++ .../excel/CSVImporterColumnMappingTest.java | 23 ++ .../ai/chat2db/spi/sql/Chat2DBContext.java | 5 + 13 files changed, 543 insertions(+), 114 deletions(-) create mode 100644 chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/ParallelImportLifecycleTest.java diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/service/task/TaskExecutionContext.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/service/task/TaskExecutionContext.java index 5909e53549..d49f05d9bd 100644 --- a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/service/task/TaskExecutionContext.java +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/service/task/TaskExecutionContext.java @@ -26,6 +26,10 @@ default Long taskId() { void registerCancelable(TaskCancelable resource); + /** Cancels registered work after execution fails, without changing the task's failure status. */ + default void cancelResources() { + } + ArtifactDraft createArtifact(String outputDirectory, String fileName, String mediaType); void write(String content); diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/AdaptiveBatchSizer.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/AdaptiveBatchSizer.java index acba6eecbb..9feb48c77f 100644 --- a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/AdaptiveBatchSizer.java +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/AdaptiveBatchSizer.java @@ -7,8 +7,7 @@ * the sizer hill-climbs on the measured throughput (rows per second): a batch that beats the * running reference by {@link #GROW_MARGIN} doubles the size, one that falls short by * {@link #SHRINK_MARGIN} halves it. Sizes therefore follow what the machine and the target - * database actually sustain instead of a fixed guess, and there is deliberately no upper bound on - * growth - the throughput feedback is the only ceiling. {@link #MIN_BATCH} rows keeps a batch + * database actually sustain, bounded by {@link #MAX_BATCH}. {@link #MIN_BATCH} rows keeps a batch * worth sending even on the slowest target. */ public final class AdaptiveBatchSizer { @@ -16,6 +15,8 @@ public final class AdaptiveBatchSizer { /** Lowest batch the tuner will settle on (1 thread x 100 rows contract floor). */ private static final int MIN_BATCH = 100; + static final int MAX_BATCH = 50_000; + private static final double GROW_MARGIN = 1.10D; private static final double SHRINK_MARGIN = 0.90D; @@ -47,7 +48,7 @@ public synchronized void record(int rows, long nanos) { int current = batchSize.get(); if (referenceThroughput > 0.0D) { if (throughput > referenceThroughput * GROW_MARGIN) { - batchSize.set(clamp(current * 2)); + batchSize.set(clamp((long) current * 2)); } else if (throughput < referenceThroughput * SHRINK_MARGIN) { batchSize.set(clamp(current / 2)); } @@ -57,7 +58,7 @@ public synchronized void record(int rows, long nanos) { : REFERENCE_ALPHA * throughput + (1.0D - REFERENCE_ALPHA) * referenceThroughput; } - private int clamp(int value) { - return Math.max(MIN_BATCH, value); + private int clamp(long value) { + return (int) Math.max(MIN_BATCH, Math.min(MAX_BATCH, value)); } } diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/RunningTask.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/RunningTask.java index c46559578b..253c920d0a 100644 --- a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/RunningTask.java +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/RunningTask.java @@ -45,6 +45,8 @@ final class RunningTask { private volatile boolean closed; + private boolean resourcesCancelled; + RunningTask(Long taskId) { this(taskId, CANCELLATION_EXECUTOR); } @@ -81,7 +83,7 @@ boolean requestCancellation(boolean mayInterruptIfRunning) { return false; } currentFuture = future; - currentCancelables = List.copyOf(cancelables); + currentCancelables = cancelResourcesLocked(); } if (currentFuture != null) { currentFuture.cancel(mayInterruptIfRunning); @@ -90,13 +92,29 @@ boolean requestCancellation(boolean mayInterruptIfRunning) { return true; } + void cancelResources() { + List resources; + synchronized (cancellationLock) { + resources = cancelResourcesLocked(); + } + resources.forEach(this::cancelRegisteredResourceAsync); + } + + private List cancelResourcesLocked() { + if (resourcesCancelled) { + return List.of(); + } + resourcesCancelled = true; + return List.copyOf(cancelables); + } + void registerCancelable(TaskCancelable resource) { if (resource == null) { return; } boolean cancelImmediately; synchronized (cancellationLock) { - cancelImmediately = cancelables.add(resource) && cancellationToken.isCancelled(); + cancelImmediately = cancelables.add(resource) && resourcesCancelled; } if (cancelImmediately) { cancelRegisteredResourceAsync(resource); diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/TaskExecutionContextImpl.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/TaskExecutionContextImpl.java index 14f800feac..05fc2c2a6b 100644 --- a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/TaskExecutionContextImpl.java +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/TaskExecutionContextImpl.java @@ -96,6 +96,11 @@ public void registerCancelable(TaskCancelable resource) { runningTask.registerCancelable(resource); } + @Override + public void cancelResources() { + runningTask.cancelResources(); + } + @Override public synchronized ArtifactDraft createArtifact(String outputDirectory, String fileName, String mediaType) { checkCancelled(); diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/TaskRunner.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/TaskRunner.java index 58e83f9626..bbfc53cdb6 100644 --- a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/TaskRunner.java +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/TaskRunner.java @@ -67,7 +67,7 @@ public void run() { bindExecutionContext(); taskExtensionManager.runGuarded(submission.extensionContext(), () -> { try (Chat2DBContext.StatementGuardScope ignored = - Chat2DBContext.bindStatementGuard(taskExtensionManager::beforeStatement)) { + Chat2DBContext.bindStatementGuard(taskExtensionManager.captureStatementGuard())) { taskExecutor.execute(submission.spec(), executionContext); } }); diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/extension/TaskExtensionManager.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/extension/TaskExtensionManager.java index 24addc3fe6..2755332264 100644 --- a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/extension/TaskExtensionManager.java +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/extension/TaskExtensionManager.java @@ -8,6 +8,7 @@ import org.springframework.stereotype.Component; import java.util.List; +import java.util.function.Consumer; @Component public class TaskExtensionManager { @@ -41,11 +42,20 @@ public void runGuarded(TaskExecutionContext context, Runnable runnable) { } } + /** Captures the task so worker threads run the same statement guards. */ + public Consumer captureStatementGuard() { + TaskExecutionContext context = currentTask.get(); + return sql -> beforeStatement(context, sql); + } + public void beforeStatement(String sql) { + beforeStatement(currentTask.get(), sql); + } + + private void beforeStatement(TaskExecutionContext context, String sql) { if (executionGuards.isEmpty()) { return; } - TaskExecutionContext context = currentTask.get(); if (context == null) { throw new IllegalStateException("Task statement guard requires an active task context"); } diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/ImportColumnResolver.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/ImportColumnResolver.java index 7dcaaca827..189b6479a0 100644 --- a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/ImportColumnResolver.java +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/ImportColumnResolver.java @@ -14,7 +14,7 @@ /** * Resolves which file column feeds which table column. Explicit mappings win; otherwise matching is - * case-insensitive on trimmed names. + * case-insensitive, preserving whitespace as in the existing preview and ordinary importer. */ public final class ImportColumnResolver { @@ -66,8 +66,7 @@ private static Resolution resolve(List tableColumns, List f || StringUtils.isBlank(mapping.getTargetColumn())) { throw new ParamBusinessException("columnMappings"); } - Integer sourceIndex = indexOfSource(mapping.getSourceColumn().trim(), fileHeaders, - byNormalizedName); + Integer sourceIndex = byNormalizedName.get(normalize(mapping.getSourceColumn())); if (sourceIndex == null) { throw new ParamBusinessException("columnMappings source: " + mapping.getSourceColumn()); } @@ -103,31 +102,7 @@ private static Resolution resolve(List tableColumns, List f return new Resolution(resolvedColumns, fileIndexes, missingTableColumns); } - private static Integer indexOfSource(String source, List fileHeaders, - Map byNormalizedName) { - Integer namedIndex = byNormalizedName.get(normalize(source)); - if (namedIndex != null) { - return namedIndex; - } - try { - int index = Integer.parseInt(source); - return index >= 0 && index < fileHeaders.size() ? index : null; - } catch (NumberFormatException ignored) { - return byNormalizedName.get(normalize(source)); - } - } - - /** - * Case-insensitive match on trimmed names, ignoring a leading UTF-8 BOM. - */ private static String normalize(String name) { - if (name == null) { - return ""; - } - String trimmed = name; - if (!trimmed.isEmpty() && trimmed.charAt(0) == '\ufeff') { - trimmed = trimmed.substring(1); - } - return trimmed.trim().toLowerCase(java.util.Locale.ROOT); + return name == null ? "" : name.toUpperCase(java.util.Locale.ROOT); } } diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/ImportRowBatcher.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/ImportRowBatcher.java index 7a57b59808..a75db3a158 100644 --- a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/ImportRowBatcher.java +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/ImportRowBatcher.java @@ -7,6 +7,9 @@ import ai.chat2db.community.domain.api.model.task.TaskCancelledException; import ai.chat2db.community.domain.api.model.task.TaskErrorCode; import ai.chat2db.community.domain.api.model.task.TaskExecutionException; +import ai.chat2db.community.domain.api.model.task.TaskStage; +import ai.chat2db.community.tools.model.Context; +import ai.chat2db.community.tools.util.ContextUtils; import ai.chat2db.community.domain.api.model.value.SQLDataValue; import ai.chat2db.community.domain.api.service.task.TaskExecutionContext; import ai.chat2db.community.domain.core.impl.task.AdaptiveBatchSizer; @@ -21,6 +24,8 @@ import ai.chat2db.spi.sql.Chat2DBContext; import ai.chat2db.spi.sql.ConnectionPool; import lombok.extern.slf4j.Slf4j; +import org.slf4j.MDC; +import java.util.function.Consumer; import org.apache.commons.lang3.StringUtils; import java.util.ArrayList; @@ -38,31 +43,21 @@ import java.util.concurrent.atomic.LongAdder; /** - * Turns file rows into buffered {@code INSERT} statements and executes them in JDBC batches. - * Any row conversion or batch execution error fails the import without retrying rows. - * - *

Parallel execution starts at the fast-mode contract baseline of {@code 4} workers and - * {@code 20000} rows per batch, shrinks to at most {@code 1} worker and {@code 100} rows when the - * target is slow, and grows in steps while the measured throughput keeps improving - bounded by - * the machine's available parallelism, so it can never out-run the threads this computer actually - * has left. The - * {@code chat2db.task.import.parallelism} system property overrides the fan-out - * ({@code 1} forces the serial path, a larger value pins the worker count). Finished batches are - * handed to partitioned queues: per worker the order is strict, while workers run in parallel, and - * the queue set grows together with the adaptive gate. The number of active - * workers and the batch size are self-tuning (see {@link AdaptiveConcurrencyGate} and - * {@link AdaptiveBatchSizer}), so the pipeline converges to the throughput the target database - * actually sustains. Rows have no ordering constraints, so inter-worker interleaving is safe; the - * only visible effect is that auto-generated key values may interleave across workers. + * Buffers parsed CSV rows for parallel JDBC writes. Queues and batches are bounded; each worker + * owns its connection and inherits the task's statement guard. Failure cancels outstanding work + * and cleanup waits for all workers before the task can finish. */ @Slf4j public final class ImportRowBatcher implements AutoCloseable { - /** Fast-mode contract baseline: batches start at 20000 rows and may grow beyond it. */ + /** Initial batch size; the adaptive sizer caps growth at 50,000 rows. */ private static final int FAST_MODE_BATCH_ROWS = 20_000; private static final int QUEUE_CAPACITY = 4; + // About 4 MiB of UTF-16 SQL text per batch. A single larger row is sent alone. + private static final long MAX_BATCH_CHARS = 2L * 1024 * 1024; + /** Contract baseline fan-out of the fast mode; the adaptive gate grows it further on demand. */ private static final int BASE_WORKERS = 4; @@ -80,6 +75,16 @@ public final class ImportRowBatcher implements AutoCloseable { private final ConnectInfo connectInfo; + private final Context requestContext; + + private final Consumer statementGuard; + + private final Map loggingContext; + + private long bufferedChars; + + private long reportedRows; + private final AdaptiveBatchSizer batchSizer; private final LongAdder importedCount = new LongAdder(); @@ -126,6 +131,9 @@ public ImportRowBatcher(ImportTaskSpec spec, TaskExecutionContext context, Resol this.valueProcessor = valueProcessor; this.sqlBuilder = Chat2DBContext.getSqlBuilder(); this.connectInfo = Chat2DBContext.getConnectInfo(); + this.requestContext = ContextUtils.queryContext(); + this.statementGuard = Chat2DBContext.captureStatementGuard(); + this.loggingContext = MDC.getCopyOfContextMap(); this.batchSizer = new AdaptiveBatchSizer(FAST_MODE_BATCH_ROWS); int requestedWorkers = effectiveWorkerCount(connectInfo); List> builtQueues = null; @@ -138,7 +146,7 @@ public ImportRowBatcher(ImportTaskSpec spec, TaskExecutionContext context, Resol builtQueues.add(new ArrayBlockingQueue<>(QUEUE_CAPACITY)); } // The fan-out may grow, but never past the machine's available parallelism: the - // AIMD tuning moves inside [BASE_WORKERS, machineThreadCeiling()], and an explicit + // The adaptive gate stays inside [1, machineThreadCeiling()], and an explicit // chat2db.task.import.parallelism pin is bounded by the same ceiling. int gateCeiling = parallelismPinned() ? Math.min(requestedWorkers, machineThreadCeiling()) @@ -151,8 +159,7 @@ public ImportRowBatcher(ImportTaskSpec spec, TaskExecutionContext context, Resol return thread; }); } catch (Throwable parallelStartupFailure) { - // Adaptive parallel plumbing must never block the import: fall back to the exact - // serial path, which stays fully supported. + // Keep fast-mode batching on the calling thread if parallel infrastructure fails. log.warn("Parallel import infrastructure failed to start; degrading to serial execution", parallelStartupFailure); if (builtPool != null) { @@ -189,11 +196,15 @@ private void acceptRow(long fileRowNumber, List fileValues) { context.checkCancelled(); throwIfFailed(); String sql = buildInsert(fileRowNumber, fileValues); + if (!bufferedSqls.isEmpty() && bufferedChars + sql.length() > MAX_BATCH_CHARS) { + flushBufferedBatch(); + } if (bufferedSqls.isEmpty()) { firstBufferedRow = fileRowNumber; } bufferedSqls.add(sql); - if (bufferedSqls.size() >= batchSizer.batchSize()) { + bufferedChars += sql.length(); + if (bufferedSqls.size() >= batchSizer.batchSize() || bufferedChars >= MAX_BATCH_CHARS) { flushBufferedBatch(); } } @@ -257,6 +268,7 @@ private void flushBufferedBatch() { PendingBatch batch = new PendingBatch(List.copyOf(bufferedSqls), submittedBatches, firstBufferedRow); submittedBatches++; bufferedSqls.clear(); + bufferedChars = 0; executeBatch(batch); } @@ -277,8 +289,9 @@ private void executePendingBatch(PendingBatch batch) { int rows = batch.sqls().size(); try { DefaultSQLExecutor.getInstance().executeJdbcBatchInsert( - Chat2DBContext.getConnection(), batch.sqls(), context, context::checkCancelled); + Chat2DBContext.getConnection(), batch.sqls(), context, this::checkActive); importedCount.add(rows); + reportProgress(); context.logInfo("BATCH_EXECUTED", "SQL batch executed", Map.of("batch", batch.seq() + 1, "statementCount", rows)); } catch (RuntimeException | Error batchFailure) { @@ -303,21 +316,15 @@ private void executePendingBatch(PendingBatch batch) { } } - // --- parallel plumbing --------------------------------------------------------------- + private synchronized void reportProgress() { + long rows = importedCount.sum(); + if (rows > reportedRows) { + context.reportProgress((int) (20 + Math.min(70L, rows / 100)), TaskStage.IMPORTING.name(), + "Imported " + rows + " rows"); + reportedRows = rows; + } + } - /** - * Resolves the worker count from the {@code chat2db.task.import.parallelism} system property: - * {@code 0}, the default, picks the adaptive band ceiling {@code max(2, min(16, CPU cores))}; - * {@code 1} forces the serial path; explicit values are clamped into the [2, ceiling] band so - * a pinned value can neither exceed the machine nor drop below the minimum fan-out. Parallel - * workers each need their own connection, so without a JDBC url (test fixtures and - * non-relational sources bind a prebuilt connection instead) the batcher stays serial. - */ - /** - * Upper bound of the import fan-out: how many processors this JVM may use. The adaptive gate - * grows at most to that many concurrent batches, so an import can never request more threads - * than the machine has left to run them. - */ private static int machineThreadCeiling() { return Math.max(1, Runtime.getRuntime().availableProcessors()); } @@ -338,14 +345,13 @@ private static int effectiveWorkerCount(ConnectInfo connectInfo) { if (configured > 1) { return Math.min(configured, machineThreadCeiling()); } - return BASE_WORKERS; + return Math.min(BASE_WORKERS, machineThreadCeiling()); } /** * Grows the live worker set to match the adaptive gate: once the AIMD tuning admits more - * concurrent batches than there are workers, another queue/worker pair is added. Growth is - * unbounded by contract; the throughput feedback inside the gate is the only ceiling, so the - * pool ends up exactly as wide as this machine and target database sustain. + * concurrent batches than there are workers, another queue/worker pair is added up to the + * configured ceiling. */ private void ensureWorkerCapacity() { AdaptiveConcurrencyGate liveGate = gate; @@ -397,7 +403,7 @@ private void batchCompleted() { private void awaitQuiesce() { synchronized (quiesceMonitor) { while (inFlightBatches.get() > 0) { - throwIfFailed(); + checkActive(); try { quiesceMonitor.wait(50L); } catch (InterruptedException e) { @@ -412,24 +418,27 @@ private void awaitQuiesce() { private void runWorker(int workerIndex) { Thread.currentThread().setName("chat2db-import-" + context.taskId() + "-" + workerIndex); - // Created on first use and owned by this worker until it exits; copy() carries no - // connection, so Chat2DBContext.getConnection() builds a dedicated one per worker. + // Created on first use and owned by this worker until it exits; workers never borrow + // or return pooled connections. ConnectInfo isolated = null; - try { + try (var ignored = Chat2DBContext.bindStatementGuard(statementGuard)) { isolated = connectInfo.copy(); - isolated.setLoginUser("task-" + context.taskId() + "#import-" + workerIndex); Chat2DBContext.putContext(isolated); + ContextUtils.setContext(requestContext); + if (loggingContext != null) { + MDC.setContextMap(loggingContext); + } while (true) { PendingBatch batch = queues.get(workerIndex).take(); if (batch == END_OF_QUEUE) { return; } - gate.awaitPermit(() -> { - throwIfFailed(); - context.checkCancelled(); - }); + gate.awaitPermit(this::checkActive); try { - throwIfFailed(); + checkActive(); + if (isolated.getConnection() == null) { + ConnectionPool.createNewConnection(isolated); + } executePendingBatch(batch); } finally { gate.release(); @@ -441,23 +450,39 @@ private void runWorker(int workerIndex) { } catch (Throwable t) { recordFailure(t); } finally { - if (isolated != null) { - // Hand the dedicated connection back to the pool (or close it) instead of - // leaking it until the JVM exits. - ConnectionPool.close(isolated); + try { + if (isolated != null) { + // Worker connections are dedicated: close them and never return them to a pool. + try { + isolated.close(); + } finally { + isolated.setConnection(null); + } + } + } finally { + Chat2DBContext.removeContext(); + ContextUtils.removeContext(); + MDC.clear(); } - Chat2DBContext.removeContext(); } } private void recordFailure(Throwable taskFailure) { - failure.compareAndSet(null, taskFailure); + boolean firstFailure = failure.compareAndSet(null, taskFailure); aborted.set(true); + if (firstFailure) { + context.cancelResources(); + } synchronized (quiesceMonitor) { quiesceMonitor.notifyAll(); } } + private void checkActive() { + throwIfFailed(); + context.checkCancelled(); + } + private void throwIfFailed() { if (aborted.get()) { Throwable cause = failure.get(); @@ -525,25 +550,29 @@ public void close() { } finally { if (workerPool != null) { closing.set(true); - for (BlockingQueue queue : queues) { - while (!queue.offer(END_OF_QUEUE)) { - if (aborted.get()) { - break; - } - } - } if (aborted.get()) { workerPool.shutdownNow(); } else { + for (BlockingQueue queue : queues) { + queue.add(END_OF_QUEUE); // Successful flush drained all submitted work. + } workerPool.shutdown(); } + boolean interrupted = Thread.interrupted(); try { - if (!workerPool.awaitTermination(30L, TimeUnit.SECONDS)) { - workerPool.shutdownNow(); + while (!workerPool.isTerminated()) { + try { + workerPool.awaitTermination(200L, TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + interrupted = true; + recordFailure(new TaskCancelledException()); + workerPool.shutdownNow(); + } + } + } finally { + if (interrupted) { + Thread.currentThread().interrupt(); } - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - workerPool.shutdownNow(); } } totalImportNanos = System.nanoTime() - createdNanos; diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/AdaptiveBatchSizerTest.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/AdaptiveBatchSizerTest.java index cb22de69e8..d695c428ea 100644 --- a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/AdaptiveBatchSizerTest.java +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/AdaptiveBatchSizerTest.java @@ -3,12 +3,11 @@ import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; /** * Tuning behaviour of the fast-mode batch-size observer: the size starts at the configured * baseline, grows while the measured throughput keeps improving, shrinks when it regresses, never - * falls below the contract floor of 100 rows, and is not capped on the way up. + * falls below the contract floor of 100 rows, and respects the row-count ceiling. */ class AdaptiveBatchSizerTest { @@ -18,8 +17,7 @@ class AdaptiveBatchSizerTest { void floorsInitialValueAtTheContractMinimum() { assertEquals(100, new AdaptiveBatchSizer(1).batchSize()); assertEquals(20_000, new AdaptiveBatchSizer(20_000).batchSize()); - assertEquals(500_000, new AdaptiveBatchSizer(500_000).batchSize(), - "the baseline may be large: there is no upper bound on the configured start"); + assertEquals(50_000, new AdaptiveBatchSizer(Integer.MAX_VALUE).batchSize()); } @Test @@ -40,7 +38,7 @@ void shrinksWhenThroughputRegressesAndStopsAtTheFloor() { AdaptiveBatchSizer sizer = new AdaptiveBatchSizer(8_000); long nanos = 5 * MILLI; sizer.record(8_000, nanos); // reference - for (int round = 0; round < 12; round++) { + for (int round = 0; round < 64; round++) { nanos = nanos * 4; // four times slower: throughput clearly regresses sizer.record(sizer.batchSize(), nanos); } @@ -48,16 +46,15 @@ void shrinksWhenThroughputRegressesAndStopsAtTheFloor() { } @Test - void hasNoUpperBoundOnGrowth() { + void repeatedGrowthStaysBoundedWithoutOverflow() { AdaptiveBatchSizer sizer = new AdaptiveBatchSizer(1_000); long nanos = 10 * MILLI; sizer.record(1_000, nanos); - for (int round = 0; round < 12; round++) { + for (int round = 0; round < 64; round++) { nanos = Math.max(1L, nanos / 2); sizer.record(1_000, nanos); } - assertTrue(sizer.batchSize() > 1_000_000, - "growth is unbounded by contract, the throughput feedback is the only ceiling"); + assertEquals(50_000, sizer.batchSize()); } @Test diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/ParallelImportLifecycleTest.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/ParallelImportLifecycleTest.java new file mode 100644 index 0000000000..6664ffeaa8 --- /dev/null +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/ParallelImportLifecycleTest.java @@ -0,0 +1,341 @@ +package ai.chat2db.community.domain.core.impl.task; + +import ai.chat2db.community.domain.api.config.DBConfig; +import ai.chat2db.community.domain.api.config.DriverConfig; +import ai.chat2db.community.domain.api.model.task.*; +import ai.chat2db.community.domain.api.model.task.extension.TaskOperation; +import ai.chat2db.community.domain.api.model.task.extension.TaskStatementContext; +import ai.chat2db.community.domain.api.service.task.TaskStorage; +import ai.chat2db.community.domain.api.service.task.extension.ITaskExecutionGuard; +import ai.chat2db.community.domain.core.impl.task.extension.TaskExtensionManager; +import ai.chat2db.community.domain.core.impl.task.imports.excel.CSVImporter; +import ai.chat2db.community.tools.model.Context; +import ai.chat2db.community.tools.util.ContextUtils; +import ai.chat2db.spi.DefaultDBManager; +import ai.chat2db.spi.IDbManager; +import ai.chat2db.spi.IPlugin; +import ai.chat2db.spi.model.datasource.ConnectInfo; +import ai.chat2db.spi.sql.Chat2DBContext; +import ai.chat2db.spi.sql.ConnectionPool; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.slf4j.MDC; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Proxy; +import java.nio.file.Files; +import java.nio.file.Path; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.List; +import java.util.Map; +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +import static org.junit.jupiter.api.Assertions.*; + +class ParallelImportLifecycleTest { + private static final String TYPE = "PARALLEL_IMPORT_LIFECYCLE_TEST"; + private static final long DATASOURCE_ID = 92908001L; + @TempDir Path directory; + private Connection observer; + private ConnectInfo connectInfo; + private Context requestContext; + private IPlugin previousPlugin; + private String previousParallelism; + private final List opened = new CopyOnWriteArrayList<>(); + private final List progress = new CopyOnWriteArrayList<>(); + private final List batchChars = new CopyOnWriteArrayList<>(); + private final List batchRows = new CopyOnWriteArrayList<>(); + private final AtomicInteger closeCalls = new AtomicInteger(); + private final AtomicInteger executeCalls = new AtomicInteger(); + private final CountDownLatch blocked = new CountDownLatch(1); + private final CountDownLatch cancelled = new CountDownLatch(1); + private final CountDownLatch driverExited = new CountDownLatch(1); + private final CountDownLatch releaseDriver = new CountDownLatch(1); + private volatile boolean failWithBlockedPeer; + private volatile boolean blockUntilCancellation; + + @BeforeEach + void setUp() throws Exception { + previousParallelism = System.setProperty("chat2db.task.import.parallelism", "2"); + String url = "jdbc:h2:mem:parallel_lifecycle_" + System.nanoTime() + ";DB_CLOSE_DELAY=-1"; + observer = DriverManager.getConnection(url); + try (Statement statement = observer.createStatement()) { + statement.execute("CREATE TABLE ROWS_TARGET (ID INT PRIMARY KEY, NAME VARCHAR(10000))"); + } + requestContext = new Context(); + ContextUtils.setContext(requestContext); + MDC.put("taskId", "review-task"); + connectInfo = new ConnectInfo(); + connectInfo.setDbType(TYPE); + connectInfo.setDataSourceId(DATASOURCE_ID); + connectInfo.setUrl(url); + connectInfo.setDriverConfig(new DriverConfig()); + connectInfo.setConnection(observer); + DBConfig config = new DBConfig(); + config.setDbType(TYPE); + config.setDefaultDriverConfig(new DriverConfig()); + previousPlugin = Chat2DBContext.PLUGIN_MAP.put(TYPE, new IPlugin() { + public DBConfig getDBConfig() { return config; } + public IDbManager getDbManager() { + return new DefaultDBManager() { + @Override public Connection getConnection(ConnectInfo info) { + assertSame(requestContext, ContextUtils.queryContext()); + assertEquals("review-task", MDC.get("taskId")); + try { + Connection real = DriverManager.getConnection(url); + opened.add(real); + return wrapConnection(real); + } catch (SQLException e) { + throw new RuntimeException(e); + } + } + }; + } + }); + Chat2DBContext.putContext(connectInfo); + } + + @AfterEach + void tearDown() throws Exception { + releaseDriver.countDown(); + Chat2DBContext.removeContext(); + ContextUtils.removeContext(); + MDC.clear(); + for (Connection connection : opened) connection.close(); + observer.close(); + try (Connection connection = DriverManager.getConnection(connectInfo.getUrl()); + Statement statement = connection.createStatement()) { + statement.execute("DROP ALL OBJECTS"); + } + ConnectionPool.removeConnection(DATASOURCE_ID); + if (previousPlugin == null) Chat2DBContext.PLUGIN_MAP.remove(TYPE); + else Chat2DBContext.PLUGIN_MAP.put(TYPE, previousPlugin); + if (previousParallelism == null) System.clearProperty("chat2db.task.import.parallelism"); + else System.setProperty("chat2db.task.import.parallelism", previousParallelism); + } + + @Test + void successClosesDedicatedConnectionsAndReportsMonotonicProgress() throws Exception { + run(spec(40_001, "ok"), task(1L)); + + assertEquals(40_001, countRows()); + assertTrue(opened.size() >= 2); + assertConnectionsClosed(); + List updates = progress.stream().filter(p -> "IMPORTING".equals(p.getStage())).toList(); + assertFalse(updates.isEmpty()); + long lastRows = 0; + int lastProgress = 20; + for (TaskProgress update : updates) { + long rows = Long.parseLong(update.getMessage().split(" ")[1]); + assertTrue(rows > lastRows); + assertTrue(update.getProgress() >= lastProgress); + lastRows = rows; + lastProgress = update.getProgress(); + } + assertEquals(40_001, lastRows); + assertEquals(90, lastProgress); + var pool = ConnectionPool.class.getDeclaredField("CONNECTION_MAP"); + pool.setAccessible(true); + assertFalse(((Map) pool.get(null)).containsKey(DATASOURCE_ID), "dedicated workers must not create pool queues"); + } + + @Test + void taskStatementGuardRunsOnWorkersBeforeAnyWrite() throws Exception { + AtomicInteger taskCalls = new AtomicInteger(); + AtomicInteger guardCalls = new AtomicInteger(); + var capturedTask = new ai.chat2db.community.domain.api.model.task.extension.TaskExecutionContext( + 2L, TaskType.DATA_FILE_IMPORT, null, null, null, List.of("ROWS_TARGET"), TaskOperation.IMPORT); + TaskExtensionManager manager = new TaskExtensionManager(List.of(), List.of(new ITaskExecutionGuard() { + public void beforeTask(ai.chat2db.community.domain.api.model.task.extension.TaskExecutionContext context) { + assertSame(capturedTask, context); + taskCalls.incrementAndGet(); + } + public void beforeStatement(TaskStatementContext context) { + guardCalls.incrementAndGet(); + assertSame(capturedTask, context.getTaskContext()); + assertSame(requestContext, ContextUtils.queryContext()); + assertEquals("review-task", MDC.get("taskId")); + throw new IllegalStateException("statement denied"); + } + })); + ImportTaskSpec spec = spec(10, "ok"); + + manager.runGuarded(capturedTask, () -> { + try (var ignored = Chat2DBContext.bindStatementGuard(manager.captureStatementGuard())) { + assertThrows(RuntimeException.class, () -> run(spec, task(2L))); + } + }); + + assertEquals(1, taskCalls.get()); + assertTrue(guardCalls.get() > 0); + assertEquals(0, executeCalls.get()); + assertEquals(0, countRows()); + assertConnectionsClosed(); + } + + @Test + void batchFailureCancelsPeerAndWaitsForItsJdbcCallToExit() throws Exception { + failWithBlockedPeer = true; + ImportTaskSpec spec = spec(40_000, "ok"); + RunningTask task = task(3L); + var executor = Executors.newSingleThreadExecutor(); + try { + Future future = executor.submit(() -> runOnCallerThread(spec, task)); + assertTrue(cancelled.await(10, TimeUnit.SECONDS), "failure must cancel the blocked JDBC statement"); + assertFalse(future.isDone(), "task must remain active until the driver exits"); + assertFalse(driverExited.await(300, TimeUnit.MILLISECONDS)); + assertFalse(task.cancellationToken().isCancelled(), "an import error must remain a failure"); + + releaseDriver.countDown(); + assertThrows(ExecutionException.class, () -> future.get(10, TimeUnit.SECONDS)); + assertTrue(driverExited.await(1, TimeUnit.SECONDS)); + assertConnectionsClosed(); + assertEquals(0, countRows()); + } finally { + releaseDriver.countDown(); + executor.shutdownNow(); + assertTrue(executor.awaitTermination(10, TimeUnit.SECONDS)); + } + } + + @Test + void userCancellationWaitsForTheWorkerEvenWhenCallerIsInterrupted() throws Exception { + blockUntilCancellation = true; + ImportTaskSpec spec = spec(10, "ok"); + RunningTask task = task(4L); + var executor = Executors.newSingleThreadExecutor(); + CountDownLatch callerExited = new CountDownLatch(1); + FutureTask future = new FutureTask<>(() -> { + try { runOnCallerThread(spec, task); } + finally { callerExited.countDown(); } + return null; + }); + task.setFuture(future); + try { + executor.execute(future); + assertTrue(blocked.await(10, TimeUnit.SECONDS)); + assertTrue(task.requestCancellation(true)); + assertTrue(cancelled.await(5, TimeUnit.SECONDS)); + assertFalse(callerExited.await(300, TimeUnit.MILLISECONDS)); + + releaseDriver.countDown(); + assertTrue(callerExited.await(10, TimeUnit.SECONDS)); + assertConnectionsClosed(); + assertEquals(0, countRows()); + } finally { + releaseDriver.countDown(); + executor.shutdownNow(); + assertTrue(executor.awaitTermination(10, TimeUnit.SECONDS)); + } + } + + @Test + void wideRowsFlushBeforeTheRowCountThreshold() throws Exception { + run(spec(600, "x".repeat(4096)), task(5L)); + + assertEquals(600, countRows()); + assertTrue(batchRows.size() >= 2, "SQL text size must bound a batch before 20,000 rows"); + assertEquals(600, batchRows.stream().mapToInt(Integer::intValue).sum()); + assertTrue(batchChars.stream().allMatch(chars -> chars <= 2L * 1024 * 1024)); + assertConnectionsClosed(); + } + + private RunningTask task(long id) { return new RunningTask(id); } + + private void run(ImportTaskSpec spec, RunningTask task) { + TaskStorage storage = (TaskStorage) Proxy.newProxyInstance(getClass().getClassLoader(), new Class[]{TaskStorage.class}, + (proxy, method, args) -> { + if (method.getName().equals("updateProgressIfRunning")) { progress.add((TaskProgress) args[1]); return true; } + if (method.getName().equals("appendEvent")) return args[0]; + throw new UnsupportedOperationException(method.getName()); + }); + new CSVImporter().run(spec, new TaskExecutionContextImpl(task.taskId(), task, storage, new ArtifactServiceImpl())); + } + + private void runOnCallerThread(ImportTaskSpec spec, RunningTask task) { + Chat2DBContext.putContext(connectInfo); + ContextUtils.setContext(requestContext); + MDC.put("taskId", "review-task"); + try { run(spec, task); } + finally { ContextUtils.removeContext(); MDC.clear(); Chat2DBContext.removeContext(); } + } + + private ImportTaskSpec spec(int count, String name) throws Exception { + Path csv = directory.resolve("rows.csv"); + try (var writer = Files.newBufferedWriter(csv)) { + writer.write("ID,NAME\n"); + for (int id = 1; id <= count; id++) writer.write(id + "," + name + "\n"); + } + return ImportTaskSpec.builder().sourceFile(csv.toString()).format("CSV").mode("ULTRA_FAST") + .target(TaskTargetSnapshot.builder().tableName("ROWS_TARGET").build()) + .columnMappings(List.of(new ImportColumnMapping("ID", "ID"), new ImportColumnMapping("NAME", "NAME"))).build(); + } + + private Connection wrapConnection(Connection connection) { + return (Connection) Proxy.newProxyInstance(getClass().getClassLoader(), new Class[]{Connection.class}, (proxy, method, args) -> { + if (method.getName().equals("close")) closeCalls.incrementAndGet(); + try { + Object value = method.invoke(connection, args); + return method.getName().equals("createStatement") ? wrapStatement((Statement) value) : value; + } catch (InvocationTargetException e) { throw e.getCause(); } + }); + } + + private Statement wrapStatement(Statement statement) { + AtomicInteger rows = new AtomicInteger(); + AtomicLong chars = new AtomicLong(); + return (Statement) Proxy.newProxyInstance(getClass().getClassLoader(), new Class[]{Statement.class}, (proxy, method, args) -> { + switch (method.getName()) { + case "addBatch" -> { rows.incrementAndGet(); chars.addAndGet(((String) args[0]).length()); } + case "cancel" -> { cancelled.countDown(); return null; } + case "executeBatch" -> { + int execution = executeCalls.incrementAndGet(); + batchChars.add(chars.get()); + batchRows.add(rows.get()); + if (blockUntilCancellation || failWithBlockedPeer && execution == 1) { + blocked.countDown(); + boolean interrupted = false; + try { + while (true) { + try { releaseDriver.await(); break; } + catch (InterruptedException e) { interrupted = true; } + } + throw new SQLException("driver acknowledged cancellation"); + } finally { + driverExited.countDown(); + if (interrupted) Thread.currentThread().interrupt(); + } + } + if (failWithBlockedPeer) { + if (!blocked.await(5, TimeUnit.SECONDS)) throw new AssertionError("peer did not start"); + throw new SQLException("injected batch failure"); + } + } + } + try { return method.invoke(statement, args); } + catch (InvocationTargetException e) { throw e.getCause(); } + }); + } + + private int countRows() throws Exception { + try (Connection connection = DriverManager.getConnection(connectInfo.getUrl()); + Statement statement = connection.createStatement(); ResultSet rows = statement.executeQuery("SELECT COUNT(*) FROM ROWS_TARGET")) { + assertTrue(rows.next()); + return rows.getInt(1); + } + } + + private void assertConnectionsClosed() throws Exception { + assertFalse(opened.isEmpty()); + for (Connection connection : opened) assertTrue(connection.isClosed()); + assertEquals(opened.size(), closeCalls.get(), "each connection must be closed exactly once"); + } +} diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/RunningTaskTest.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/RunningTaskTest.java index 00f6109fd7..e3b60419c7 100644 --- a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/RunningTaskTest.java +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/RunningTaskTest.java @@ -143,6 +143,27 @@ void parallelStatementsAreCancelledOnceAndClosedStatementsAreUnregistered() { assertEquals(1, secondCount.get()); } + @Test + void failureCancellationKeepsFailureStatusAndCancelsLateStatementsOnce() { + List cancellations = new java.util.ArrayList<>(); + RunningTask runningTask = new RunningTask(42L, cancellations::add); + TaskExecutionContextImpl context = new TaskExecutionContextImpl(42L, runningTask, null, null); + AtomicInteger firstCount = new AtomicInteger(); + AtomicInteger lateCount = new AtomicInteger(); + context.onStatementCreated(statement(firstCount)); + + context.cancelResources(); + assertFalse(runningTask.cancellationToken().isCancelled()); + context.onStatementCreated(statement(lateCount)); + context.cancelResources(); + runningTask.requestCancellation(true); + + assertEquals(2, cancellations.size()); + cancellations.forEach(Runnable::run); + assertEquals(1, firstCount.get()); + assertEquals(1, lateCount.get()); + } + private static Statement statement(AtomicInteger count) { return (Statement) Proxy.newProxyInstance(Statement.class.getClassLoader(), new Class[] {Statement.class}, (proxy, method, args) -> { diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/imports/excel/CSVImporterColumnMappingTest.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/imports/excel/CSVImporterColumnMappingTest.java index eddcb5ecb9..4802fdb299 100644 --- a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/imports/excel/CSVImporterColumnMappingTest.java +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/imports/excel/CSVImporterColumnMappingTest.java @@ -274,6 +274,29 @@ void preservesQuotedEmptyTextAndUnquotedNull(@TempDir Path directory) throws Exc } } + @Test + void fastAndStandardModesPreserveWhitespaceInSourceNames(@TempDir Path directory) throws Exception { + Path input = Files.writeString(directory.resolve("spaces.csv"), "Name, Name\nplain,spaced\n"); + for (String mode : List.of("STANDARD", "ULTRA_FAST")) { + try (Statement statement = connection.createStatement()) { + statement.execute("DELETE FROM orders"); + } + ImportTaskSpec spec = ImportTaskSpec.builder().sourceFile(input.toString()).mode(mode) + .target(TaskTargetSnapshot.builder().tableName("orders").build()) + .columnMappings(List.of(new ImportColumnMapping(" Name", "name"), + new ImportColumnMapping("Name", "note"))).build(); + + new CSVImporter().doImportData(spec, new RecordingTaskExecutionContext(), columns()); + + try (Statement statement = connection.createStatement(); + ResultSet rows = statement.executeQuery("SELECT name, note FROM orders")) { + org.junit.jupiter.api.Assertions.assertTrue(rows.next()); + assertEquals("spaced", rows.getString(1), mode); + assertEquals("plain", rows.getString(2), mode); + } + } + } + private void assertRowCount(int expected) throws Exception { try (Statement statement = connection.createStatement(); ResultSet result = statement.executeQuery("SELECT COUNT(*) FROM orders")) { diff --git a/chat2db-community-server/chat2db-community-spi/src/main/java/ai/chat2db/spi/sql/Chat2DBContext.java b/chat2db-community-server/chat2db-community-spi/src/main/java/ai/chat2db/spi/sql/Chat2DBContext.java index 75db407f87..8a3bec85a4 100644 --- a/chat2db-community-server/chat2db-community-spi/src/main/java/ai/chat2db/spi/sql/Chat2DBContext.java +++ b/chat2db-community-server/chat2db-community-spi/src/main/java/ai/chat2db/spi/sql/Chat2DBContext.java @@ -139,6 +139,11 @@ public static Connection getConnection() { return StatementGuardConnection.wrap(connection, STATEMENT_GUARD_THREAD_LOCAL.get()); } + /** Returns the guard bound to the current task for propagation to its workers. */ + public static Consumer captureStatementGuard() { + return STATEMENT_GUARD_THREAD_LOCAL.get(); + } + public static StatementGuardScope bindStatementGuard(Consumer statementGuard) { Consumer previous = STATEMENT_GUARD_THREAD_LOCAL.get(); if (statementGuard == null) { From 4543d8ccbaab5cff6c108f4baac7b257e87512e2 Mon Sep 17 00:00:00 2001 From: zgq Date: Tue, 15 Sep 2026 11:43:06 +0800 Subject: [PATCH 29/38] refactor(import): define STANDARD and FAST enum names --- .../components/ImportModeControl/index.tsx | 4 +-- .../src/service/importExport.ts | 2 +- .../src/typings/importExport.ts | 2 +- .../api/model/task/TaskExecutionMode.java | 26 ++++++------------- .../impl/task/imports/excel/CSVImporter.java | 2 +- .../db/DbMappedImportServiceImplTest.java | 4 +-- .../core/impl/task/CsvImportPipelineTest.java | 2 +- .../task/ImportRowBatcherParallelTest.java | 2 +- .../task/ParallelImportLifecycleTest.java | 2 +- .../excel/CSVImporterColumnMappingTest.java | 4 +-- .../api/converter/task/TaskWebConverter.java | 2 +- .../model/request/task/TaskImportRequest.java | 2 +- .../converter/task/TaskWebConverterTest.java | 22 ++++++++++++++-- 13 files changed, 42 insertions(+), 34 deletions(-) diff --git a/chat2db-community-client/src/blocks/ImportAndExport/components/ImportModeControl/index.tsx b/chat2db-community-client/src/blocks/ImportAndExport/components/ImportModeControl/index.tsx index 9c7a9528ff..850264d71f 100644 --- a/chat2db-community-client/src/blocks/ImportAndExport/components/ImportModeControl/index.tsx +++ b/chat2db-community-client/src/blocks/ImportAndExport/components/ImportModeControl/index.tsx @@ -18,7 +18,7 @@ export default function ImportModeControl({ value, onChange, disabled }: Props) content: i18n('workspace.importExport.ultraModeAcknowledge'), okText: i18n('workspace.importExport.ultraModeConfirm'), cancelText: i18n('common.button.cancel'), - onOk: () => onChange('ULTRA_FAST'), + onOk: () => onChange('FAST'), }); return; } @@ -33,7 +33,7 @@ export default function ImportModeControl({ value, onChange, disabled }: Props) diff --git a/chat2db-community-client/src/service/importExport.ts b/chat2db-community-client/src/service/importExport.ts index f013e86fb5..e9cf48dcdc 100644 --- a/chat2db-community-client/src/service/importExport.ts +++ b/chat2db-community-client/src/service/importExport.ts @@ -65,7 +65,7 @@ export interface ImportTaskParams extends IDatabaseBaseInfo { format: ImportExportFileType; dataTimeFormat?: string; csvOptions?: ICsvOptions; - mode?: 'ULTRA_FAST' | 'STANDARD'; + mode?: 'FAST' | 'STANDARD'; } const submitExport = createRequest('/api/tasks/export', { method: 'post' }); diff --git a/chat2db-community-client/src/typings/importExport.ts b/chat2db-community-client/src/typings/importExport.ts index 5f6896f25f..ab8ac6ea88 100644 --- a/chat2db-community-client/src/typings/importExport.ts +++ b/chat2db-community-client/src/typings/importExport.ts @@ -30,7 +30,7 @@ export interface ImportExportTaskDetails { } /** CSV import execution mode; absent resolves to STANDARD on the backend. */ -export type ImportExecutionMode = 'ULTRA_FAST' | 'STANDARD'; +export type ImportExecutionMode = 'FAST' | 'STANDARD'; export interface ImportExportTaskEvent { eventId: number; diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/task/TaskExecutionMode.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/task/TaskExecutionMode.java index b090c8b61a..155f3d06af 100644 --- a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/task/TaskExecutionMode.java +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/task/TaskExecutionMode.java @@ -1,22 +1,12 @@ package ai.chat2db.community.domain.api.model.task; -import org.apache.commons.lang3.StringUtils; - -/** - * CSV import execution mode. ULTRA_FAST uses parallel row batches with adaptive tuning; - * absent or unknown values resolve to STANDARD. - */ -public final class TaskExecutionMode { - - public static final String ULTRA_FAST = "ULTRA_FAST"; - - public static final String STANDARD = "STANDARD"; - - private TaskExecutionMode() { - } - - /** True only for an explicit {@code ULTRA_FAST}; anything else (null, blank, unknown) is standard. */ - public static boolean isUltraFast(String mode) { - return ULTRA_FAST.equalsIgnoreCase(StringUtils.trimToEmpty(mode)); +/** CSV import modes. Request fields remain strings containing these uppercase names. */ +public enum TaskExecutionMode { + STANDARD, + FAST; + + /** Only an explicit FAST selects parallel execution; other values retain ordinary behavior. */ + public static boolean isFast(String mode) { + return FAST.name().equals(mode); } } diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/excel/CSVImporter.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/excel/CSVImporter.java index ff4bfb6895..656f35a136 100644 --- a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/excel/CSVImporter.java +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/excel/CSVImporter.java @@ -18,7 +18,7 @@ public class CSVImporter extends BaseExcelImporter implements IImportStrategy { @Override protected void doImportData(ImportTaskSpec spec, TaskExecutionContext context, List columns) { - if (TaskExecutionMode.isUltraFast(spec.getMode())) { + if (TaskExecutionMode.isFast(spec.getMode())) { new ParallelCSVImporter().doImportData(spec, context, columns); return; } diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/db/DbMappedImportServiceImplTest.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/db/DbMappedImportServiceImplTest.java index 06401bc1b8..9b09382bba 100644 --- a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/db/DbMappedImportServiceImplTest.java +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/db/DbMappedImportServiceImplTest.java @@ -81,10 +81,10 @@ void forwardsFastModeAlongsideExistingMappingAndCsvOptions(@TempDir Path directo var options = ai.chat2db.community.domain.api.model.task.CsvOptions.defaults(); options.setDelimiter(";"); execution.setCsvOptions(options); - execution.setMode("ULTRA_FAST"); + execution.setMode("FAST"); assertEquals(42L, service.submit(execution)); - assertEquals("ULTRA_FAST", submitted.get().getMode()); + assertEquals("FAST", submitted.get().getMode()); assertEquals(";", submitted.get().getCsvOptions().getDelimiter()); assertEquals("name", submitted.get().getColumnMappings().get(0).getTargetColumn()); } diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/CsvImportPipelineTest.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/CsvImportPipelineTest.java index 3a6876c988..767765002e 100644 --- a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/CsvImportPipelineTest.java +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/CsvImportPipelineTest.java @@ -213,7 +213,7 @@ private ImportTaskSpec csvSpec(Path csv) { .taskType("DATA_FILE_IMPORT") .sourceFile(csv.toString()) .format("CSV") - .mode("ULTRA_FAST") + .mode("FAST") .target(TaskTargetSnapshot.builder().dataSourceId(1L).tableName("TARGET_ROWS").build()) .columnMappings(List.of( new ImportColumnMapping("ROW_ID", "ID"), diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/ImportRowBatcherParallelTest.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/ImportRowBatcherParallelTest.java index af23cea924..36fb1a9416 100644 --- a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/ImportRowBatcherParallelTest.java +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/ImportRowBatcherParallelTest.java @@ -160,7 +160,7 @@ private ImportTaskSpec csvSpec(Path csv) { .importFileId("parallel-import-test-source") .format("CSV") .target(TaskTargetSnapshot.builder().dataSourceId(1L).tableName("BULK_ROWS").build()) - .mode("ULTRA_FAST") + .mode("FAST") .columnMappings(List.of( new ImportColumnMapping("ID", "ID"), new ImportColumnMapping("NAME", "NAME"))) diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/ParallelImportLifecycleTest.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/ParallelImportLifecycleTest.java index 6664ffeaa8..67e7c0feaf 100644 --- a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/ParallelImportLifecycleTest.java +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/ParallelImportLifecycleTest.java @@ -274,7 +274,7 @@ private ImportTaskSpec spec(int count, String name) throws Exception { writer.write("ID,NAME\n"); for (int id = 1; id <= count; id++) writer.write(id + "," + name + "\n"); } - return ImportTaskSpec.builder().sourceFile(csv.toString()).format("CSV").mode("ULTRA_FAST") + return ImportTaskSpec.builder().sourceFile(csv.toString()).format("CSV").mode("FAST") .target(TaskTargetSnapshot.builder().tableName("ROWS_TARGET").build()) .columnMappings(List.of(new ImportColumnMapping("ID", "ID"), new ImportColumnMapping("NAME", "NAME"))).build(); } diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/imports/excel/CSVImporterColumnMappingTest.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/imports/excel/CSVImporterColumnMappingTest.java index 4802fdb299..bab23d43e6 100644 --- a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/imports/excel/CSVImporterColumnMappingTest.java +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/imports/excel/CSVImporterColumnMappingTest.java @@ -237,7 +237,7 @@ void csvRowRangeAndFormatsDriveThePersistedValues(@TempDir Path directory) throw @Test void fastModeRejectsDuplicateMappingBeforeWritingRows(@TempDir Path directory) throws Exception { Path input = Files.writeString(directory.resolve("duplicates.csv"), "Full Name,status\nAlice,OVERRIDE\n"); - ImportTaskSpec spec = ImportTaskSpec.builder().mode("ULTRA_FAST").sourceFile(input.toString()) + ImportTaskSpec spec = ImportTaskSpec.builder().mode("FAST").sourceFile(input.toString()) .target(TaskTargetSnapshot.builder().tableName("orders").build()) .columnMappings(List.of(new ImportColumnMapping("Full Name", "name"), new ImportColumnMapping("status", "name"))).build(); @@ -277,7 +277,7 @@ void preservesQuotedEmptyTextAndUnquotedNull(@TempDir Path directory) throws Exc @Test void fastAndStandardModesPreserveWhitespaceInSourceNames(@TempDir Path directory) throws Exception { Path input = Files.writeString(directory.resolve("spaces.csv"), "Name, Name\nplain,spaced\n"); - for (String mode : List.of("STANDARD", "ULTRA_FAST")) { + for (String mode : List.of("STANDARD", "FAST")) { try (Statement statement = connection.createStatement()) { statement.execute("DELETE FROM orders"); } diff --git a/chat2db-community-server/chat2db-community-web/src/main/java/ai/chat2db/community/web/api/converter/task/TaskWebConverter.java b/chat2db-community-server/chat2db-community-web/src/main/java/ai/chat2db/community/web/api/converter/task/TaskWebConverter.java index 8caf0f248b..c172b9ace2 100644 --- a/chat2db-community-server/chat2db-community-web/src/main/java/ai/chat2db/community/web/api/converter/task/TaskWebConverter.java +++ b/chat2db-community-server/chat2db-community-web/src/main/java/ai/chat2db/community/web/api/converter/task/TaskWebConverter.java @@ -64,7 +64,7 @@ public ImportTaskSpec importRequest2spec(TaskImportRequest request) { .format(format) .dataTimeFormat(request.getDataTimeFormat()) .csvOptions(csvOptions(format, request.getCsvOptions())) - .mode(normalize(request.getMode())) + .mode(request.getMode()) .build(); } diff --git a/chat2db-community-server/chat2db-community-web/src/main/java/ai/chat2db/community/web/api/model/request/task/TaskImportRequest.java b/chat2db-community-server/chat2db-community-web/src/main/java/ai/chat2db/community/web/api/model/request/task/TaskImportRequest.java index 58621cd87c..cd0c4df209 100644 --- a/chat2db-community-server/chat2db-community-web/src/main/java/ai/chat2db/community/web/api/model/request/task/TaskImportRequest.java +++ b/chat2db-community-server/chat2db-community-web/src/main/java/ai/chat2db/community/web/api/model/request/task/TaskImportRequest.java @@ -25,7 +25,7 @@ public class TaskImportRequest extends DataSourceBaseRequest { private CsvOptions csvOptions; - /** Execution mode: ULTRA_FAST or STANDARD (default when absent). */ + /** Execution mode: FAST or STANDARD (default when absent). */ private String mode; } diff --git a/chat2db-community-server/chat2db-community-web/src/test/java/ai/chat2db/community/web/api/converter/task/TaskWebConverterTest.java b/chat2db-community-server/chat2db-community-web/src/test/java/ai/chat2db/community/web/api/converter/task/TaskWebConverterTest.java index 32cc8ad724..6821ec835d 100644 --- a/chat2db-community-server/chat2db-community-web/src/test/java/ai/chat2db/community/web/api/converter/task/TaskWebConverterTest.java +++ b/chat2db-community-server/chat2db-community-web/src/test/java/ai/chat2db/community/web/api/converter/task/TaskWebConverterTest.java @@ -24,10 +24,28 @@ void importPreservesStagedSourceAndExecutionMode() { var request = new ai.chat2db.community.web.api.model.request.task.TaskImportRequest(); request.setFileId("staged-source"); request.setFormat("CSV"); - request.setMode("ULTRA_FAST"); + request.setMode("FAST"); var result = new TaskWebConverter().importRequest2spec(request); assertEquals("staged-source", result.getImportFileId()); - assertEquals("ULTRA_FAST", result.getMode()); + assertEquals("FAST", result.getMode()); + } + + @Test + void modeStaysAStringAndOnlyUppercaseFastEnablesParallelExecution() throws Exception { + assertEquals(String.class, TaskImportRequest.class.getDeclaredField("mode").getType()); + assertEquals(String.class, ai.chat2db.community.web.api.model.request.db.ImportExecuteRequest.class + .getDeclaredField("mode").getType()); + assertEquals(List.of("STANDARD", "FAST"), java.util.Arrays.stream( + ai.chat2db.community.domain.api.model.task.TaskExecutionMode.values()).map(Enum::name).toList()); + for (String mode : java.util.Arrays.asList(null, "STANDARD", "FAST", "fast", " FAST ", "unknown")) { + TaskImportRequest request = new TaskImportRequest(); + request.setFormat("CSV"); + request.setMode(mode); + ImportTaskSpec spec = new TaskWebConverter().importRequest2spec(request); + assertEquals(mode, spec.getMode()); + assertEquals("FAST".equals(mode), + ai.chat2db.community.domain.api.model.task.TaskExecutionMode.isFast(spec.getMode())); + } } private final TaskWebConverter converter = new TaskWebConverter(); From 4152c3085550075bca517713a33286c2efc31d43 Mon Sep 17 00:00:00 2001 From: zgq Date: Tue, 15 Sep 2026 14:40:12 +0800 Subject: [PATCH 30/38] fix(import): synchronize tuning samples and ignore failed batches --- .../impl/task/AdaptiveConcurrencyGate.java | 33 ++++++++-------- .../impl/task/imports/ImportRowBatcher.java | 10 ++--- .../task/AdaptiveConcurrencyGateTest.java | 38 +++++++++++++++++++ 3 files changed, 59 insertions(+), 22 deletions(-) diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/AdaptiveConcurrencyGate.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/AdaptiveConcurrencyGate.java index b07a2012e4..23b0b3be45 100644 --- a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/AdaptiveConcurrencyGate.java +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/AdaptiveConcurrencyGate.java @@ -3,7 +3,6 @@ import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicLong; import lombok.extern.slf4j.Slf4j; @@ -12,7 +11,7 @@ * work; the permit count starts low and is tuned by a throughput observer in an AIMD style: every * time {@link #WINDOW_ROWS} rows of data have flowed through since the last evaluation, the gate * compares the throughput of the finished window with the previous one and grows by one permit on - * improvement, or gives back a quarter of the permits on regression. The fan-out therefore + * improvement above 10%, or gives back a quarter of the permits on regression above 10%. The fan-out therefore * converges to the level the target system actually tolerates instead of a fixed guess, and it * backs off on its own when the source or the target becomes the bottleneck. * @@ -46,9 +45,13 @@ public final class AdaptiveConcurrencyGate extends Semaphore { /** Total permits in circulation; only the tuning paths change it, and never past maxPermits. */ private final AtomicInteger totalPermits; - private final AtomicLong windowRows = new AtomicLong(); + private static final double GROW_MARGIN = 1.10D; - private final AtomicLong windowNanos = new AtomicLong(); + private static final double SHRINK_MARGIN = 0.90D; + + private long windowRows; + + private long windowNanos; private double lastThroughput = -1.0D; @@ -67,22 +70,18 @@ public static AdaptiveConcurrencyGate create(int initialPermits, int maxPermits) * Records one completed work unit ({@code rows} rows over {@code nanos} wall time); once the * observation window fills, the fan-out is retuned. Never throws into the caller. */ - public void record(long rows, long nanos) { + public synchronized void record(long rows, long nanos) { if (rows <= 0 || nanos <= 0) { return; } - windowRows.addAndGet(rows); - windowNanos.addAndGet(nanos); - if (windowRows.get() < WINDOW_ROWS) { + windowRows += rows; + windowNanos += nanos; + if (windowRows < WINDOW_ROWS) { return; } - synchronized (this) { - if (windowRows.get() < WINDOW_ROWS) { - // A concurrent caller already consumed this window. - return; - } - tuneThroughput(windowRows.getAndSet(0L), windowNanos.getAndSet(0L)); - } + tuneThroughput(windowRows, windowNanos); + windowRows = 0L; + windowNanos = 0L; } /** Waits for a permit while checking cancellation between bounded waits. */ @@ -97,14 +96,14 @@ private void tuneThroughput(long rows, long nanos) { try { double throughput = rows * 1_000_000.0D / Math.max(1L, nanos); if (lastThroughput > 0.0D) { - if (throughput > lastThroughput) { + if (throughput > lastThroughput * GROW_MARGIN) { // Additive increase, capped by the hard total so growth cannot overshoot the // configured ceiling even while workers hold permits. if (totalPermits.get() < maxPermits) { release(); totalPermits.incrementAndGet(); } - } else if (throughput < lastThroughput && totalPermits.get() > floor) { + } else if (throughput < lastThroughput * SHRINK_MARGIN && totalPermits.get() > floor) { // Multiplicative decrease: a regression cuts fast, growth is careful so a // lucky window cannot oversubscribe the target system. int cut = Math.max(1, totalPermits.get() / 4); diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/ImportRowBatcher.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/ImportRowBatcher.java index a75db3a158..f9c5fd7acf 100644 --- a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/ImportRowBatcher.java +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/ImportRowBatcher.java @@ -290,6 +290,11 @@ private void executePendingBatch(PendingBatch batch) { try { DefaultSQLExecutor.getInstance().executeJdbcBatchInsert( Chat2DBContext.getConnection(), batch.sqls(), context, this::checkActive); + long elapsed = System.nanoTime() - started; + if (gate != null) { + gate.record(rows, elapsed); + } + batchSizer.record(rows, elapsed); importedCount.add(rows); reportProgress(); context.logInfo("BATCH_EXECUTED", "SQL batch executed", @@ -305,11 +310,6 @@ private void executePendingBatch(PendingBatch batch) { } throw batchFailure; } finally { - long elapsed = System.nanoTime() - started; - if (gate != null) { - gate.record(rows, elapsed); - } - batchSizer.record(rows, elapsed); if (workerPool != null) { batchCompleted(); } diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/AdaptiveConcurrencyGateTest.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/AdaptiveConcurrencyGateTest.java index ea7ffee264..98a0c373ce 100644 --- a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/AdaptiveConcurrencyGateTest.java +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/AdaptiveConcurrencyGateTest.java @@ -97,6 +97,44 @@ void keepsAStableThroughputFlat() { assertEquals(2, gate.currentPermits()); } + @Test + void ignoresThroughputChangesWithinTenPercentIncludingBoundaries() { + AdaptiveConcurrencyGate gate = AdaptiveConcurrencyGate.create(2, 4); + gate.record(100_000, 100 * MILLI); + gate.record(110_000, 100 * MILLI); + assertEquals(2, gate.totalPermits()); + gate.record(100_000, 100 * MILLI); + gate.record(90_000, 100 * MILLI); + assertEquals(2, gate.totalPermits()); + gate.record(100_000, 100 * MILLI); + assertEquals(3, gate.totalPermits()); + } + + @Test + void concurrentSamplesWithEqualEfficiencyDoNotChangePermits() throws Exception { + AdaptiveConcurrencyGate gate = AdaptiveConcurrencyGate.create(2, 4); + var executor = Executors.newFixedThreadPool(4); + CountDownLatch start = new CountDownLatch(1); + var futures = new java.util.ArrayList>(); + try { + for (int worker = 1; worker <= 4; worker++) { + long rows = worker * 10_000L; + futures.add(executor.submit(() -> { + start.await(); + for (int sample = 0; sample < 2_000; sample++) { + gate.record(rows, rows * MILLI); + } + return null; + })); + } + start.countDown(); + for (var future : futures) future.get(10, TimeUnit.SECONDS); + assertEquals(2, gate.totalPermits()); + } finally { + executor.shutdownNow(); + } + } + @Test void ignoresInvalidObservations() { AdaptiveConcurrencyGate gate = AdaptiveConcurrencyGate.create(2, 4); From c332361262a81a0cbb06ac3021f244f12d2fc9e6 Mon Sep 17 00:00:00 2001 From: zgq Date: Tue, 15 Sep 2026 14:47:37 +0800 Subject: [PATCH 31/38] refactor(import): remove parallelism system property override --- .../impl/task/imports/ImportRowBatcher.java | 21 +------------------ .../task/ImportRowBatcherParallelTest.java | 18 +--------------- .../task/ParallelImportLifecycleTest.java | 4 ---- 3 files changed, 2 insertions(+), 41 deletions(-) diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/ImportRowBatcher.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/ImportRowBatcher.java index f9c5fd7acf..0162285322 100644 --- a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/ImportRowBatcher.java +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/ImportRowBatcher.java @@ -145,14 +145,7 @@ public ImportRowBatcher(ImportTaskSpec spec, TaskExecutionContext context, Resol for (int index = 0; index < requestedWorkers; index++) { builtQueues.add(new ArrayBlockingQueue<>(QUEUE_CAPACITY)); } - // The fan-out may grow, but never past the machine's available parallelism: the - // The adaptive gate stays inside [1, machineThreadCeiling()], and an explicit - // chat2db.task.import.parallelism pin is bounded by the same ceiling. - int gateCeiling = parallelismPinned() - ? Math.min(requestedWorkers, machineThreadCeiling()) - : machineThreadCeiling(); - builtGate = AdaptiveConcurrencyGate.create(Math.min(BASE_WORKERS, requestedWorkers), - gateCeiling); + builtGate = AdaptiveConcurrencyGate.create(requestedWorkers, machineThreadCeiling()); builtPool = Executors.newCachedThreadPool(runnable -> { Thread thread = new Thread(runnable, "chat2db-import-" + context.taskId()); thread.setDaemon(true); @@ -329,22 +322,10 @@ private static int machineThreadCeiling() { return Math.max(1, Runtime.getRuntime().availableProcessors()); } - /** Whether {@code chat2db.task.import.parallelism} pins the fan-out explicitly. */ - private static boolean parallelismPinned() { - return Integer.getInteger("chat2db.task.import.parallelism", 0) > 1; - } - private static int effectiveWorkerCount(ConnectInfo connectInfo) { if (StringUtils.isBlank(connectInfo.getUrl())) { return 1; } - int configured = Integer.getInteger("chat2db.task.import.parallelism", 0); - if (configured == 1) { - return 1; - } - if (configured > 1) { - return Math.min(configured, machineThreadCeiling()); - } return Math.min(BASE_WORKERS, machineThreadCeiling()); } diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/ImportRowBatcherParallelTest.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/ImportRowBatcherParallelTest.java index 36fb1a9416..9a3613ed7d 100644 --- a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/ImportRowBatcherParallelTest.java +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/ImportRowBatcherParallelTest.java @@ -56,8 +56,6 @@ class ImportRowBatcherParallelTest { private static final String DB_TYPE = "PARALLEL_IMPORT_TEST"; - private static final String PARALLELISM_PROPERTY = "chat2db.task.import.parallelism"; - private static final String H2_DRIVER_NAME = "parallel-import-test-h2.jar"; private static String previousUserHome; @@ -68,7 +66,6 @@ class ImportRowBatcherParallelTest { private java.sql.Connection connection; private IPlugin previousPlugin; private InMemoryTaskStorage storage; - private String previousParallelism; @BeforeAll static void isolateHomeAndSeedDriver() throws Exception { @@ -92,7 +89,6 @@ static void restoreHome() { @BeforeEach void setUp() throws Exception { - previousParallelism = System.clearProperty(PARALLELISM_PROPERTY); DBConfig config = new DBConfig(); config.setDbType(DB_TYPE); config.setDefaultDriverConfig(new DriverConfig()); @@ -130,11 +126,6 @@ void tearDown() throws Exception { } else { Chat2DBContext.PLUGIN_MAP.put(DB_TYPE, previousPlugin); } - if (previousParallelism == null) { - System.clearProperty(PARALLELISM_PROPERTY); - } else { - System.setProperty(PARALLELISM_PROPERTY, previousParallelism); - } connection.close(); } @@ -190,7 +181,6 @@ private List importedIds() throws Exception { @Test void parallelWorkersInsertEveryRowExactlyOnce() throws Exception { - System.setProperty(PARALLELISM_PROPERTY, "4"); // Enough rows that even the contract baseline batch (20000 rows) splits into many batches, // so the assertion below observes real overlap instead of a single-batch edge case. int rows = 200_000; @@ -211,15 +201,12 @@ void parallelWorkersInsertEveryRowExactlyOnce() throws Exception { assertTrue(tuning.batches() > 1, "the import must be split into several batches"); assertTrue(tuning.peakInFlightBatches() > 1, "the producer must have more than one submitted batch in flight"); - assertTrue(tuning.gatePermits() <= 4, - "an explicit parallelism pin must cap the adaptive fan-out, got " + tuning.gatePermits()); assertTrue(tuning.gatePermits() <= Runtime.getRuntime().availableProcessors(), "the fan-out must never exceed the machine's available parallelism"); } @Test void parallelImportUsesParsedRowsWithQuotedNewlines() throws Exception { - System.setProperty(PARALLELISM_PROPERTY, "2"); ImportTaskSpec spec = csvSpec(writeCsv("1,\"Alice\nCooper\"", "2,Bob")); new CSVImporter().run(spec, contextFor(spec)); @@ -230,13 +217,12 @@ void parallelImportUsesParsedRowsWithQuotedNewlines() throws Exception { assertTrue(rows.next()); assertEquals("Alice\nCooper", rows.getString(1)); } - assertEquals(Math.min(2, Runtime.getRuntime().availableProcessors()), + assertEquals(Math.min(4, Runtime.getRuntime().availableProcessors()), ImportRowBatcher.lastTuningSnapshot().workers()); } @Test void excelPreservesSequentialFailureEvenWithAFastModeField() throws Exception { - System.setProperty(PARALLELISM_PROPERTY, "2"); Path workbook = tempDirectory.resolve("bulk.xlsx"); com.alibaba.excel.EasyExcel.write(workbook.toFile()) .head(List.of(List.of("ID"), List.of("NAME"))) @@ -252,7 +238,6 @@ void excelPreservesSequentialFailureEvenWithAFastModeField() throws Exception { @Test void failedFinalBatchPropagatesWorkerFailureAndKeepsCommittedRows() throws Exception { - System.setProperty(PARALLELISM_PROPERTY, "2"); ImportTaskSpec spec = csvSpec(writeCsv("1,ok", "1,duplicate", "2,ok")); TaskExecutionContextImpl context = contextFor(spec); @@ -266,7 +251,6 @@ void failedFinalBatchPropagatesWorkerFailureAndKeepsCommittedRows() throws Excep @Test void failedParallelBatchesReportFailureWithPartialWrites() throws Exception { - System.setProperty(PARALLELISM_PROPERTY, "2"); String[] lines = new String[80_000]; for (int index = 0; index < lines.length; index++) { // Every batch contains constraint violations, so the task must report failure. diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/ParallelImportLifecycleTest.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/ParallelImportLifecycleTest.java index 67e7c0feaf..fe63e1d5d5 100644 --- a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/ParallelImportLifecycleTest.java +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/ParallelImportLifecycleTest.java @@ -48,7 +48,6 @@ class ParallelImportLifecycleTest { private ConnectInfo connectInfo; private Context requestContext; private IPlugin previousPlugin; - private String previousParallelism; private final List opened = new CopyOnWriteArrayList<>(); private final List progress = new CopyOnWriteArrayList<>(); private final List batchChars = new CopyOnWriteArrayList<>(); @@ -64,7 +63,6 @@ class ParallelImportLifecycleTest { @BeforeEach void setUp() throws Exception { - previousParallelism = System.setProperty("chat2db.task.import.parallelism", "2"); String url = "jdbc:h2:mem:parallel_lifecycle_" + System.nanoTime() + ";DB_CLOSE_DELAY=-1"; observer = DriverManager.getConnection(url); try (Statement statement = observer.createStatement()) { @@ -118,8 +116,6 @@ void tearDown() throws Exception { ConnectionPool.removeConnection(DATASOURCE_ID); if (previousPlugin == null) Chat2DBContext.PLUGIN_MAP.remove(TYPE); else Chat2DBContext.PLUGIN_MAP.put(TYPE, previousPlugin); - if (previousParallelism == null) System.clearProperty("chat2db.task.import.parallelism"); - else System.setProperty("chat2db.task.import.parallelism", previousParallelism); } @Test From 2effb6848eb2e48f302e04e8f3f89bfa7c025dcd Mon Sep 17 00:00:00 2001 From: zgq Date: Tue, 15 Sep 2026 15:06:40 +0800 Subject: [PATCH 32/38] refactor(import): share existing row-to-SQL conversion between modes --- .../impl/task/imports/ImportRowBatcher.java | 72 +------- .../task/imports/ImportRowSqlBuilder.java | 154 ++++++++++++++++++ .../task/imports/excel/BaseExcelImporter.java | 140 +--------------- .../imports/excel/ParallelCSVImporter.java | 17 +- 4 files changed, 172 insertions(+), 211 deletions(-) create mode 100644 chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/ImportRowSqlBuilder.java diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/ImportRowBatcher.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/ImportRowBatcher.java index 0162285322..1fa58d23e1 100644 --- a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/ImportRowBatcher.java +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/ImportRowBatcher.java @@ -1,26 +1,16 @@ package ai.chat2db.community.domain.core.impl.task.imports; -import ai.chat2db.community.domain.api.model.metadata.DataType; -import ai.chat2db.community.domain.api.model.metadata.TableColumn; -import ai.chat2db.community.domain.api.model.task.CsvOptions; -import ai.chat2db.community.domain.api.model.task.ImportTaskSpec; import ai.chat2db.community.domain.api.model.task.TaskCancelledException; import ai.chat2db.community.domain.api.model.task.TaskErrorCode; import ai.chat2db.community.domain.api.model.task.TaskExecutionException; import ai.chat2db.community.domain.api.model.task.TaskStage; import ai.chat2db.community.tools.model.Context; import ai.chat2db.community.tools.util.ContextUtils; -import ai.chat2db.community.domain.api.model.value.SQLDataValue; import ai.chat2db.community.domain.api.service.task.TaskExecutionContext; import ai.chat2db.community.domain.core.impl.task.AdaptiveBatchSizer; import ai.chat2db.community.domain.core.impl.task.AdaptiveConcurrencyGate; -import ai.chat2db.community.domain.core.impl.task.imports.ImportColumnResolver.Resolution; -import ai.chat2db.community.domain.core.impl.task.imports.excel.CsvImportValueNormalizer; -import ai.chat2db.spi.ISqlBuilder; import ai.chat2db.spi.DefaultSQLExecutor; -import ai.chat2db.spi.IValueProcessor; import ai.chat2db.spi.model.datasource.ConnectInfo; -import ai.chat2db.spi.model.request.SingleInsertSqlRequest; import ai.chat2db.spi.sql.Chat2DBContext; import ai.chat2db.spi.sql.ConnectionPool; import lombok.extern.slf4j.Slf4j; @@ -61,18 +51,8 @@ public final class ImportRowBatcher implements AutoCloseable { /** Contract baseline fan-out of the fast mode; the adaptive gate grows it further on demand. */ private static final int BASE_WORKERS = 4; - private final ImportTaskSpec spec; - private final TaskExecutionContext context; - private final Resolution resolution; - - private final CsvOptions csvOptions; - - private final IValueProcessor valueProcessor; - - private final ISqlBuilder sqlBuilder; - private final ConnectInfo connectInfo; private final Context requestContext; @@ -122,14 +102,8 @@ public final class ImportRowBatcher implements AutoCloseable { private volatile long totalImportNanos; - public ImportRowBatcher(ImportTaskSpec spec, TaskExecutionContext context, Resolution resolution, - IValueProcessor valueProcessor) { - this.spec = spec; + public ImportRowBatcher(TaskExecutionContext context) { this.context = context; - this.resolution = resolution; - this.csvOptions = spec.getCsvOptions() == null ? null : spec.getCsvOptions().validate(); - this.valueProcessor = valueProcessor; - this.sqlBuilder = Chat2DBContext.getSqlBuilder(); this.connectInfo = Chat2DBContext.getConnectInfo(); this.requestContext = ContextUtils.queryContext(); this.statementGuard = Chat2DBContext.captureStatementGuard(); @@ -176,19 +150,18 @@ public ImportRowBatcher(ImportTaskSpec spec, TaskExecutionContext context, Resol } } - public void accept(long fileRowNumber, List fileValues) { + public void accept(long fileRowNumber, String sql) { try { - acceptRow(fileRowNumber, fileValues); + acceptRow(fileRowNumber, sql); } catch (RuntimeException taskFailure) { recordFailure(taskFailure); throw taskFailure; } } - private void acceptRow(long fileRowNumber, List fileValues) { + private void acceptRow(long fileRowNumber, String sql) { context.checkCancelled(); throwIfFailed(); - String sql = buildInsert(fileRowNumber, fileValues); if (!bufferedSqls.isEmpty() && bufferedChars + sql.length() > MAX_BATCH_CHARS) { flushBufferedBatch(); } @@ -479,43 +452,6 @@ private void throwIfFailed() { private record PendingBatch(List sqls, long seq, long firstRowNumber) { } - private String buildInsert(long fileRowNumber, List fileValues) { - List tableColumnNames = new ArrayList<>(resolution.tableColumns().size()); - List values = new ArrayList<>(resolution.tableColumns().size()); - for (int index = 0; index < resolution.tableColumns().size(); index++) { - TableColumn column = resolution.tableColumns().get(index); - Integer sourceIndex = resolution.fileIndexes().get(index); - String raw = sourceIndex != null && sourceIndex < fileValues.size() - ? fileValues.get(sourceIndex) : null; - tableColumnNames.add(column.getName()); - values.add(toSqlLiteral(column, raw, fileRowNumber)); - } - return sqlBuilder.dml().buildInsert(SingleInsertSqlRequest.builder() - .databaseName(connectInfo.getDatabaseName()) - .schemaName(connectInfo.getSchemaName()) - .tableName(spec.getTarget().getTableName()) - .columnList(tableColumnNames) - .valueList(values) - .build()); - } - - private String toSqlLiteral(TableColumn column, String raw, long fileRowNumber) { - if (raw == null) { - return null; - } - if (csvOptions != null) { - raw = CsvImportValueNormalizer.normalize(raw, column, csvOptions, fileRowNumber); - } - DataType dataType = new DataType(); - dataType.setDataTypeName(column.getColumnType()); - dataType.setScale(column.getDecimalDigits()); - dataType.setPrecision(column.getColumnSize()); - SQLDataValue sqlDataValue = new SQLDataValue(); - sqlDataValue.setDataType(dataType); - sqlDataValue.setValue(raw); - return valueProcessor.getSqlValueString(sqlDataValue); - } - /** Stops pending writes when the source parser fails outside the batch executor. */ public void abort(RuntimeException sourceFailure) { recordFailure(sourceFailure); diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/ImportRowSqlBuilder.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/ImportRowSqlBuilder.java new file mode 100644 index 0000000000..74bcebe347 --- /dev/null +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/ImportRowSqlBuilder.java @@ -0,0 +1,154 @@ +package ai.chat2db.community.domain.core.impl.task.imports; + +import ai.chat2db.community.domain.api.model.metadata.DataType; +import ai.chat2db.community.domain.api.model.metadata.TableColumn; +import ai.chat2db.community.domain.api.model.value.SQLDataValue; +import ai.chat2db.community.domain.api.model.task.CsvOptions; +import ai.chat2db.community.domain.api.model.task.ImportColumnMapping; +import ai.chat2db.community.domain.api.model.task.ImportTaskSpec; +import ai.chat2db.community.domain.api.model.task.UnmappedTargetStrategy; +import ai.chat2db.community.domain.core.impl.task.imports.excel.CsvImportValueNormalizer; +import ai.chat2db.spi.ISqlBuilder; +import ai.chat2db.spi.IValueProcessor; +import ai.chat2db.spi.model.datasource.ConnectInfo; +import ai.chat2db.spi.model.request.SingleInsertSqlRequest; +import ai.chat2db.spi.sql.Chat2DBContext; +import org.apache.commons.lang3.StringUtils; + +import java.util.*; + +/** Maps an import row to INSERT SQL using the existing import conversion rules. */ +public final class ImportRowSqlBuilder { + private final ImportTaskSpec spec; + private final List columns; + private final IValueProcessor valueProcessor; + private final ConnectInfo connectInfo; + private final ISqlBuilder sqlBuilder; + private final CsvOptions csvOptions; + private Map headMap; + private Map mappedHeadMap; + private List tableColumns; + private List tableColumnList; + + public ImportRowSqlBuilder(ImportTaskSpec spec, List columns) { + this.spec = spec; + this.columns = columns; + this.valueProcessor = Chat2DBContext.getDbMetaData().getValueProcessor(); + this.connectInfo = Chat2DBContext.getConnectInfo(); + this.sqlBuilder = Chat2DBContext.getSqlBuilder(); + this.csvOptions = spec.getCsvOptions() == null ? null : spec.getCsvOptions().validate(); + } + + public void acceptHead(Map headers) { + this.headMap = invertMap(headers); + this.mappedHeadMap = mappedHeadMap(); + this.tableColumns = getTableColumns(columns, headMap); + } + + public String build(Map row, long sourceRowNumber) { + return getInsertSql(getValueList(row, sourceRowNumber)); + } + + private List getTableColumns(List columns, Map headMap) { + List tableColumns = new ArrayList<>(); + this.tableColumnList = new ArrayList<>(); + for (TableColumn column : columns) { + if (shouldInclude(column)) { + tableColumns.add(column); + this.tableColumnList.add(column.getName()); + } + } + return tableColumns; + } + + private Map invertMap(Map map) { + Map out = new HashMap(map.size()); + Iterator it = map.entrySet().iterator(); + while (it.hasNext()) { + Map.Entry entry = (Map.Entry) it.next(); + if (entry.getValue() != null) { + out.put(entry.getValue().toUpperCase(Locale.ROOT), entry.getKey()); + } + } + return out; + } + + + private List getValueList(Map data, long sourceRowNumber) { + List values = new ArrayList<>(); + for (TableColumn column : tableColumns) { + Integer index = sourceIndex(column.getName()); + if (index == null) { + values.add(null); + continue; + } + String value = data.get(index); + if (value == null) { + values.add(null); + } else { + if (csvOptions != null) { + value = CsvImportValueNormalizer.normalize(value, column, csvOptions, sourceRowNumber); + } + String stringValue = valueProcessor.getSqlValueString(getSQLDataValue(value, column)); + values.add(stringValue); + } + } + return values; + } + + private Map mappedHeadMap() { + Map mapped = new HashMap<>(); + if (spec.getColumnMappings() == null) { + return mapped; + } + for (ImportColumnMapping mapping : spec.getColumnMappings()) { + String source = mapping.getSourceColumn(); + String target = mapping.getTargetColumn(); + Integer sourceIndex = headMap.get(source == null ? null : source.toUpperCase(Locale.ROOT)); + if (sourceIndex != null && StringUtils.isNotBlank(target)) { + mapped.put(target.toUpperCase(Locale.ROOT), sourceIndex); + } + } + return mapped; + } + + private Integer sourceIndex(String targetColumn) { + String target = targetColumn.toUpperCase(Locale.ROOT); + if (spec.getColumnMappings() != null) { + return mappedHeadMap.get(target); + } + return headMap.get(target); + } + + private boolean shouldInclude(TableColumn column) { + if (spec.getColumnMappings() == null) { + return sourceIndex(column.getName()) != null; + } + if (sourceIndex(column.getName()) != null) { + return true; + } + return spec.getUnmappedTarget() == UnmappedTargetStrategy.NULL + && !Boolean.TRUE.equals(column.getAutoIncrement()); + } + + private String getInsertSql(List values) { + return sqlBuilder.dml().buildInsert(SingleInsertSqlRequest.builder() + .databaseName(connectInfo.getDatabaseName()) + .schemaName(connectInfo.getSchemaName()) + .tableName(spec.getTarget().getTableName()) + .columnList(this.tableColumnList) + .valueList(values) + .build()); + } + + private SQLDataValue getSQLDataValue(String value, TableColumn column) { + DataType dataType = new DataType(); + dataType.setDataTypeName(column.getColumnType()); + dataType.setScale(column.getDecimalDigits()); + dataType.setPrecision(column.getColumnSize()); + SQLDataValue sqlDataValue = new SQLDataValue(); + sqlDataValue.setDataType(dataType); + sqlDataValue.setValue(value); + return sqlDataValue; + } +} diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/excel/BaseExcelImporter.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/excel/BaseExcelImporter.java index 7958050975..abca725fdd 100644 --- a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/excel/BaseExcelImporter.java +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/excel/BaseExcelImporter.java @@ -2,21 +2,14 @@ import ai.chat2db.community.domain.core.impl.task.imports.BaseImporter; import ai.chat2db.community.domain.core.impl.task.imports.ImportSqlExecutor; +import ai.chat2db.community.domain.core.impl.task.imports.ImportRowSqlBuilder; import ai.chat2db.community.domain.api.model.task.TaskConstants; import ai.chat2db.community.domain.api.model.task.TaskCancelledException; import ai.chat2db.community.domain.api.model.task.ImportTaskSpec; -import ai.chat2db.community.domain.api.model.task.ImportColumnMapping; -import ai.chat2db.community.domain.api.model.task.CsvOptions; import ai.chat2db.community.domain.api.model.task.TaskEventCode; import ai.chat2db.community.domain.api.model.task.TaskStage; -import ai.chat2db.community.domain.api.model.task.UnmappedTargetStrategy; import ai.chat2db.community.domain.api.service.task.TaskExecutionContext; -import ai.chat2db.spi.ISqlBuilder; -import ai.chat2db.spi.IValueProcessor; import ai.chat2db.community.domain.api.model.metadata.TableColumn; -import ai.chat2db.spi.sql.Chat2DBContext; -import ai.chat2db.spi.model.datasource.ConnectInfo; -import ai.chat2db.spi.model.request.SingleInsertSqlRequest; import com.alibaba.excel.EasyExcel; import com.alibaba.excel.context.AnalysisContext; import com.alibaba.excel.event.AnalysisEventListener; @@ -50,21 +43,8 @@ protected void doImportData(ImportTaskSpec spec, TaskExecutionContext context, L public class NoModelDataListener extends AnalysisEventListener> { - - private final ImportTaskSpec spec; - private final TaskExecutionContext taskContext; - private final List columns; - - private Map headMap; - - private Map mappedHeadMap; - - private List tableColumns; - - private List tableColumnList; - private List sqlList; private long successCount; @@ -73,28 +53,15 @@ public class NoModelDataListener extends AnalysisEventListener columns) { - this.spec = spec; - this.columns = columns; this.taskContext = taskContext; - this.valueProcessor = Chat2DBContext.getDbMetaData().getValueProcessor(); - this.connectInfo = Chat2DBContext.getConnectInfo(); - this.sqlBuilder = Chat2DBContext.getSqlBuilder(); this.sqlExecutor = new ImportSqlExecutor(taskContext); - this.csvOptions = spec.getCsvOptions() == null ? null : spec.getCsvOptions().validate(); + this.rowSqlBuilder = new ImportRowSqlBuilder(spec, columns); } @@ -105,36 +72,9 @@ public void invokeHead(Map> headMap, AnalysisContext co void acceptHead(Map map) { this.taskContext.checkCancelled(); - this.headMap = invertMap(map); - this.mappedHeadMap = mappedHeadMap(); - this.tableColumns = getTableColumns(columns, this.headMap); - } - - private List getTableColumns(List columns, Map headMap) { - List tableColumns = new ArrayList<>(); - this.tableColumnList = new ArrayList<>(); - for (TableColumn column : columns) { - if (shouldInclude(column)) { - tableColumns.add(column); - this.tableColumnList.add(column.getName()); - } - } - return tableColumns; + rowSqlBuilder.acceptHead(map); } - private Map invertMap(Map map) { - Map out = new HashMap(map.size()); - Iterator it = map.entrySet().iterator(); - while (it.hasNext()) { - Map.Entry entry = (Map.Entry) it.next(); - if (entry.getValue() != null) { - out.put(entry.getValue().toUpperCase(Locale.ROOT), entry.getKey()); - } - } - return out; - } - - @Override public void invoke(Map data, AnalysisContext context) { acceptRow(data); @@ -146,14 +86,11 @@ void acceptRow(Map data) { void acceptRow(Map data, long sourceRowNumber) { this.taskContext.checkCancelled(); - this.sourceRowNumber = sourceRowNumber; if (data == null || data.isEmpty()) { skippedCount++; return; } - List values = getValueList(data); - - String sql = getInsertSql(values); + String sql = rowSqlBuilder.build(data, sourceRowNumber); if (StringUtils.isBlank(sql)) { skippedCount++; @@ -170,73 +107,6 @@ void acceptRow(Map data, long sourceRowNumber) { } } - private List getValueList(Map data) { - List values = new ArrayList<>(); - for (TableColumn column : tableColumns) { - Integer index = sourceIndex(column.getName()); - if (index == null) { - values.add(null); - continue; - } - String value = data.get(index); - if (value == null) { - values.add(null); - } else { - if (csvOptions != null) { - value = CsvImportValueNormalizer.normalize(value, column, csvOptions, sourceRowNumber); - } - String stringValue = valueProcessor.getSqlValueString(getSQLDataValue(value, column)); - values.add(stringValue); - } - } - return values; - } - - private Map mappedHeadMap() { - Map mapped = new HashMap<>(); - if (spec.getColumnMappings() == null) { - return mapped; - } - for (ImportColumnMapping mapping : spec.getColumnMappings()) { - String source = mapping.getSourceColumn(); - String target = mapping.getTargetColumn(); - Integer sourceIndex = headMap.get(source == null ? null : source.toUpperCase(Locale.ROOT)); - if (sourceIndex != null && StringUtils.isNotBlank(target)) { - mapped.put(target.toUpperCase(Locale.ROOT), sourceIndex); - } - } - return mapped; - } - - private Integer sourceIndex(String targetColumn) { - String target = targetColumn.toUpperCase(Locale.ROOT); - if (spec.getColumnMappings() != null) { - return mappedHeadMap.get(target); - } - return headMap.get(target); - } - - private boolean shouldInclude(TableColumn column) { - if (spec.getColumnMappings() == null) { - return sourceIndex(column.getName()) != null; - } - if (sourceIndex(column.getName()) != null) { - return true; - } - return spec.getUnmappedTarget() == UnmappedTargetStrategy.NULL - && !Boolean.TRUE.equals(column.getAutoIncrement()); - } - - private String getInsertSql(List values) { - return sqlBuilder.dml().buildInsert(SingleInsertSqlRequest.builder() - .databaseName(connectInfo.getDatabaseName()) - .schemaName(connectInfo.getSchemaName()) - .tableName(spec.getTarget().getTableName()) - .columnList(this.tableColumnList) - .valueList(values) - .build()); - } - @Override public void doAfterAllAnalysed(AnalysisContext context) { finish(); diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/excel/ParallelCSVImporter.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/excel/ParallelCSVImporter.java index ddc2323b03..d0d9b152ec 100644 --- a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/excel/ParallelCSVImporter.java +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/excel/ParallelCSVImporter.java @@ -8,7 +8,7 @@ import ai.chat2db.community.domain.core.impl.task.imports.BaseImporter; import ai.chat2db.community.domain.core.impl.task.imports.ImportColumnResolver; import ai.chat2db.community.domain.core.impl.task.imports.ImportRowBatcher; -import ai.chat2db.spi.sql.Chat2DBContext; +import ai.chat2db.community.domain.core.impl.task.imports.ImportRowSqlBuilder; import java.nio.file.Path; import java.util.ArrayList; @@ -24,12 +24,13 @@ protected void doImportData(ImportTaskSpec spec, TaskExecutionContext context, CsvOptions options = (spec.getCsvOptions() == null ? CsvOptions.defaults() : spec.getCsvOptions()).validate(); spec.setCsvOptions(options); ImportRowBatcher[] batcher = {null}; + ImportRowSqlBuilder rowSqlBuilder = new ImportRowSqlBuilder(spec, columns); int[] sourceRow = {0}; try { new CsvParser(options).forEachRow(Path.of(spec.getSourceFile()), row -> { int rowNumber = ++sourceRow[0]; if (Boolean.TRUE.equals(options.getHasHeader()) && rowNumber == options.getHeaderRow()) { - batcher[0] = createBatcher(spec, context, columns, values(row)); + batcher[0] = createBatcher(spec, context, columns, row, rowSqlBuilder); return; } if (rowNumber < options.getDataStartRow() @@ -38,9 +39,9 @@ protected void doImportData(ImportTaskSpec spec, TaskExecutionContext context, } if (batcher[0] == null) { int width = Math.max(row.size(), CSVImporter.mappedSourceColumnCount(spec)); - batcher[0] = createBatcher(spec, context, columns, values(CSVImporter.syntheticHeader(width))); + batcher[0] = createBatcher(spec, context, columns, CSVImporter.syntheticHeader(width), rowSqlBuilder); } - batcher[0].accept(rowNumber, values(row)); + batcher[0].accept(rowNumber, rowSqlBuilder.build(row, rowNumber)); }, context::checkCancelled); if (batcher[0] != null) { batcher[0].flush(); @@ -60,12 +61,12 @@ protected void doImportData(ImportTaskSpec spec, TaskExecutionContext context, } private ImportRowBatcher createBatcher(ImportTaskSpec spec, TaskExecutionContext context, - List columns, List headers) { - ImportColumnResolver.Resolution resolution = ImportColumnResolver.resolveForSpec(columns, headers, spec); + List columns, Map headers, ImportRowSqlBuilder rowSqlBuilder) { + ImportColumnResolver.Resolution resolution = ImportColumnResolver.resolveForSpec(columns, values(headers), spec); ImportColumnResolver.validateForImport(columns, resolution, spec); - return new ImportRowBatcher(spec, context, resolution, - Chat2DBContext.getDbMetaData().getValueProcessor()); + rowSqlBuilder.acceptHead(headers); + return new ImportRowBatcher(context); } private static List values(Map row) { From 680176a7129b2a9c9c099544b6589853ca922c47 Mon Sep 17 00:00:00 2001 From: zgq Date: Tue, 15 Sep 2026 15:43:02 +0800 Subject: [PATCH 33/38] refactor(import): use existing mapping behavior in fast mode --- .../task/imports/ImportColumnResolver.java | 108 ------------------ .../imports/excel/ParallelCSVImporter.java | 23 +--- .../excel/CSVImporterColumnMappingTest.java | 26 +++-- 3 files changed, 22 insertions(+), 135 deletions(-) delete mode 100644 chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/ImportColumnResolver.java diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/ImportColumnResolver.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/ImportColumnResolver.java deleted file mode 100644 index 189b6479a0..0000000000 --- a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/ImportColumnResolver.java +++ /dev/null @@ -1,108 +0,0 @@ -package ai.chat2db.community.domain.core.impl.task.imports; - -import ai.chat2db.community.domain.api.model.metadata.TableColumn; -import ai.chat2db.community.domain.api.model.task.ImportColumnMapping; -import ai.chat2db.community.domain.api.model.task.ImportTaskSpec; -import ai.chat2db.community.domain.api.model.task.UnmappedTargetStrategy; -import ai.chat2db.community.tools.exception.ParamBusinessException; -import org.apache.commons.lang3.StringUtils; - -import java.util.ArrayList; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; - -/** - * Resolves which file column feeds which table column. Explicit mappings win; otherwise matching is - * case-insensitive, preserving whitespace as in the existing preview and ordinary importer. - */ -public final class ImportColumnResolver { - - /** - * Ordered pair lists: entry {@code i} binds {@code fileValues[fileIndexes[i]]} to - * {@code tableColumns[i]}. - */ - public record Resolution(List tableColumns, List fileIndexes, - List missingTableColumns) { - } - - private ImportColumnResolver() { - } - - public static Resolution resolveForSpec(List tableColumns, List fileHeaders, - ImportTaskSpec spec) { - return resolve(tableColumns, fileHeaders, spec.getColumnMappings(), spec.getUnmappedTarget()); - } - - public static void validateForImport(List columns, Resolution resolution, ImportTaskSpec spec) { - if (resolution.fileIndexes().stream().noneMatch(java.util.Objects::nonNull)) { - throw new ParamBusinessException("At least one import column mapping is required"); - } - for (TableColumn column : columns) { - if (resolution.missingTableColumns().contains(column.getName()) - && Integer.valueOf(0).equals(column.getNullable()) - && !Boolean.TRUE.equals(column.getAutoIncrement()) - && (spec.getUnmappedTarget() == UnmappedTargetStrategy.NULL || column.getDefaultValue() == null)) { - throw new ParamBusinessException("Required import column is unmapped: " + column.getName()); - } - } - } - - private static Resolution resolve(List tableColumns, List fileHeaders, - List mappings, UnmappedTargetStrategy unmappedTarget) { - Map byNormalizedName = new LinkedHashMap<>(); - for (int index = 0; index < fileHeaders.size(); index++) { - if (byNormalizedName.putIfAbsent(normalize(fileHeaders.get(index)), index) != null) { - throw new ParamBusinessException("Duplicate import source column: " + fileHeaders.get(index)); - } - } - Map explicitTargets = new LinkedHashMap<>(); - java.util.Set explicitSources = new java.util.HashSet<>(); - java.util.Set knownTargets = tableColumns.stream().map(column -> normalize(column.getName())) - .collect(java.util.stream.Collectors.toSet()); - if (mappings != null) { - for (ImportColumnMapping mapping : mappings) { - if (mapping == null || StringUtils.isBlank(mapping.getSourceColumn()) - || StringUtils.isBlank(mapping.getTargetColumn())) { - throw new ParamBusinessException("columnMappings"); - } - Integer sourceIndex = byNormalizedName.get(normalize(mapping.getSourceColumn())); - if (sourceIndex == null) { - throw new ParamBusinessException("columnMappings source: " + mapping.getSourceColumn()); - } - String target = normalize(mapping.getTargetColumn()); - if (!knownTargets.contains(target) || !explicitSources.add(sourceIndex) - || explicitTargets.putIfAbsent(target, sourceIndex) != null) { - throw new ParamBusinessException("Duplicate or invalid import column mapping"); - } - } - } - - List resolvedColumns = new ArrayList<>(); - List fileIndexes = new ArrayList<>(); - List missingTableColumns = new ArrayList<>(); - for (TableColumn column : tableColumns) { - Integer sourceIndex = explicitTargets.get(normalize(column.getName())); - if (sourceIndex == null && mappings == null) { - sourceIndex = byNormalizedName.get(normalize(column.getName())); - } - if (sourceIndex != null) { - resolvedColumns.add(column); - fileIndexes.add(sourceIndex); - } else { - missingTableColumns.add(column.getName()); - if (mappings != null && unmappedTarget == UnmappedTargetStrategy.NULL - && !Boolean.TRUE.equals(column.getAutoIncrement())) { - resolvedColumns.add(column); - fileIndexes.add(null); - } - } - } - - return new Resolution(resolvedColumns, fileIndexes, missingTableColumns); - } - - private static String normalize(String name) { - return name == null ? "" : name.toUpperCase(java.util.Locale.ROOT); - } -} diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/excel/ParallelCSVImporter.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/excel/ParallelCSVImporter.java index d0d9b152ec..00b1e2715c 100644 --- a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/excel/ParallelCSVImporter.java +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/task/imports/excel/ParallelCSVImporter.java @@ -6,12 +6,10 @@ import ai.chat2db.community.domain.api.service.task.TaskExecutionContext; import ai.chat2db.community.domain.core.impl.db.CsvParser; import ai.chat2db.community.domain.core.impl.task.imports.BaseImporter; -import ai.chat2db.community.domain.core.impl.task.imports.ImportColumnResolver; import ai.chat2db.community.domain.core.impl.task.imports.ImportRowBatcher; import ai.chat2db.community.domain.core.impl.task.imports.ImportRowSqlBuilder; import java.nio.file.Path; -import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -30,7 +28,7 @@ protected void doImportData(ImportTaskSpec spec, TaskExecutionContext context, new CsvParser(options).forEachRow(Path.of(spec.getSourceFile()), row -> { int rowNumber = ++sourceRow[0]; if (Boolean.TRUE.equals(options.getHasHeader()) && rowNumber == options.getHeaderRow()) { - batcher[0] = createBatcher(spec, context, columns, row, rowSqlBuilder); + batcher[0] = createBatcher(context, row, rowSqlBuilder); return; } if (rowNumber < options.getDataStartRow() @@ -39,7 +37,7 @@ protected void doImportData(ImportTaskSpec spec, TaskExecutionContext context, } if (batcher[0] == null) { int width = Math.max(row.size(), CSVImporter.mappedSourceColumnCount(spec)); - batcher[0] = createBatcher(spec, context, columns, CSVImporter.syntheticHeader(width), rowSqlBuilder); + batcher[0] = createBatcher(context, CSVImporter.syntheticHeader(width), rowSqlBuilder); } batcher[0].accept(rowNumber, rowSqlBuilder.build(row, rowNumber)); }, context::checkCancelled); @@ -60,22 +58,9 @@ protected void doImportData(ImportTaskSpec spec, TaskExecutionContext context, } } - private ImportRowBatcher createBatcher(ImportTaskSpec spec, TaskExecutionContext context, - List columns, Map headers, ImportRowSqlBuilder rowSqlBuilder) { - ImportColumnResolver.Resolution resolution = ImportColumnResolver.resolveForSpec(columns, values(headers), spec); - - ImportColumnResolver.validateForImport(columns, resolution, spec); + private ImportRowBatcher createBatcher(TaskExecutionContext context, + Map headers, ImportRowSqlBuilder rowSqlBuilder) { rowSqlBuilder.acceptHead(headers); return new ImportRowBatcher(context); } - - private static List values(Map row) { - int count = row.keySet().stream().mapToInt(Integer::intValue).max().orElse(-1) + 1; - List values = new ArrayList<>(count); - for (int index = 0; index < count; index++) { - values.add(row.get(index)); - } - return values; - } - } diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/imports/excel/CSVImporterColumnMappingTest.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/imports/excel/CSVImporterColumnMappingTest.java index bab23d43e6..a3c3605ca1 100644 --- a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/imports/excel/CSVImporterColumnMappingTest.java +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/task/imports/excel/CSVImporterColumnMappingTest.java @@ -235,16 +235,26 @@ void csvRowRangeAndFormatsDriveThePersistedValues(@TempDir Path directory) throw } @Test - void fastModeRejectsDuplicateMappingBeforeWritingRows(@TempDir Path directory) throws Exception { + void bothModesKeepTheLastMappingForADuplicateTarget(@TempDir Path directory) throws Exception { Path input = Files.writeString(directory.resolve("duplicates.csv"), "Full Name,status\nAlice,OVERRIDE\n"); - ImportTaskSpec spec = ImportTaskSpec.builder().mode("FAST").sourceFile(input.toString()) - .target(TaskTargetSnapshot.builder().tableName("orders").build()) - .columnMappings(List.of(new ImportColumnMapping("Full Name", "name"), - new ImportColumnMapping("status", "name"))).build(); + for (String mode : List.of("STANDARD", "FAST")) { + try (Statement statement = connection.createStatement()) { + statement.execute("DELETE FROM orders"); + } + ImportTaskSpec spec = ImportTaskSpec.builder().mode(mode).sourceFile(input.toString()) + .target(TaskTargetSnapshot.builder().tableName("orders").build()) + .columnMappings(List.of(new ImportColumnMapping("Full Name", "name"), + new ImportColumnMapping("status", "name"))).build(); - assertThrows(RuntimeException.class, - () -> new CSVImporter().doImportData(spec, new RecordingTaskExecutionContext(), columns())); - assertRowCount(0); + new CSVImporter().doImportData(spec, new RecordingTaskExecutionContext(), columns()); + + try (Statement statement = connection.createStatement(); + ResultSet rows = statement.executeQuery("SELECT name FROM orders")) { + org.junit.jupiter.api.Assertions.assertTrue(rows.next()); + assertEquals("OVERRIDE", rows.getString(1), mode); + assertEquals(false, rows.next()); + } + } } @Test From 47b8f2fae4c454659202604891e0fb74aea0a76b Mon Sep 17 00:00:00 2001 From: zgq Date: Tue, 15 Sep 2026 16:10:49 +0800 Subject: [PATCH 34/38] refactor(import): restore existing target table display --- .../ImportAndExport/components/ImportExportFile/index.tsx | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/chat2db-community-client/src/blocks/ImportAndExport/components/ImportExportFile/index.tsx b/chat2db-community-client/src/blocks/ImportAndExport/components/ImportExportFile/index.tsx index 8cabe1ab21..b7bbbe8862 100644 --- a/chat2db-community-client/src/blocks/ImportAndExport/components/ImportExportFile/index.tsx +++ b/chat2db-community-client/src/blocks/ImportAndExport/components/ImportExportFile/index.tsx @@ -137,11 +137,9 @@ const ImportExportFile = forwardRef((props: IProps, ref: ForwardedRef - {isExport && ( - - - - )} + + +