diff --git a/CHANGES.txt b/CHANGES.txt index 3b382f539..c8842853e 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -5,6 +5,7 @@ 0.3.0 ----- + * Adding endpoint for verifying files post data copy during live migration (CASSSIDECAR-226) * RangeManager should be singleton in CDCModule (CASSSIDECAR-411) * CDC: Add end-to-end CDC integration tests (CASSSIDECAR-308) * SchemaStorePublisherFactory should be Injectable in CachingSchemaStore (CASSSIDECAR-408) diff --git a/client-common/src/main/java/org/apache/cassandra/sidecar/common/ApiEndpointsV1.java b/client-common/src/main/java/org/apache/cassandra/sidecar/common/ApiEndpointsV1.java index 024329f33..8e2ba0d15 100644 --- a/client-common/src/main/java/org/apache/cassandra/sidecar/common/ApiEndpointsV1.java +++ b/client-common/src/main/java/org/apache/cassandra/sidecar/common/ApiEndpointsV1.java @@ -171,6 +171,8 @@ public final class ApiEndpointsV1 public static final String LIVE_MIGRATION_DATA_COPY_TASK_ROUTE = LIVE_MIGRATION_DATA_COPY_TASKS_ROUTE + "/:taskId"; public static final String LIVE_MIGRATION_STATUS_ROUTE = LIVE_MIGRATION_API_PREFIX + "/status"; + public static final String LIVE_MIGRATION_FILES_VERIFICATION_TASKS_ROUTE = LIVE_MIGRATION_API_PREFIX + "/files-verification-tasks"; + public static final String LIVE_MIGRATION_FILES_VERIFICATION_TASK_ROUTE = LIVE_MIGRATION_FILES_VERIFICATION_TASKS_ROUTE + "/:taskId"; public static final String OPENAPI_JSON_ROUTE = "/spec/openapi.json"; public static final String OPENAPI_YAML_ROUTE = "/spec/openapi.yaml"; diff --git a/client-common/src/main/java/org/apache/cassandra/sidecar/common/request/LiveMigrationFileDigestRequest.java b/client-common/src/main/java/org/apache/cassandra/sidecar/common/request/LiveMigrationFileDigestRequest.java new file mode 100644 index 000000000..8d625f9ed --- /dev/null +++ b/client-common/src/main/java/org/apache/cassandra/sidecar/common/request/LiveMigrationFileDigestRequest.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.sidecar.common.request; + +import java.util.Objects; + +import io.netty.handler.codec.http.HttpMethod; +import org.apache.cassandra.sidecar.common.response.DigestResponse; + +/** + * Represents a request to retrieve file digest for validation during live migration. + * Supports configurable digest algorithms for verification. + */ +public class LiveMigrationFileDigestRequest extends JsonRequest +{ + public static final String DIGEST_ALGORITHM_PARAM = "digestAlgorithm"; + + /** + * Private constructor for internal use by the factory method + * + * @param fullRequestURI the complete URI with query parameters + */ + private LiveMigrationFileDigestRequest(String fullRequestURI) + { + super(fullRequestURI); + } + + /** + * Creates a live migration file digest request with validation + * + * @param requestURI the base URI of the request + * @param digestAlgorithm the digest algorithm to use (e.g., "MD5", "XXHash32") + * @return a new LiveMigrationFileDigestRequest instance + * @throws NullPointerException if requestURI or digestAlgorithm is null + * @throws IllegalArgumentException if digestAlgorithm is empty + */ + public static LiveMigrationFileDigestRequest create(String requestURI, String digestAlgorithm) + { + Objects.requireNonNull(requestURI, "requestURI cannot be null"); + Objects.requireNonNull(digestAlgorithm, "digestAlgorithm cannot be null"); + if (digestAlgorithm.isEmpty()) + { + throw new IllegalArgumentException("digestAlgorithm cannot be empty"); + } + + String fullURI = String.format("%s?%s=%s", requestURI, DIGEST_ALGORITHM_PARAM, digestAlgorithm); + + return new LiveMigrationFileDigestRequest(fullURI); + } + + @Override + public HttpMethod method() + { + return HttpMethod.GET; + } +} diff --git a/client-common/src/main/java/org/apache/cassandra/sidecar/common/request/LiveMigrationFilesVerificationRequest.java b/client-common/src/main/java/org/apache/cassandra/sidecar/common/request/LiveMigrationFilesVerificationRequest.java new file mode 100644 index 000000000..257d4a6ec --- /dev/null +++ b/client-common/src/main/java/org/apache/cassandra/sidecar/common/request/LiveMigrationFilesVerificationRequest.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.sidecar.common.request; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * Request to verify file integrity during live migration by computing and comparing digests. + * Supports configurable concurrency and digest algorithms. + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class LiveMigrationFilesVerificationRequest +{ + private final int maxConcurrency; + private final String digestAlgorithm; + + @JsonCreator + public LiveMigrationFilesVerificationRequest(@JsonProperty("maxConcurrency") int maxConcurrency, + @JsonProperty("digestAlgorithm") String digestAlgorithm) + { + if (maxConcurrency <= 0) + { + throw new IllegalArgumentException("maxConcurrency must be >= 1"); + } + if (digestAlgorithm == null || digestAlgorithm.trim().isEmpty()) + { + throw new IllegalArgumentException("digestAlgorithm cannot be null or empty"); + } + + this.maxConcurrency = maxConcurrency; + this.digestAlgorithm = digestAlgorithm; + } + + @JsonProperty("maxConcurrency") + public int maxConcurrency() + { + return maxConcurrency; + } + + @JsonProperty("digestAlgorithm") + public String digestAlgorithm() + { + return digestAlgorithm; + } + + @Override + public String toString() + { + return "LiveMigrationFilesVerificationRequest{" + + "maxConcurrency=" + maxConcurrency + + ", digestAlgorithm='" + digestAlgorithm + '\'' + + '}'; + } +} diff --git a/client-common/src/main/java/org/apache/cassandra/sidecar/common/request/data/MD5Digest.java b/client-common/src/main/java/org/apache/cassandra/sidecar/common/request/data/MD5Digest.java index 30c9c1e27..f8a1e4b76 100644 --- a/client-common/src/main/java/org/apache/cassandra/sidecar/common/request/data/MD5Digest.java +++ b/client-common/src/main/java/org/apache/cassandra/sidecar/common/request/data/MD5Digest.java @@ -30,6 +30,8 @@ */ public class MD5Digest implements Digest { + public static final String MD5_ALGORITHM = "MD5"; + private final @NotNull String value; /** @@ -54,7 +56,7 @@ public String value() @Override public String algorithm() { - return "MD5"; + return MD5_ALGORITHM; } /** diff --git a/client-common/src/main/java/org/apache/cassandra/sidecar/common/request/data/XXHash32Digest.java b/client-common/src/main/java/org/apache/cassandra/sidecar/common/request/data/XXHash32Digest.java index 7ed192a06..56e7fffb0 100644 --- a/client-common/src/main/java/org/apache/cassandra/sidecar/common/request/data/XXHash32Digest.java +++ b/client-common/src/main/java/org/apache/cassandra/sidecar/common/request/data/XXHash32Digest.java @@ -33,6 +33,7 @@ */ public class XXHash32Digest implements Digest { + public static final String XXHASH_32_ALGORITHM = "XXHash32"; private final @NotNull String value; private final @Nullable String seedHex; @@ -90,7 +91,7 @@ public String value() @Override public String algorithm() { - return "XXHash32"; + return XXHASH_32_ALGORITHM; } /** diff --git a/client-common/src/main/java/org/apache/cassandra/sidecar/common/response/DigestResponse.java b/client-common/src/main/java/org/apache/cassandra/sidecar/common/response/DigestResponse.java new file mode 100644 index 000000000..97d69a36b --- /dev/null +++ b/client-common/src/main/java/org/apache/cassandra/sidecar/common/response/DigestResponse.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.sidecar.common.response; + +import java.util.Objects; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.cassandra.sidecar.common.request.data.Digest; +import org.apache.cassandra.sidecar.common.request.data.MD5Digest; +import org.apache.cassandra.sidecar.common.request.data.XXHash32Digest; + +/** + * Response object containing a cryptographic digest value for file verification purposes. + */ +public class DigestResponse +{ + @JsonProperty("digest") + public final String digest; + @JsonProperty("digestAlgorithm") + public final String digestAlgorithm; + + @JsonCreator + public DigestResponse(@JsonProperty("digest") String digest, + @JsonProperty("digestAlgorithm") String digestAlgorithm) + { + Objects.requireNonNull(digest, "digest is required"); + Objects.requireNonNull(digestAlgorithm, "digestAlgorithm is required"); + this.digest = digest; + this.digestAlgorithm = digestAlgorithm; + } + + @Override + public String toString() + { + return "DigestResponse{" + + "digest='" + digest + '\'' + + ", digestAlgorithm='" + digestAlgorithm + '\'' + + '}'; + } + + @JsonIgnore + public Digest toDigest() + { + if (digestAlgorithm.equalsIgnoreCase(MD5Digest.MD5_ALGORITHM)) + { + return new MD5Digest(digest); + } + else if (digestAlgorithm.equalsIgnoreCase(XXHash32Digest.XXHASH_32_ALGORITHM)) + { + return new XXHash32Digest(digest); + } + + throw new IllegalArgumentException("Digest algorithm " + digestAlgorithm + " is unknown"); + } +} diff --git a/client-common/src/main/java/org/apache/cassandra/sidecar/common/response/InstanceFileInfo.java b/client-common/src/main/java/org/apache/cassandra/sidecar/common/response/InstanceFileInfo.java index 5dc3d9c63..e3b802618 100644 --- a/client-common/src/main/java/org/apache/cassandra/sidecar/common/response/InstanceFileInfo.java +++ b/client-common/src/main/java/org/apache/cassandra/sidecar/common/response/InstanceFileInfo.java @@ -76,4 +76,15 @@ public int hashCode() { return Objects.hash(fileUrl, size, fileType, lastModifiedTime); } + + @Override + public String toString() + { + return "InstanceFileInfo{" + + "fileUrl='" + fileUrl + '\'' + + ", size=" + size + + ", fileType=" + fileType + + ", lastModifiedTime=" + lastModifiedTime + + '}'; + } } diff --git a/client-common/src/main/java/org/apache/cassandra/sidecar/common/response/LiveMigrationTaskResponse.java b/client-common/src/main/java/org/apache/cassandra/sidecar/common/response/LiveMigrationDataCopyResponse.java similarity index 91% rename from client-common/src/main/java/org/apache/cassandra/sidecar/common/response/LiveMigrationTaskResponse.java rename to client-common/src/main/java/org/apache/cassandra/sidecar/common/response/LiveMigrationDataCopyResponse.java index bfaf32d4a..d31ab148e 100644 --- a/client-common/src/main/java/org/apache/cassandra/sidecar/common/response/LiveMigrationTaskResponse.java +++ b/client-common/src/main/java/org/apache/cassandra/sidecar/common/response/LiveMigrationDataCopyResponse.java @@ -29,7 +29,7 @@ /** * Represents response of Live Migration task. */ -public class LiveMigrationTaskResponse +public class LiveMigrationDataCopyResponse { private final String taskId; @@ -40,23 +40,23 @@ public class LiveMigrationTaskResponse private final String source; private final int port; - public LiveMigrationTaskResponse(String taskId, - String source, - int port, - LiveMigrationDataCopyRequest request, - List statusList) + public LiveMigrationDataCopyResponse(String taskId, + String source, + int port, + LiveMigrationDataCopyRequest request, + List statusList) { this(taskId, source, port, request.maxIterations, request.successThreshold, request.maxConcurrency, statusList); } @JsonCreator - public LiveMigrationTaskResponse(@JsonProperty("taskId") String taskId, - @JsonProperty("source") String source, - @JsonProperty("port") int port, - @JsonProperty("maxIterations") int maxIterations, - @JsonProperty("successThreshold") double successThreshold, - @JsonProperty("maxConcurrency") int maxConcurrency, - @JsonProperty("status") List statusList) + public LiveMigrationDataCopyResponse(@JsonProperty("taskId") String taskId, + @JsonProperty("source") String source, + @JsonProperty("port") int port, + @JsonProperty("maxIterations") int maxIterations, + @JsonProperty("successThreshold") double successThreshold, + @JsonProperty("maxConcurrency") int maxConcurrency, + @JsonProperty("status") List statusList) { this.taskId = taskId; this.source = source; diff --git a/client-common/src/main/java/org/apache/cassandra/sidecar/common/response/LiveMigrationFilesVerificationResponse.java b/client-common/src/main/java/org/apache/cassandra/sidecar/common/response/LiveMigrationFilesVerificationResponse.java new file mode 100644 index 000000000..07d95e5ff --- /dev/null +++ b/client-common/src/main/java/org/apache/cassandra/sidecar/common/response/LiveMigrationFilesVerificationResponse.java @@ -0,0 +1,186 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.sidecar.common.response; + +import java.util.Objects; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * Response object for live migration file verification operations, containing statistics about + * files not found at source/target and digest mismatches during verification. + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class LiveMigrationFilesVerificationResponse +{ + private final String id; + private final String state; + private final String source; + private final int port; + private final int filesNotFoundAtSource; + private final int filesNotFoundAtDestination; + private final int metadataMatched; + private final int metadataMismatches; + private final int digestMismatches; + private final int digestVerificationFailures; + private final int filesMatched; + private final String digestAlgorithm; + + @JsonCreator + public LiveMigrationFilesVerificationResponse(@JsonProperty("id") String id, + @JsonProperty("digestAlgorithm") String digestAlgorithm, + @JsonProperty("state") String state, + @JsonProperty("source") String source, + @JsonProperty("port") int port, + @JsonProperty("filesNotFoundAtSource") int filesNotFoundAtSource, + @JsonProperty("filesNotFoundAtDestination") int filesNotFoundAtDestination, + @JsonProperty("metadataMatched") int metadataMatched, + @JsonProperty("metadataMismatches") int metadataMismatches, + @JsonProperty("digestMismatches") int digestMismatches, + @JsonProperty("digestVerificationFailures") int digestVerificationFailures, + @JsonProperty("filesMatched") int filesMatched) + { + Objects.requireNonNull(id, "id of files verification task must be specified"); + Objects.requireNonNull(state, "state of files verification task must be specified"); + this.id = id; + this.digestAlgorithm = digestAlgorithm; + this.state = state; + this.source = source; + this.port = port; + this.filesNotFoundAtSource = filesNotFoundAtSource; + this.filesNotFoundAtDestination = filesNotFoundAtDestination; + this.metadataMatched = metadataMatched; + this.metadataMismatches = metadataMismatches; + this.digestMismatches = digestMismatches; + this.digestVerificationFailures = digestVerificationFailures; + this.filesMatched = filesMatched; + } + + @JsonProperty("id") + public String id() + { + return id; + } + + @JsonProperty("state") + public String state() + { + return state; + } + + @JsonProperty("digestAlgorithm") + public String digestAlgorithm() + { + return digestAlgorithm; + } + + @JsonProperty("source") + public String source() + { + return source; + } + + @JsonProperty("port") + public int port() + { + return port; + } + + @JsonProperty("filesNotFoundAtSource") + public int filesNotFoundAtSource() + { + return filesNotFoundAtSource; + } + + @JsonProperty("filesNotFoundAtDestination") + public int filesNotFoundAtDestination() + { + return filesNotFoundAtDestination; + } + + @JsonProperty("metadataMatched") + public int metadataMatched() + { + return metadataMatched; + } + + @JsonProperty("metadataMismatches") + public int metadataMismatches() + { + return metadataMismatches; + } + + @JsonProperty("digestMismatches") + public int digestMismatches() + { + return digestMismatches; + } + + @JsonProperty("digestVerificationFailures") + public int digestVerificationFailures() + { + return digestVerificationFailures; + } + + @JsonProperty("filesMatched") + public int filesMatched() + { + return filesMatched; + } + + /** + * Determines whether the live migration file verification completed successfully. + * + * @return true if and only if the verification state is COMPLETED and there are no + * files missing at source or destination, no metadata mismatches, no digest + * mismatches, and no digest verification failures. + */ + @JsonIgnore + public boolean isVerificationSuccessful() + { + return "COMPLETED".equals(state) + && filesNotFoundAtSource == 0 + && filesNotFoundAtDestination == 0 + && metadataMismatches == 0 + && digestMismatches == 0 + && digestVerificationFailures == 0; + } + + @Override + public String toString() + { + return "LiveMigrationFilesVerificationResponse{" + + "id='" + id + '\'' + + ", digestAlgorithm='" + digestAlgorithm + '\'' + + ", state=" + state + + ", source='" + source + '\'' + + ", port=" + port + + ", filesNotFoundAtSource=" + filesNotFoundAtSource + + ", filesNotFoundAtDestination=" + filesNotFoundAtDestination + + ", metadataMatched=" + metadataMatched + + ", metadataMismatches=" + metadataMismatches + + ", digestMismatches=" + digestMismatches + + ", digestVerificationFailures=" + digestVerificationFailures + + ", filesMatched=" + filesMatched + + '}'; + } +} diff --git a/client-common/src/main/java/org/apache/cassandra/sidecar/common/response/LiveMigrationTaskCreationResponse.java b/client-common/src/main/java/org/apache/cassandra/sidecar/common/response/LiveMigrationTaskCreationResponse.java new file mode 100644 index 000000000..7afb668b3 --- /dev/null +++ b/client-common/src/main/java/org/apache/cassandra/sidecar/common/response/LiveMigrationTaskCreationResponse.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.sidecar.common.response; + +import java.util.Objects; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * Response object returned when a live migration task is created. + * Contains the task identifier and a URL to query the task status. + */ +public class LiveMigrationTaskCreationResponse +{ + private final String taskId; + private final String statusUrl; + + @JsonCreator + public LiveMigrationTaskCreationResponse(@JsonProperty("taskId") String taskId, + @JsonProperty("statusUrl") String statusUrl) + { + Objects.requireNonNull(taskId, "taskId cannot be null"); + Objects.requireNonNull(statusUrl, "statusUrl cannot be null"); + + this.taskId = taskId; + this.statusUrl = statusUrl; + } + + @JsonProperty("taskId") + public String taskId() + { + return taskId; + } + + @JsonProperty("statusUrl") + public String statusUrl() + { + return statusUrl; + } + + @Override + public String toString() + { + return "LiveMigrationTaskCreationResponse{" + + "taskId='" + taskId + '\'' + + ", statusUrl='" + statusUrl + '\'' + + '}'; + } +} diff --git a/client-common/src/test/java/org/apache/cassandra/sidecar/common/request/LiveMigrationFilesVerificationRequestTest.java b/client-common/src/test/java/org/apache/cassandra/sidecar/common/request/LiveMigrationFilesVerificationRequestTest.java new file mode 100644 index 000000000..975de93db --- /dev/null +++ b/client-common/src/test/java/org/apache/cassandra/sidecar/common/request/LiveMigrationFilesVerificationRequestTest.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.sidecar.common.request; + +import java.util.stream.Stream; + +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class LiveMigrationFilesVerificationRequestTest +{ + private final ObjectMapper objectMapper = new ObjectMapper(); + + static Stream concurrencyAndAlgorithm() + { + return Stream.of( + Arguments.of(1, "MD5"), + Arguments.of(8, "XXHash32"), + Arguments.of(10, "MD5"), + Arguments.of(Integer.MAX_VALUE, "XXHash32") + ); + } + + static Stream invalidDigestAlgorithms() + { + return Stream.of(null, "", " "); + } + + @ParameterizedTest + @MethodSource("concurrencyAndAlgorithm") + void testSerializationDeserializationRoundTrip(int maxConcurrency, String digestAlgorithm) throws Exception + { + LiveMigrationFilesVerificationRequest original = + new LiveMigrationFilesVerificationRequest(maxConcurrency, digestAlgorithm); + + String json = objectMapper.writeValueAsString(original); + LiveMigrationFilesVerificationRequest deserialized = + objectMapper.readValue(json, LiveMigrationFilesVerificationRequest.class); + + assertThat(deserialized.maxConcurrency()).isEqualTo(original.maxConcurrency()); + assertThat(deserialized.digestAlgorithm()).isEqualTo(original.digestAlgorithm()); + } + + @ParameterizedTest + @ValueSource(ints = { 0, -1, Integer.MIN_VALUE }) + void testInvalidMaxConcurrency(int maxConcurrency) + { + assertThatThrownBy(() -> new LiveMigrationFilesVerificationRequest(maxConcurrency, "MD5")) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("maxConcurrency must be >= 1"); + } + + @ParameterizedTest + @MethodSource("invalidDigestAlgorithms") + void testInvalidDigestAlgorithm(String digestAlgorithm) + { + assertThatThrownBy(() -> new LiveMigrationFilesVerificationRequest(10, digestAlgorithm)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("digestAlgorithm cannot be null or empty"); + } +} diff --git a/client-common/src/test/java/org/apache/cassandra/sidecar/common/response/LiveMigrationTaskResponseTest.java b/client-common/src/test/java/org/apache/cassandra/sidecar/common/response/LiveMigrationDataCopyResponseTest.java similarity index 90% rename from client-common/src/test/java/org/apache/cassandra/sidecar/common/response/LiveMigrationTaskResponseTest.java rename to client-common/src/test/java/org/apache/cassandra/sidecar/common/response/LiveMigrationDataCopyResponseTest.java index 71b934ed6..3b82bc951 100644 --- a/client-common/src/test/java/org/apache/cassandra/sidecar/common/response/LiveMigrationTaskResponseTest.java +++ b/client-common/src/test/java/org/apache/cassandra/sidecar/common/response/LiveMigrationDataCopyResponseTest.java @@ -24,14 +24,14 @@ import org.junit.jupiter.api.Test; import com.fasterxml.jackson.databind.ObjectMapper; -import org.apache.cassandra.sidecar.common.response.LiveMigrationTaskResponse.Status; +import org.apache.cassandra.sidecar.common.response.LiveMigrationDataCopyResponse.Status; import static org.assertj.core.api.Assertions.assertThat; /** - * Unit tests for {@link LiveMigrationTaskResponse} JSON serialization and deserialization. + * Unit tests for {@link LiveMigrationDataCopyResponse} JSON serialization and deserialization. */ -class LiveMigrationTaskResponseTest +class LiveMigrationDataCopyResponseTest { private final ObjectMapper objectMapper = new ObjectMapper(); @@ -44,7 +44,7 @@ void testSerializationDeserializationRoundTrip() throws Exception new Status(3, "SUCCESS", 1000000L, 100, 200000L, 20, 20, 0, 200000L) ); - LiveMigrationTaskResponse original = new LiveMigrationTaskResponse( + LiveMigrationDataCopyResponse original = new LiveMigrationDataCopyResponse( "task-123", "192.168.1.100", 9043, @@ -55,7 +55,7 @@ void testSerializationDeserializationRoundTrip() throws Exception ); String json = objectMapper.writeValueAsString(original); - LiveMigrationTaskResponse deserialized = objectMapper.readValue(json, LiveMigrationTaskResponse.class); + LiveMigrationDataCopyResponse deserialized = objectMapper.readValue(json, LiveMigrationDataCopyResponse.class); assertThat(deserialized.taskId()).isEqualTo(original.taskId()); assertThat(deserialized.source()).isEqualTo(original.source()); diff --git a/client-common/src/test/java/org/apache/cassandra/sidecar/common/response/LiveMigrationFilesVerificationResponseTest.java b/client-common/src/test/java/org/apache/cassandra/sidecar/common/response/LiveMigrationFilesVerificationResponseTest.java new file mode 100644 index 000000000..81269de94 --- /dev/null +++ b/client-common/src/test/java/org/apache/cassandra/sidecar/common/response/LiveMigrationFilesVerificationResponseTest.java @@ -0,0 +1,322 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.sidecar.common.response; + +import org.junit.jupiter.api.Test; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class LiveMigrationFilesVerificationResponseTest +{ + private final ObjectMapper mapper = new ObjectMapper(); + + @Test + void testSerializationRoundTrip() throws Exception + { + LiveMigrationFilesVerificationResponse original = new LiveMigrationFilesVerificationResponse( + "test-id-123", + "MD5", + "COMPLETED", + "192.168.1.100", + 9042, + 5, + 10, + 100, + 15, + 20, + 3, + 85 + ); + + // Serialize to JSON + String json = mapper.writeValueAsString(original); + + // Deserialize back to object + LiveMigrationFilesVerificationResponse deserialized = + mapper.readValue(json, LiveMigrationFilesVerificationResponse.class); + + // Verify all fields match + assertThat(deserialized.id()).isEqualTo(original.id()); + assertThat(deserialized.digestAlgorithm()).isEqualTo(original.digestAlgorithm()); + assertThat(deserialized.state()).isEqualTo(original.state()); + assertThat(deserialized.source()).isEqualTo(original.source()); + assertThat(deserialized.port()).isEqualTo(original.port()); + assertThat(deserialized.filesNotFoundAtSource()).isEqualTo(original.filesNotFoundAtSource()); + assertThat(deserialized.filesNotFoundAtDestination()).isEqualTo(original.filesNotFoundAtDestination()); + assertThat(deserialized.metadataMatched()).isEqualTo(original.metadataMatched()); + assertThat(deserialized.metadataMismatches()).isEqualTo(original.metadataMismatches()); + assertThat(deserialized.digestMismatches()).isEqualTo(original.digestMismatches()); + assertThat(deserialized.digestVerificationFailures()).isEqualTo(original.digestVerificationFailures()); + assertThat(deserialized.filesMatched()).isEqualTo(original.filesMatched()); + assertThat(deserialized.isVerificationSuccessful()).isEqualTo(original.isVerificationSuccessful()); + } + + @Test + void testSerializationRoundTripInProgressState() throws Exception + { + LiveMigrationFilesVerificationResponse original = new LiveMigrationFilesVerificationResponse( + "test-id-456", + "XXHash32", + "IN_PROGRESS", + "192.168.1.200", + 7000, + 0, + 0, + 50, + 0, + 0, + 0, + 50 + ); + + // Serialize to JSON + String json = mapper.writeValueAsString(original); + + // Deserialize back to object + LiveMigrationFilesVerificationResponse deserialized = + mapper.readValue(json, LiveMigrationFilesVerificationResponse.class); + + // Verify all fields match + assertThat(deserialized.id()).isEqualTo(original.id()); + assertThat(deserialized.digestAlgorithm()).isEqualTo(original.digestAlgorithm()); + assertThat(deserialized.state()).isEqualTo(original.state()); + assertThat(deserialized.source()).isEqualTo(original.source()); + assertThat(deserialized.port()).isEqualTo(original.port()); + assertThat(deserialized.filesNotFoundAtSource()).isEqualTo(original.filesNotFoundAtSource()); + assertThat(deserialized.filesNotFoundAtDestination()).isEqualTo(original.filesNotFoundAtDestination()); + assertThat(deserialized.metadataMatched()).isEqualTo(original.metadataMatched()); + assertThat(deserialized.metadataMismatches()).isEqualTo(original.metadataMismatches()); + assertThat(deserialized.digestMismatches()).isEqualTo(original.digestMismatches()); + assertThat(deserialized.digestVerificationFailures()).isEqualTo(original.digestVerificationFailures()); + assertThat(deserialized.filesMatched()).isEqualTo(original.filesMatched()); + assertThat(deserialized.isVerificationSuccessful()).isEqualTo(original.isVerificationSuccessful()); + } + + @Test + void testIsVerificationSuccessfulAllConditionsMet() + { + LiveMigrationFilesVerificationResponse response = new LiveMigrationFilesVerificationResponse( + "test-id", + "MD5", + "COMPLETED", + "192.168.1.1", + 9042, + 0, // filesNotFoundAtSource + 0, // filesNotFoundAtDestination + 100, // metadataMatched + 0, // metadataMismatches + 0, // digestMismatches + 0, // digestVerificationFailures + 100 // filesMatched + ); + + assertThat(response.isVerificationSuccessful()).isTrue(); + } + + @Test + void testIsVerificationSuccessfulStateNotCompleted() + { + LiveMigrationFilesVerificationResponse response = new LiveMigrationFilesVerificationResponse( + "test-id", + "MD5", + "IN_PROGRESS", + "192.168.1.1", + 9042, + 0, + 0, + 100, + 0, + 0, + 0, + 100 + ); + + assertThat(response.isVerificationSuccessful()).isFalse(); + } + + @Test + void testIsVerificationSuccessfulFilesNotFoundAtSource() + { + LiveMigrationFilesVerificationResponse response = new LiveMigrationFilesVerificationResponse( + "test-id", + "MD5", + "COMPLETED", + "192.168.1.1", + 9042, + 5, // filesNotFoundAtSource > 0 + 0, + 100, + 0, + 0, + 0, + 100 + ); + + assertThat(response.isVerificationSuccessful()).isFalse(); + } + + @Test + void testIsVerificationSuccessfulFilesNotFoundAtDestination() + { + LiveMigrationFilesVerificationResponse response = new LiveMigrationFilesVerificationResponse( + "test-id", + "MD5", + "COMPLETED", + "192.168.1.1", + 9042, + 0, + 3, // filesNotFoundAtDestination > 0 + 100, + 0, + 0, + 0, + 100 + ); + + assertThat(response.isVerificationSuccessful()).isFalse(); + } + + @Test + void testIsVerificationSuccessfulMetadataMismatches() + { + LiveMigrationFilesVerificationResponse response = new LiveMigrationFilesVerificationResponse( + "test-id", + "MD5", + "COMPLETED", + "192.168.1.1", + 9042, + 0, + 0, + 95, + 5, // metadataMismatches > 0 + 0, + 0, + 100 + ); + + assertThat(response.isVerificationSuccessful()).isFalse(); + } + + @Test + void testIsVerificationSuccessfulDigestMismatches() + { + LiveMigrationFilesVerificationResponse response = new LiveMigrationFilesVerificationResponse( + "test-id", + "MD5", + "COMPLETED", + "192.168.1.1", + 9042, + 0, + 0, + 100, + 0, + 10, // digestMismatches > 0 + 0, + 90 + ); + + assertThat(response.isVerificationSuccessful()).isFalse(); + } + + @Test + void testIsVerificationSuccessfulDigestVerificationFailures() + { + LiveMigrationFilesVerificationResponse response = new LiveMigrationFilesVerificationResponse( + "test-id", + "MD5", + "COMPLETED", + "192.168.1.1", + 9042, + 0, + 0, + 100, + 0, + 0, + 2, // digestVerificationFailures > 0 + 98 + ); + + assertThat(response.isVerificationSuccessful()).isFalse(); + } + + @Test + void testIsVerificationSuccessfulMultipleFailureConditions() + { + LiveMigrationFilesVerificationResponse response = new LiveMigrationFilesVerificationResponse( + "test-id", + "MD5", + "COMPLETED", + "192.168.1.1", + 9042, + 2, // filesNotFoundAtSource > 0 + 3, // filesNotFoundAtDestination > 0 + 90, + 5, // metadataMismatches > 0 + 8, // digestMismatches > 0 + 1, // digestVerificationFailures > 0 + 85 + ); + + assertThat(response.isVerificationSuccessful()).isFalse(); + } + + @Test + void testConstructorThrowsNullPointerExceptionForNullId() + { + assertThatThrownBy(() -> new LiveMigrationFilesVerificationResponse( + null, // null id + "MD5", + "COMPLETED", + "192.168.1.1", + 9042, + 0, + 0, + 100, + 0, + 0, + 0, + 100 + )).isInstanceOf(NullPointerException.class) + .hasMessageContaining("id of files verification task must be specified"); + } + + @Test + void testIsVerificationSuccessfulFailedState() + { + LiveMigrationFilesVerificationResponse response = new LiveMigrationFilesVerificationResponse( + "test-id", + "MD5", + "FAILED", // FAILED state + "192.168.1.1", + 9042, + 0, // all counters are perfect + 0, + 100, + 0, + 0, + 0, + 100 + ); + + assertThat(response.isVerificationSuccessful()).isFalse(); + } +} diff --git a/client-common/src/test/java/org/apache/cassandra/sidecar/common/response/LiveMigrationTaskCreationResponseTest.java b/client-common/src/test/java/org/apache/cassandra/sidecar/common/response/LiveMigrationTaskCreationResponseTest.java new file mode 100644 index 000000000..08321355b --- /dev/null +++ b/client-common/src/test/java/org/apache/cassandra/sidecar/common/response/LiveMigrationTaskCreationResponseTest.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.sidecar.common.response; + +import org.junit.jupiter.api.Test; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class LiveMigrationTaskCreationResponseTest +{ + private final ObjectMapper mapper = new ObjectMapper(); + + @Test + void testSerializationRoundTrip() throws Exception + { + LiveMigrationTaskCreationResponse original = new LiveMigrationTaskCreationResponse( + "task-123-abc", + "/api/v1/live-migration/status/task-123-abc"); + + // Serialize to JSON + String json = mapper.writeValueAsString(original); + + // Deserialize back to object + LiveMigrationTaskCreationResponse deserialized = + mapper.readValue(json, LiveMigrationTaskCreationResponse.class); + + assertThat(deserialized.taskId()).isEqualTo(original.taskId()); + assertThat(deserialized.statusUrl()).isEqualTo(original.statusUrl()); + } + + @Test + void testConstructorInvalidValues() + { + assertThatThrownBy(() -> new LiveMigrationTaskCreationResponse("task-123", null)) + .isInstanceOf(NullPointerException.class) + .hasMessageContaining("statusUrl cannot be null"); + + assertThatThrownBy(() -> new LiveMigrationTaskCreationResponse( + null, + "/api/v1/live-migration/status/task-123" + )) + .isInstanceOf(NullPointerException.class) + .hasMessageContaining("taskId cannot be null"); + } +} diff --git a/client/src/main/java/org/apache/cassandra/sidecar/client/SidecarClient.java b/client/src/main/java/org/apache/cassandra/sidecar/client/SidecarClient.java index 6d6068c7d..3b9bd48d0 100644 --- a/client/src/main/java/org/apache/cassandra/sidecar/client/SidecarClient.java +++ b/client/src/main/java/org/apache/cassandra/sidecar/client/SidecarClient.java @@ -44,6 +44,7 @@ import org.apache.cassandra.sidecar.common.request.DeleteServiceConfigRequest; import org.apache.cassandra.sidecar.common.request.ImportSSTableRequest; import org.apache.cassandra.sidecar.common.request.ListCdcSegmentsRequest; +import org.apache.cassandra.sidecar.common.request.LiveMigrationFileDigestRequest; import org.apache.cassandra.sidecar.common.request.LiveMigrationListInstanceFilesRequest; import org.apache.cassandra.sidecar.common.request.LiveMigrationStatusRequest; import org.apache.cassandra.sidecar.common.request.RepairRequest; @@ -68,6 +69,7 @@ import org.apache.cassandra.sidecar.common.response.CompactionStatsResponse; import org.apache.cassandra.sidecar.common.response.CompactionStopResponse; import org.apache.cassandra.sidecar.common.response.ConnectedClientStatsResponse; +import org.apache.cassandra.sidecar.common.response.DigestResponse; import org.apache.cassandra.sidecar.common.response.GossipInfoResponse; import org.apache.cassandra.sidecar.common.response.HealthResponse; import org.apache.cassandra.sidecar.common.response.InstanceFilesListResponse; @@ -1064,6 +1066,24 @@ public CompletableFuture liveMigrationStreamFileAsync(SidecarInstance inst .build()); } + + /** + * Retrieves the digest of a file calculated using the specified algorithm during live migration. + * + * @param instance the instance where the request will be executed + * @param fileUrl the file url for which digest should be calculated + * @param digestAlgorithm the digest algorithm to use (e.g., "md5", "xxhash32") + * @return a completable future of the digest response + */ + public CompletableFuture liveMigrationFileDigestAsync(SidecarInstance instance, + String fileUrl, + String digestAlgorithm) + { + return executor.executeRequestAsync(requestBuilder().singleInstanceSelectionPolicy(instance) + .request(LiveMigrationFileDigestRequest.create(fileUrl, digestAlgorithm)) + .build()); + } + /** * Requests for the live migration status using sidecar for given {@code instance}. * diff --git a/client/src/testFixtures/java/org/apache/cassandra/sidecar/client/SidecarClientTest.java b/client/src/testFixtures/java/org/apache/cassandra/sidecar/client/SidecarClientTest.java index 336f636f8..d4b584623 100644 --- a/client/src/testFixtures/java/org/apache/cassandra/sidecar/client/SidecarClientTest.java +++ b/client/src/testFixtures/java/org/apache/cassandra/sidecar/client/SidecarClientTest.java @@ -83,6 +83,7 @@ import org.apache.cassandra.sidecar.common.response.CompactionStatsResponse; import org.apache.cassandra.sidecar.common.response.CompactionStopResponse; import org.apache.cassandra.sidecar.common.response.ConnectedClientStatsResponse; +import org.apache.cassandra.sidecar.common.response.DigestResponse; import org.apache.cassandra.sidecar.common.response.GossipInfoResponse; import org.apache.cassandra.sidecar.common.response.HealthResponse; import org.apache.cassandra.sidecar.common.response.InstanceFileInfo; @@ -2152,6 +2153,29 @@ void testLiveMigrationStreamFileAsyncBadRequest(@TempDir Path tempDirectory) thr validateResponseServed(fileUrl); } + @Test + void testLiveMigrationFileDigestAsync() throws ExecutionException, InterruptedException + { + MockResponse response = new MockResponse(); + response.setResponseCode(200); + response.setHeader("content-type", "application/json"); + String digest = "0123456789abcdef"; + String digestAlgorithm = "md5"; + + response.setBody("{\"digest\":\"" + digest + "\",\"digestAlgorithm\":\"md5\"}"); + enqueue(response); + + SidecarInstance instance = instances.get(0); + String url = LIVE_MIGRATION_FILES_ROUTE + "/data/0/test_file.text"; + + DigestResponse digestResponse = client.liveMigrationFileDigestAsync(instance, url, digestAlgorithm).get(); + + assertThat(digestResponse).isNotNull(); + assertThat(digestResponse.digest).isEqualTo(digest); + + validateResponseServed(url + "?digestAlgorithm=md5"); + } + @Test void testNodeLifecycleInfo() throws Exception { diff --git a/conf/sidecar.yaml b/conf/sidecar.yaml index 62e521858..2a2e6d9c3 100644 --- a/conf/sidecar.yaml +++ b/conf/sidecar.yaml @@ -483,7 +483,7 @@ live_migration: - glob:${DATA_FILE_DIR}/*/*/snapshots # Excludes snapshot directories in data folder to copy to destination migration_map: # Map of source and destination Cassandra instances # localhost1: localhost4 # This entry says that localhost1 will be migrated to localhost4 - max_concurrent_downloads: 20 # Maximum number of concurrent downloads allowed + max_concurrent_file_requests: 20 # Maximum number of concurrent file requests allowed # Configuration to allow sidecar start and stop Cassandra instances via the lifecycle API (disabled by default) lifecycle: diff --git a/examples/lifecycle/conf/sidecar.yaml.template b/examples/lifecycle/conf/sidecar.yaml.template index fd312423f..24d9acd68 100644 --- a/examples/lifecycle/conf/sidecar.yaml.template +++ b/examples/lifecycle/conf/sidecar.yaml.template @@ -441,7 +441,7 @@ live_migration: - glob:${DATA_FILE_DIR}/*/*/snapshots # Excludes snapshot directories in data folder to copy to destination migration_map: # Map of source and destination Cassandra instances # localhost: localhost4 # This entry says that localhost will be migrated to localhost4 - max_concurrent_downloads: 20 # Maximum number of concurrent downloads allowed + max_concurrent_file_requests: 20 # Maximum number of concurrent file requests allowed # Configuration to allow sidecar start and stop Cassandra instances via the lifecycle API (disabled by default) lifecycle: diff --git a/integration-tests/src/integrationTest/resources/config/sidecar.yaml.template b/integration-tests/src/integrationTest/resources/config/sidecar.yaml.template index 48a334402..79613d401 100644 --- a/integration-tests/src/integrationTest/resources/config/sidecar.yaml.template +++ b/integration-tests/src/integrationTest/resources/config/sidecar.yaml.template @@ -401,7 +401,7 @@ live_migration: - glob:${DATA_FILE_DIR}/*/*/snapshots # Excludes snapshot directories in data folder to copy to destination migration_map: # Map of source and destination Cassandra instances # localhost1: localhost4 # This entry says that localhost1 will be migrated to localhost4 - max_concurrent_downloads: 20 # Maximum number of concurrent downloads allowed + max_concurrent_file_requests: 20 # Maximum number of concurrent file requests allowed # Configuration to allow sidecar start and stop Cassandra instances via the lifecycle API (disabled by default) lifecycle: diff --git a/server/src/main/java/org/apache/cassandra/sidecar/config/LiveMigrationConfiguration.java b/server/src/main/java/org/apache/cassandra/sidecar/config/LiveMigrationConfiguration.java index 392b8a20a..1b99e133a 100644 --- a/server/src/main/java/org/apache/cassandra/sidecar/config/LiveMigrationConfiguration.java +++ b/server/src/main/java/org/apache/cassandra/sidecar/config/LiveMigrationConfiguration.java @@ -49,7 +49,9 @@ public interface LiveMigrationConfiguration Map migrationMap(); /** - * Maximum number of concurrent downloads allowed. + * Maximum number of concurrent file requests allowed. + * This limits concurrent file downloads and digest calculation requests to protect the source node + * from rogue clients. */ - int maxConcurrentDownloads(); + int maxConcurrentFileRequests(); } diff --git a/server/src/main/java/org/apache/cassandra/sidecar/config/yaml/LiveMigrationConfigurationImpl.java b/server/src/main/java/org/apache/cassandra/sidecar/config/yaml/LiveMigrationConfigurationImpl.java index a1e8c7104..25c777cf2 100644 --- a/server/src/main/java/org/apache/cassandra/sidecar/config/yaml/LiveMigrationConfigurationImpl.java +++ b/server/src/main/java/org/apache/cassandra/sidecar/config/yaml/LiveMigrationConfigurationImpl.java @@ -32,34 +32,34 @@ public class LiveMigrationConfigurationImpl implements LiveMigrationConfiguration { - public static final int DEFAULT_MAX_CONCURRENT_DOWNLOADS = 20; + public static final int DEFAULT_MAX_CONCURRENT_FILE_REQUESTS = 20; private final Set filesToExclude; private final Set directoriesToExclude; private final Map migrationMap; - private final int maxConcurrentDownloads; + private final int maxConcurrentFileRequests; public LiveMigrationConfigurationImpl() { - this(Collections.emptySet(), Collections.emptySet(), Collections.emptyMap(), DEFAULT_MAX_CONCURRENT_DOWNLOADS); + this(Collections.emptySet(), Collections.emptySet(), Collections.emptyMap(), DEFAULT_MAX_CONCURRENT_FILE_REQUESTS); } @JsonCreator public LiveMigrationConfigurationImpl(@JsonProperty("files_to_exclude") Set filesToExclude, @JsonProperty("dirs_to_exclude") Set directoriesToExclude, @JsonProperty("migration_map") Map migrationMap, - @JsonProperty("max_concurrent_downloads") int maxConcurrentDownloads) + @JsonProperty("max_concurrent_file_requests") int maxConcurrentFileRequests) { this.filesToExclude = filesToExclude; this.directoriesToExclude = directoriesToExclude; this.migrationMap = migrationMap == null ? Collections.emptyMap() : Collections.unmodifiableMap(migrationMap); - if (maxConcurrentDownloads < 1) + if (maxConcurrentFileRequests < 1) { - throw new IllegalArgumentException("Invalid max_concurrent_downloads " + maxConcurrentDownloads + + throw new IllegalArgumentException("Invalid max_concurrent_file_requests " + maxConcurrentFileRequests + ". It must be >= 1"); } - this.maxConcurrentDownloads = maxConcurrentDownloads; + this.maxConcurrentFileRequests = maxConcurrentFileRequests; } @Override @@ -84,9 +84,9 @@ public Map migrationMap() } @Override - @JsonProperty("max_concurrent_downloads") - public int maxConcurrentDownloads() + @JsonProperty("max_concurrent_file_requests") + public int maxConcurrentFileRequests() { - return maxConcurrentDownloads; + return maxConcurrentFileRequests; } } diff --git a/server/src/main/java/org/apache/cassandra/sidecar/exceptions/LiveMigrationExceptions.java b/server/src/main/java/org/apache/cassandra/sidecar/exceptions/LiveMigrationExceptions.java index eb0b1304a..8def795f9 100644 --- a/server/src/main/java/org/apache/cassandra/sidecar/exceptions/LiveMigrationExceptions.java +++ b/server/src/main/java/org/apache/cassandra/sidecar/exceptions/LiveMigrationExceptions.java @@ -27,12 +27,13 @@ public class LiveMigrationExceptions { /** - * There can be only one Live Migration task running at any time. This exception is thrown when client is trying - * to create a new Live Migration data copy task while another task is in progress. + * Thrown when attempting to create a new live migration task while another task is already in progress + * for the same instance. Only one live migration task (e.g., data copy, file digest verification) can be + * active per instance at a time to prevent resource conflicts and ensure data integrity. */ - public static class LiveMigrationDataCopyInProgressException extends IllegalArgumentException + public static class LiveMigrationTaskInProgressException extends IllegalStateException { - public LiveMigrationDataCopyInProgressException(String message) + public LiveMigrationTaskInProgressException(String message) { super(message); } @@ -53,6 +54,11 @@ public LiveMigrationInvalidRequestException(String message) { super(message); } + + public LiveMigrationInvalidRequestException(String message, Throwable cause) + { + super(message, cause); + } } /** @@ -65,4 +71,44 @@ public LiveMigrationTaskNotFoundException(String message) super(message); } } + + /** + * Exception thrown when file verification fails during live migration. + */ + public static class FileVerificationFailureException extends Exception + { + public FileVerificationFailureException(String message) + { + super(message); + } + } + + /** + * Exception thrown when file digest verification fails during live migration. + * This indicates that the digest of a file at the destination does not match + * the digest of the same file at the source, suggesting data corruption or + * incomplete file transfer. + */ + public static class DigestMismatchException extends Exception + { + private final String path; + private final String fileUrl; + + public DigestMismatchException(String path, String fileUrl, Throwable cause) + { + super("Digest mismatch for file: " + fileUrl + " (local path: " + path + ")", cause); + this.path = path; + this.fileUrl = fileUrl; + } + + public String path() + { + return path; + } + + public String fileUrl() + { + return fileUrl; + } + } } diff --git a/server/src/main/java/org/apache/cassandra/sidecar/handlers/livemigration/LiveMigrationCancelDataCopyTaskHandler.java b/server/src/main/java/org/apache/cassandra/sidecar/handlers/livemigration/LiveMigrationCancelDataCopyTaskHandler.java index 9e3500ce5..9766a25ec 100644 --- a/server/src/main/java/org/apache/cassandra/sidecar/handlers/livemigration/LiveMigrationCancelDataCopyTaskHandler.java +++ b/server/src/main/java/org/apache/cassandra/sidecar/handlers/livemigration/LiveMigrationCancelDataCopyTaskHandler.java @@ -30,6 +30,7 @@ import io.vertx.ext.auth.authorization.Authorization; import io.vertx.ext.web.RoutingContext; import org.apache.cassandra.sidecar.acl.authorization.BasicPermissions; +import org.apache.cassandra.sidecar.common.response.LiveMigrationDataCopyResponse; import org.apache.cassandra.sidecar.concurrent.ExecutorPools; import org.apache.cassandra.sidecar.exceptions.LiveMigrationExceptions.LiveMigrationTaskNotFoundException; import org.apache.cassandra.sidecar.handlers.AbstractHandler; @@ -75,7 +76,7 @@ protected void handleInternal(RoutingContext context, { try { - LiveMigrationTask task = dataCopyTaskManager.cancelTask(taskId, host); + LiveMigrationTask task = dataCopyTaskManager.cancelTask(taskId, host); LOGGER.info("Successfully cancelled the data copy task with TaskID={}", taskId); context.json(task.getResponse()); } diff --git a/server/src/main/java/org/apache/cassandra/sidecar/handlers/livemigration/LiveMigrationCancelFilesVerificationTaskHandler.java b/server/src/main/java/org/apache/cassandra/sidecar/handlers/livemigration/LiveMigrationCancelFilesVerificationTaskHandler.java new file mode 100644 index 000000000..9a141d5a9 --- /dev/null +++ b/server/src/main/java/org/apache/cassandra/sidecar/handlers/livemigration/LiveMigrationCancelFilesVerificationTaskHandler.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.sidecar.handlers.livemigration; + +import java.util.Set; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.inject.Inject; +import io.netty.handler.codec.http.HttpResponseStatus; +import io.vertx.core.http.HttpServerRequest; +import io.vertx.core.net.SocketAddress; +import io.vertx.ext.auth.authorization.Authorization; +import io.vertx.ext.web.RoutingContext; +import org.apache.cassandra.sidecar.common.response.LiveMigrationFilesVerificationResponse; +import org.apache.cassandra.sidecar.concurrent.ExecutorPools; +import org.apache.cassandra.sidecar.exceptions.LiveMigrationExceptions.LiveMigrationTaskNotFoundException; +import org.apache.cassandra.sidecar.handlers.AbstractHandler; +import org.apache.cassandra.sidecar.handlers.AccessProtected; +import org.apache.cassandra.sidecar.livemigration.FilesVerificationTaskManager; +import org.apache.cassandra.sidecar.livemigration.LiveMigrationTask; +import org.apache.cassandra.sidecar.utils.CassandraInputValidator; +import org.apache.cassandra.sidecar.utils.InstanceMetadataFetcher; +import org.jetbrains.annotations.NotNull; + +import static org.apache.cassandra.sidecar.acl.authorization.BasicPermissions.DATA_COPY; +import static org.apache.cassandra.sidecar.utils.HttpExceptions.wrapHttpException; + +/** + * Handler for canceling an active live migration files verification task. + * Accepts a task ID and cancels the corresponding verification task on the specified host. + */ +public class LiveMigrationCancelFilesVerificationTaskHandler extends AbstractHandler implements AccessProtected +{ + private static final Logger LOGGER = LoggerFactory.getLogger(LiveMigrationCancelFilesVerificationTaskHandler.class); + + private final FilesVerificationTaskManager taskManager; + + /** + * Constructs a handler with the provided {@code metadataFetcher} + * + * @param metadataFetcher the interface to retrieve instance metadata + * @param executorPools the executor pools for blocking executions + * @param validator a validator instance to validate Cassandra-specific input + */ + @Inject + public LiveMigrationCancelFilesVerificationTaskHandler(InstanceMetadataFetcher metadataFetcher, + ExecutorPools executorPools, + CassandraInputValidator validator, + FilesVerificationTaskManager taskManager) + { + super(metadataFetcher, executorPools, validator); + this.taskManager = taskManager; + } + + @Override + protected String extractParamsOrThrow(RoutingContext context) + { + return context.pathParam("taskId"); + } + + @Override + protected void handleInternal(RoutingContext context, + HttpServerRequest httpRequest, + @NotNull String host, + SocketAddress remoteAddress, + String taskId) + { + try + { + LiveMigrationTask task = taskManager.cancelTask(taskId, host); + LOGGER.info("Successfully cancelled the files verification task with taskId={} host={}", taskId, host); + context.json(task.getResponse()); + } + catch (LiveMigrationTaskNotFoundException e) + { + LOGGER.warn("Live migration files verification task not found for cancellation. " + + "taskId={} host={}", taskId, host); + context.fail(wrapHttpException(HttpResponseStatus.NOT_FOUND, + "No live migration task found with id " + taskId)); + } + catch (Exception e) + { + LOGGER.error("Failed to cancel files verification task. taskId={} host={}", taskId, host, e); + context.fail(wrapHttpException(HttpResponseStatus.INTERNAL_SERVER_ERROR, + "Failed to cancel task: " + e.getMessage())); + } + } + + @Override + public Set requiredAuthorizations() + { + return Set.of(DATA_COPY.toAuthorization()); + } +} diff --git a/server/src/main/java/org/apache/cassandra/sidecar/handlers/livemigration/LiveMigrationConcurrencyLimitHandler.java b/server/src/main/java/org/apache/cassandra/sidecar/handlers/livemigration/LiveMigrationConcurrencyLimitHandler.java new file mode 100644 index 000000000..20aed9563 --- /dev/null +++ b/server/src/main/java/org/apache/cassandra/sidecar/handlers/livemigration/LiveMigrationConcurrencyLimitHandler.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.sidecar.handlers.livemigration; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.inject.Inject; +import com.google.inject.Singleton; +import io.netty.handler.codec.http.HttpResponseStatus; +import io.vertx.core.Handler; +import io.vertx.ext.web.RoutingContext; +import org.apache.cassandra.sidecar.concurrent.ConcurrencyLimiter; +import org.apache.cassandra.sidecar.config.SidecarConfiguration; + +import static org.apache.cassandra.sidecar.utils.HttpExceptions.wrapHttpException; + +/** + * Handler that enforces concurrency limits for live migration file operations. + * Returns HTTP 503 (SERVICE_UNAVAILABLE) when the maximum concurrent file requests limit is exceeded. + */ +@Singleton +public class LiveMigrationConcurrencyLimitHandler implements Handler +{ + private static final Logger LOGGER = LoggerFactory.getLogger(LiveMigrationConcurrencyLimitHandler.class); + + private final ConcurrencyLimiter concurrencyLimiter; + + @Inject + public LiveMigrationConcurrencyLimitHandler(SidecarConfiguration sidecarConfiguration) + { + this.concurrencyLimiter = + new ConcurrencyLimiter(() -> sidecarConfiguration.liveMigrationConfiguration().maxConcurrentFileRequests()); + } + + @Override + public void handle(RoutingContext rc) + { + if (!concurrencyLimiter.tryAcquire()) + { + LOGGER.warn("Too many concurrent live migration file requests. Path={}", rc.request().path()); + rc.fail(wrapHttpException(HttpResponseStatus.SERVICE_UNAVAILABLE, + "Server is busy processing live migration file requests, " + + "please try again later")); + return; + } + rc.addEndHandler(v -> concurrencyLimiter.releasePermit()); + rc.next(); + } +} diff --git a/server/src/main/java/org/apache/cassandra/sidecar/handlers/livemigration/LiveMigrationCreateDataCopyTaskHandler.java b/server/src/main/java/org/apache/cassandra/sidecar/handlers/livemigration/LiveMigrationCreateDataCopyTaskHandler.java index 14fe60b6e..8abec8f25 100644 --- a/server/src/main/java/org/apache/cassandra/sidecar/handlers/livemigration/LiveMigrationCreateDataCopyTaskHandler.java +++ b/server/src/main/java/org/apache/cassandra/sidecar/handlers/livemigration/LiveMigrationCreateDataCopyTaskHandler.java @@ -35,9 +35,10 @@ import org.apache.cassandra.sidecar.acl.authorization.BasicPermissions; import org.apache.cassandra.sidecar.common.ApiEndpointsV1; import org.apache.cassandra.sidecar.common.request.LiveMigrationDataCopyRequest; +import org.apache.cassandra.sidecar.common.response.LiveMigrationDataCopyResponse; import org.apache.cassandra.sidecar.concurrent.ExecutorPools; -import org.apache.cassandra.sidecar.exceptions.LiveMigrationExceptions.LiveMigrationDataCopyInProgressException; import org.apache.cassandra.sidecar.exceptions.LiveMigrationExceptions.LiveMigrationInvalidRequestException; +import org.apache.cassandra.sidecar.exceptions.LiveMigrationExceptions.LiveMigrationTaskInProgressException; import org.apache.cassandra.sidecar.handlers.AbstractHandler; import org.apache.cassandra.sidecar.handlers.AccessProtected; import org.apache.cassandra.sidecar.livemigration.DataCopyTaskManager; @@ -106,10 +107,10 @@ protected void handleInternal(RoutingContext context, LOGGER.error("Invalid live migration request.", throwable); context.fail(wrapHttpException(HttpResponseStatus.BAD_REQUEST, throwable.getMessage(), throwable)); } - else if (throwable instanceof LiveMigrationDataCopyInProgressException) + else if (throwable instanceof LiveMigrationTaskInProgressException) { LOGGER.error("Cannot start a new data copy task while another one is in progress."); - context.fail(wrapHttpException(HttpResponseStatus.FORBIDDEN, throwable.getMessage(), throwable)); + context.fail(wrapHttpException(HttpResponseStatus.CONFLICT, throwable.getMessage(), throwable)); } else { @@ -120,7 +121,7 @@ else if (throwable instanceof LiveMigrationDataCopyInProgressException) }); } - private JsonObject buildResponse(LiveMigrationTask task) + private JsonObject buildResponse(LiveMigrationTask task) { return new JsonObject() .put("taskId", task.id()) diff --git a/server/src/main/java/org/apache/cassandra/sidecar/handlers/livemigration/LiveMigrationCreateFilesVerificationTaskHandler.java b/server/src/main/java/org/apache/cassandra/sidecar/handlers/livemigration/LiveMigrationCreateFilesVerificationTaskHandler.java new file mode 100644 index 000000000..e69d2a97a --- /dev/null +++ b/server/src/main/java/org/apache/cassandra/sidecar/handlers/livemigration/LiveMigrationCreateFilesVerificationTaskHandler.java @@ -0,0 +1,161 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.sidecar.handlers.livemigration; + +import java.util.Set; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.inject.Inject; +import io.netty.handler.codec.http.HttpResponseStatus; +import io.vertx.core.http.HttpServerRequest; +import io.vertx.core.json.DecodeException; +import io.vertx.core.json.Json; +import io.vertx.core.net.SocketAddress; +import io.vertx.ext.auth.authorization.Authorization; +import io.vertx.ext.web.RoutingContext; +import org.apache.cassandra.sidecar.cluster.instance.InstanceMetadata; +import org.apache.cassandra.sidecar.common.request.LiveMigrationFilesVerificationRequest; +import org.apache.cassandra.sidecar.common.response.LiveMigrationTaskCreationResponse; +import org.apache.cassandra.sidecar.concurrent.ExecutorPools; +import org.apache.cassandra.sidecar.exceptions.LiveMigrationExceptions.LiveMigrationInvalidRequestException; +import org.apache.cassandra.sidecar.exceptions.LiveMigrationExceptions.LiveMigrationTaskInProgressException; +import org.apache.cassandra.sidecar.exceptions.NoSuchCassandraInstanceException; +import org.apache.cassandra.sidecar.handlers.AbstractHandler; +import org.apache.cassandra.sidecar.handlers.AccessProtected; +import org.apache.cassandra.sidecar.livemigration.FilesVerificationTaskManager; +import org.apache.cassandra.sidecar.utils.CassandraInputValidator; +import org.apache.cassandra.sidecar.utils.InstanceMetadataFetcher; +import org.jetbrains.annotations.NotNull; + +import static io.netty.handler.codec.http.HttpResponseStatus.ACCEPTED; +import static io.netty.handler.codec.http.HttpResponseStatus.BAD_REQUEST; +import static io.netty.handler.codec.http.HttpResponseStatus.CONFLICT; +import static io.netty.handler.codec.http.HttpResponseStatus.SERVICE_UNAVAILABLE; +import static org.apache.cassandra.sidecar.acl.authorization.BasicPermissions.DATA_COPY; +import static org.apache.cassandra.sidecar.common.ApiEndpointsV1.LIVE_MIGRATION_FILES_VERIFICATION_TASKS_ROUTE; +import static org.apache.cassandra.sidecar.utils.HttpExceptions.wrapHttpException; + +/** + * HTTP handler for creating file digest verification tasks during live migration. + * Manages concurrent verification tasks per instance and orchestrates the verification process. + */ +public class LiveMigrationCreateFilesVerificationTaskHandler extends AbstractHandler implements AccessProtected +{ + private static final Logger LOGGER = LoggerFactory.getLogger(LiveMigrationCreateFilesVerificationTaskHandler.class); + + private final FilesVerificationTaskManager filesVerificationTaskManager; + private final LiveMigrationMap liveMigrationMap; + + /** + * Constructs a handler with the provided {@code metadataFetcher} + * + * @param metadataFetcher the interface to retrieve instance metadata + * @param executorPools the executor pools for blocking executions + * @param validator a validator instance to validate Cassandra-specific input + */ + @Inject + protected LiveMigrationCreateFilesVerificationTaskHandler(InstanceMetadataFetcher metadataFetcher, + ExecutorPools executorPools, + CassandraInputValidator validator, + LiveMigrationMap liveMigrationMap, + FilesVerificationTaskManager filesVerificationTaskManager) + { + super(metadataFetcher, executorPools, validator); + this.liveMigrationMap = liveMigrationMap; + this.filesVerificationTaskManager = filesVerificationTaskManager; + } + + @Override + protected LiveMigrationFilesVerificationRequest extractParamsOrThrow(RoutingContext context) + { + try + { + return Json.decodeValue(context.body().buffer(), LiveMigrationFilesVerificationRequest.class); + } + catch (DecodeException decodeException) + { + throw wrapHttpException(HttpResponseStatus.BAD_REQUEST, + "Failed to parse request body, please ensure that the request is valid.", + decodeException); + } + catch (IllegalArgumentException e) + { + throw wrapHttpException(HttpResponseStatus.BAD_REQUEST, e.getMessage(), e); + } + } + + @Override + protected void handleInternal(RoutingContext context, + HttpServerRequest httpRequest, + @NotNull String host, + SocketAddress remoteAddress, + LiveMigrationFilesVerificationRequest request) + { + LOGGER.debug("Received files verification request for host {} with maxConcurrency {}", + host, request.maxConcurrency()); + InstanceMetadata localInstanceMetadata; + try + { + localInstanceMetadata = metadataFetcher.instance(host); + } + catch (NoSuchCassandraInstanceException e) + { + LOGGER.error("Failed to fetch instance metadata for host={}", host); + context.fail(wrapHttpException(SERVICE_UNAVAILABLE, e)); + return; + } + + liveMigrationMap.getSource(host) + .compose(source -> filesVerificationTaskManager.createTask(request, source, localInstanceMetadata)) + + .onSuccess(task -> { + LOGGER.info("Created files verification task {} for host {}", task.id(), host); + context.response().setStatusCode(ACCEPTED.code()); + String statusUrl = LIVE_MIGRATION_FILES_VERIFICATION_TASKS_ROUTE + "/" + task.id(); + context.json(new LiveMigrationTaskCreationResponse(task.id(), statusUrl)); + }) + + .onFailure(throwable -> { + if (throwable instanceof LiveMigrationTaskInProgressException) + { + LOGGER.error("Cannot start a new files verification task for host {} " + + "while another live migration task is in progress.", host); + context.fail(wrapHttpException(CONFLICT, throwable.getMessage(), throwable)); + return; + } + else if (throwable instanceof LiveMigrationInvalidRequestException) + { + LOGGER.error("Invalid request {}", request, throwable); + context.fail(wrapHttpException(BAD_REQUEST, throwable.getMessage(), throwable)); + return; + } + LOGGER.error("Failed to create files verification task for host {}.", host, throwable); + context.fail(wrapHttpException(HttpResponseStatus.INTERNAL_SERVER_ERROR, + throwable.getMessage(), throwable)); + }); + } + + @Override + public Set requiredAuthorizations() + { + return Set.of(DATA_COPY.toAuthorization()); + } +} diff --git a/server/src/main/java/org/apache/cassandra/sidecar/handlers/livemigration/LiveMigrationDigestHandlerWrapper.java b/server/src/main/java/org/apache/cassandra/sidecar/handlers/livemigration/LiveMigrationDigestHandlerWrapper.java new file mode 100644 index 000000000..272826430 --- /dev/null +++ b/server/src/main/java/org/apache/cassandra/sidecar/handlers/livemigration/LiveMigrationDigestHandlerWrapper.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.sidecar.handlers.livemigration; + +import com.google.inject.Inject; +import io.vertx.core.Handler; +import io.vertx.ext.web.RoutingContext; +import org.apache.cassandra.sidecar.common.request.LiveMigrationFileDigestRequest; + +import static org.apache.cassandra.sidecar.common.request.LiveMigrationFileDigestRequest.DIGEST_ALGORITHM_PARAM; + +/** + * Wrapper handler that conditionally delegates to {@link LiveMigrationFileDigestHandler} based on the presence + * of the {@link LiveMigrationFileDigestRequest#DIGEST_ALGORITHM_PARAM} query parameter, otherwise + * passes control to the next handler in the chain. + */ +public class LiveMigrationDigestHandlerWrapper implements Handler +{ + private final LiveMigrationFileDigestHandler liveMigrationFileDigestHandler; + + @Inject + public LiveMigrationDigestHandlerWrapper(LiveMigrationFileDigestHandler liveMigrationFileDigestHandler) + { + this.liveMigrationFileDigestHandler = liveMigrationFileDigestHandler; + } + + @Override + public void handle(RoutingContext context) + { + if (context.request().params().contains(DIGEST_ALGORITHM_PARAM)) + { + liveMigrationFileDigestHandler.handle(context); + } + else + { + context.next(); + } + } +} diff --git a/server/src/main/java/org/apache/cassandra/sidecar/handlers/livemigration/LiveMigrationFileDigestHandler.java b/server/src/main/java/org/apache/cassandra/sidecar/handlers/livemigration/LiveMigrationFileDigestHandler.java new file mode 100644 index 000000000..b97512c2a --- /dev/null +++ b/server/src/main/java/org/apache/cassandra/sidecar/handlers/livemigration/LiveMigrationFileDigestHandler.java @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.sidecar.handlers.livemigration; + +import java.util.Set; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.inject.Inject; +import io.netty.handler.codec.http.HttpResponseStatus; +import io.vertx.core.Vertx; +import io.vertx.core.http.HttpServerRequest; +import io.vertx.core.net.SocketAddress; +import io.vertx.ext.auth.authorization.Authorization; +import io.vertx.ext.web.RoutingContext; +import org.apache.cassandra.sidecar.acl.authorization.BasicPermissions; +import org.apache.cassandra.sidecar.common.request.LiveMigrationFileDigestRequest; +import org.apache.cassandra.sidecar.common.response.DigestResponse; +import org.apache.cassandra.sidecar.concurrent.ExecutorPools; +import org.apache.cassandra.sidecar.handlers.AbstractHandler; +import org.apache.cassandra.sidecar.handlers.AccessProtected; +import org.apache.cassandra.sidecar.handlers.FileStreamHandler; +import org.apache.cassandra.sidecar.utils.CassandraInputValidator; +import org.apache.cassandra.sidecar.utils.DigestAlgorithm; +import org.apache.cassandra.sidecar.utils.DigestAlgorithmFactory; +import org.apache.cassandra.sidecar.utils.InstanceMetadataFetcher; +import org.jetbrains.annotations.NotNull; + +import static io.netty.handler.codec.http.HttpResponseStatus.INTERNAL_SERVER_ERROR; +import static org.apache.cassandra.sidecar.common.request.LiveMigrationFileDigestRequest.DIGEST_ALGORITHM_PARAM; +import static org.apache.cassandra.sidecar.utils.AsyncFileDigestCalculator.calculateDigest; +import static org.apache.cassandra.sidecar.utils.HttpExceptions.wrapHttpException; + +/** + * Handler for calculating and returning file digests during live migration. + * Supported digest algorithms are determined by {@link DigestAlgorithmFactory} and specified + * via the {@link LiveMigrationFileDigestRequest#DIGEST_ALGORITHM_PARAM} query parameter. + */ +public class LiveMigrationFileDigestHandler extends AbstractHandler implements AccessProtected +{ + private static final Logger LOGGER = LoggerFactory.getLogger(LiveMigrationFileDigestHandler.class); + + private final Vertx vertx; + private final DigestAlgorithmFactory digestAlgorithmFactory; + + @Inject + public LiveMigrationFileDigestHandler(InstanceMetadataFetcher metadataFetcher, + ExecutorPools executorPools, + CassandraInputValidator validator, + Vertx vertx, + DigestAlgorithmFactory digestAlgorithmFactory) + { + super(metadataFetcher, executorPools, validator); + this.vertx = vertx; + this.digestAlgorithmFactory = digestAlgorithmFactory; + } + + @Override + protected DigestAlgorithm extractParamsOrThrow(RoutingContext context) + { + String digestAlgorithmParam = getDigestAlgorithmParam(context); + try + { + return digestAlgorithmFactory.getDigestAlgorithm(digestAlgorithmParam, 0); + } + catch (IllegalArgumentException e) + { + LOGGER.error("Unexpected error while getting digest algorithm for {}", digestAlgorithmParam, e); + throw wrapHttpException(HttpResponseStatus.BAD_REQUEST, e.getMessage()); + } + } + + @Override + protected void handleInternal(RoutingContext context, + HttpServerRequest httpRequest, + @NotNull String host, + SocketAddress remoteAddress, + DigestAlgorithm digestAlgorithm) + { + String file = context.get(FileStreamHandler.FILE_PATH_CONTEXT_KEY); + + if (file == null) + { + LOGGER.error("File path not found in context"); + context.fail(wrapHttpException(INTERNAL_SERVER_ERROR, "File path not available")); + return; + } + calculateDigest(vertx, file, digestAlgorithm) + .onComplete(ar -> { + if (ar.succeeded()) + { + String digestAlgorithmParam = getDigestAlgorithmParam(context); + DigestResponse digestResponse = new DigestResponse(ar.result(), digestAlgorithmParam); + context.json(digestResponse); + } + else + { + LOGGER.error("Failed to calculate digest", ar.cause()); + context.fail(wrapHttpException(INTERNAL_SERVER_ERROR, ar.cause())); + } + }); + } + + private String getDigestAlgorithmParam(RoutingContext context) + { + return context.request().getParam(DIGEST_ALGORITHM_PARAM); + } + + @Override + public Set requiredAuthorizations() + { + return Set.of(BasicPermissions.DATA_COPY.toAuthorization()); + } +} diff --git a/server/src/main/java/org/apache/cassandra/sidecar/handlers/livemigration/LiveMigrationFileStreamHandler.java b/server/src/main/java/org/apache/cassandra/sidecar/handlers/livemigration/LiveMigrationFileResolveHandler.java similarity index 91% rename from server/src/main/java/org/apache/cassandra/sidecar/handlers/livemigration/LiveMigrationFileStreamHandler.java rename to server/src/main/java/org/apache/cassandra/sidecar/handlers/livemigration/LiveMigrationFileResolveHandler.java index 4ba14912b..cae7b918b 100644 --- a/server/src/main/java/org/apache/cassandra/sidecar/handlers/livemigration/LiveMigrationFileStreamHandler.java +++ b/server/src/main/java/org/apache/cassandra/sidecar/handlers/livemigration/LiveMigrationFileResolveHandler.java @@ -61,25 +61,26 @@ import static org.apache.cassandra.sidecar.livemigration.LiveMigrationPlaceholderUtil.replacePlaceholder; /** - * Handler that allows Cassandra instance files to be downloaded during LiveMigration. This handler - * doesn't stream the file but relies on {@link FileStreamHandler} to do so. This handler doesn't allow - * using "/.." in the path to access files. This handler does not serve files which are excluded in - * Live Migration configuration. + * Handler that resolves and validates file paths for live migration operations. + * This handler validates that the requested file exists, is accessible, and is not excluded + * from live migration. It sets the resolved file path in the routing context for downstream handlers. + * This handler does not allow using "/.." in the path to access files and does not serve files + * which are excluded in the Live Migration configuration. */ -public class LiveMigrationFileStreamHandler extends AbstractHandler implements AccessProtected +public class LiveMigrationFileResolveHandler extends AbstractHandler implements AccessProtected { - private static final Logger LOGGER = LoggerFactory.getLogger(LiveMigrationFileStreamHandler.class); + private static final Logger LOGGER = LoggerFactory.getLogger(LiveMigrationFileResolveHandler.class); private final Map> fileExclusionsByInstanceId = new ConcurrentHashMap<>(); private final Map> dirExclusionsByInstanceId = new ConcurrentHashMap<>(); private final LiveMigrationConfiguration liveMigrationConfiguration; @Inject - public LiveMigrationFileStreamHandler(InstanceMetadataFetcher metadataFetcher, - ExecutorPools executorPools, - CassandraInputValidator validator, - SidecarConfiguration sidecarConfiguration) + public LiveMigrationFileResolveHandler(InstanceMetadataFetcher metadataFetcher, + ExecutorPools executorPools, + CassandraInputValidator validator, + SidecarConfiguration sidecarConfiguration) { super(metadataFetcher, executorPools, validator); this.liveMigrationConfiguration = sidecarConfiguration.liveMigrationConfiguration(); diff --git a/server/src/main/java/org/apache/cassandra/sidecar/handlers/livemigration/LiveMigrationGetAllDataCopyTasksHandler.java b/server/src/main/java/org/apache/cassandra/sidecar/handlers/livemigration/LiveMigrationGetAllDataCopyTasksHandler.java index f8d344f70..5daf4eaa3 100644 --- a/server/src/main/java/org/apache/cassandra/sidecar/handlers/livemigration/LiveMigrationGetAllDataCopyTasksHandler.java +++ b/server/src/main/java/org/apache/cassandra/sidecar/handlers/livemigration/LiveMigrationGetAllDataCopyTasksHandler.java @@ -27,7 +27,7 @@ import io.vertx.ext.auth.authorization.Authorization; import io.vertx.ext.web.RoutingContext; import org.apache.cassandra.sidecar.acl.authorization.BasicPermissions; -import org.apache.cassandra.sidecar.common.response.LiveMigrationTaskResponse; +import org.apache.cassandra.sidecar.common.response.LiveMigrationDataCopyResponse; import org.apache.cassandra.sidecar.handlers.AccessProtected; import org.apache.cassandra.sidecar.livemigration.DataCopyTaskManager; import org.apache.cassandra.sidecar.livemigration.LiveMigrationTask; @@ -51,10 +51,10 @@ public LiveMigrationGetAllDataCopyTasksHandler(DataCopyTaskManager dataCopyTaskM public void handle(RoutingContext context) { String localhost = extractHostAddressWithoutPort(context.request()); - List tasks = dataCopyTaskManager.getAllTasks(localhost) - .stream() - .map(LiveMigrationTask::getResponse) - .collect(Collectors.toList()); + List tasks = dataCopyTaskManager.getAllTasks(localhost) + .stream() + .map(LiveMigrationTask::getResponse) + .collect(Collectors.toList()); context.json(tasks); } diff --git a/server/src/main/java/org/apache/cassandra/sidecar/handlers/livemigration/LiveMigrationGetAllFilesVerificationTasksHandler.java b/server/src/main/java/org/apache/cassandra/sidecar/handlers/livemigration/LiveMigrationGetAllFilesVerificationTasksHandler.java new file mode 100644 index 000000000..a084b224e --- /dev/null +++ b/server/src/main/java/org/apache/cassandra/sidecar/handlers/livemigration/LiveMigrationGetAllFilesVerificationTasksHandler.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.sidecar.handlers.livemigration; + +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +import com.google.inject.Inject; +import io.vertx.core.Handler; +import io.vertx.ext.auth.authorization.Authorization; +import io.vertx.ext.web.RoutingContext; +import org.apache.cassandra.sidecar.common.response.LiveMigrationFilesVerificationResponse; +import org.apache.cassandra.sidecar.handlers.AccessProtected; +import org.apache.cassandra.sidecar.livemigration.FilesVerificationTaskManager; +import org.apache.cassandra.sidecar.livemigration.LiveMigrationTask; + +import static org.apache.cassandra.sidecar.acl.authorization.BasicPermissions.DATA_COPY; +import static org.apache.cassandra.sidecar.handlers.AbstractHandler.extractHostAddressWithoutPort; + +/** + * Handler for retrieving all active live migration files verification tasks. + * Returns a list of all verification tasks running on the local host. + */ +public class LiveMigrationGetAllFilesVerificationTasksHandler implements Handler, AccessProtected +{ + private final FilesVerificationTaskManager taskManager; + + @Inject + public LiveMigrationGetAllFilesVerificationTasksHandler(FilesVerificationTaskManager taskManager) + { + this.taskManager = taskManager; + } + + @Override + public void handle(RoutingContext context) + { + String localhost = extractHostAddressWithoutPort(context.request()); + List tasks = taskManager.getAllTasks(localhost) + .stream() + .map(LiveMigrationTask::getResponse) + .collect(Collectors.toList()); + context.json(tasks); + } + + @Override + public Set requiredAuthorizations() + { + return Set.of(DATA_COPY.toAuthorization()); + } +} diff --git a/server/src/main/java/org/apache/cassandra/sidecar/handlers/livemigration/LiveMigrationGetFilesVerificationTaskHandler.java b/server/src/main/java/org/apache/cassandra/sidecar/handlers/livemigration/LiveMigrationGetFilesVerificationTaskHandler.java new file mode 100644 index 000000000..13b5c57b0 --- /dev/null +++ b/server/src/main/java/org/apache/cassandra/sidecar/handlers/livemigration/LiveMigrationGetFilesVerificationTaskHandler.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.sidecar.handlers.livemigration; + +import java.util.Set; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.inject.Inject; +import io.netty.handler.codec.http.HttpResponseStatus; +import io.vertx.core.http.HttpServerRequest; +import io.vertx.core.net.SocketAddress; +import io.vertx.ext.auth.authorization.Authorization; +import io.vertx.ext.web.RoutingContext; +import org.apache.cassandra.sidecar.common.response.LiveMigrationFilesVerificationResponse; +import org.apache.cassandra.sidecar.concurrent.ExecutorPools; +import org.apache.cassandra.sidecar.exceptions.LiveMigrationExceptions.LiveMigrationTaskNotFoundException; +import org.apache.cassandra.sidecar.handlers.AbstractHandler; +import org.apache.cassandra.sidecar.handlers.AccessProtected; +import org.apache.cassandra.sidecar.livemigration.FilesVerificationTaskManager; +import org.apache.cassandra.sidecar.livemigration.LiveMigrationTask; +import org.apache.cassandra.sidecar.utils.CassandraInputValidator; +import org.apache.cassandra.sidecar.utils.InstanceMetadataFetcher; +import org.jetbrains.annotations.NotNull; + +import static org.apache.cassandra.sidecar.acl.authorization.BasicPermissions.DATA_COPY; +import static org.apache.cassandra.sidecar.utils.HttpExceptions.wrapHttpException; + +/** + * Handler for retrieving a specific live migration files verification task by task ID. + * Returns the task details if found, or a 404 error if the task does not exist on the specified host. + */ +public class LiveMigrationGetFilesVerificationTaskHandler extends AbstractHandler implements AccessProtected +{ + private static final Logger LOGGER = LoggerFactory.getLogger(LiveMigrationGetFilesVerificationTaskHandler.class); + + private final FilesVerificationTaskManager taskManager; + + /** + * Constructs a handler with the provided {@code metadataFetcher} + * + * @param metadataFetcher the interface to retrieve instance metadata + * @param executorPools the executor pools for blocking executions + * @param validator a validator instance to validate Cassandra-specific input + */ + @Inject + public LiveMigrationGetFilesVerificationTaskHandler(InstanceMetadataFetcher metadataFetcher, + ExecutorPools executorPools, + CassandraInputValidator validator, + FilesVerificationTaskManager taskManager) + { + super(metadataFetcher, executorPools, validator); + this.taskManager = taskManager; + } + + @Override + protected String extractParamsOrThrow(RoutingContext context) + { + String taskId = context.pathParam("taskId"); + if (taskId == null || taskId.isEmpty()) + { + throw wrapHttpException(HttpResponseStatus.BAD_REQUEST, "taskId is required"); + } + + return taskId; + } + + @Override + protected void handleInternal(RoutingContext context, + HttpServerRequest httpRequest, + @NotNull String host, + SocketAddress remoteAddress, + String taskId) + { + try + { + LiveMigrationTask task = taskManager.getTask(taskId, host); + LOGGER.info("Found live migration task with taskId={} on host={}", taskId, host); + context.json(task.getResponse()); + } + catch (LiveMigrationTaskNotFoundException e) + { + LOGGER.warn("Live migration task not found with taskId={} on host={}", taskId, host); + context.fail(wrapHttpException(HttpResponseStatus.NOT_FOUND, e.getMessage(), e)); + } + } + + @Override + public Set requiredAuthorizations() + { + return Set.of(DATA_COPY.toAuthorization()); + } +} diff --git a/server/src/main/java/org/apache/cassandra/sidecar/livemigration/DataCopyTaskManager.java b/server/src/main/java/org/apache/cassandra/sidecar/livemigration/DataCopyTaskManager.java index faf87080b..1b884f19e 100644 --- a/server/src/main/java/org/apache/cassandra/sidecar/livemigration/DataCopyTaskManager.java +++ b/server/src/main/java/org/apache/cassandra/sidecar/livemigration/DataCopyTaskManager.java @@ -21,9 +21,7 @@ import java.util.Collections; import java.util.List; import java.util.Objects; -import java.util.concurrent.ConcurrentHashMap; -import com.google.common.annotations.VisibleForTesting; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -35,14 +33,17 @@ import org.apache.cassandra.sidecar.cluster.InstancesMetadata; import org.apache.cassandra.sidecar.cluster.instance.InstanceMetadata; import org.apache.cassandra.sidecar.common.request.LiveMigrationDataCopyRequest; +import org.apache.cassandra.sidecar.common.response.LiveMigrationDataCopyResponse; import org.apache.cassandra.sidecar.config.SidecarConfiguration; import org.apache.cassandra.sidecar.exceptions.CassandraUnavailableException; -import org.apache.cassandra.sidecar.exceptions.LiveMigrationExceptions.LiveMigrationDataCopyInProgressException; import org.apache.cassandra.sidecar.exceptions.LiveMigrationExceptions.LiveMigrationInvalidRequestException; +import org.apache.cassandra.sidecar.exceptions.LiveMigrationExceptions.LiveMigrationTaskInProgressException; import org.apache.cassandra.sidecar.exceptions.LiveMigrationExceptions.LiveMigrationTaskNotFoundException; import org.apache.cassandra.sidecar.handlers.livemigration.LiveMigrationMap; import org.jetbrains.annotations.NotNull; +import static org.apache.cassandra.sidecar.livemigration.LiveMigrationDataCopyTask.DATA_COPY_TASK_TYPE; + /** * Manages live migration data copy tasks. Handles task creation, tracking, and cancellation. * Data copy tasks should only be submitted to destination instances. @@ -52,19 +53,20 @@ public class DataCopyTaskManager { private static final Logger LOGGER = LoggerFactory.getLogger(DataCopyTaskManager.class); - @VisibleForTesting - final ConcurrentHashMap currentTasks = new ConcurrentHashMap<>(); + private final LiveMigrationTaskManager taskManager; private final InstancesMetadata instancesMetadata; private final SidecarConfiguration sidecarConfiguration; private final LiveMigrationMap liveMigrationMap; private final LiveMigrationTaskFactory liveMigrationTaskFactory; @Inject - public DataCopyTaskManager(InstancesMetadata instancesMetadata, + public DataCopyTaskManager(LiveMigrationTaskManager taskManager, + InstancesMetadata instancesMetadata, SidecarConfiguration sidecarConfiguration, LiveMigrationMap liveMigrationMap, LiveMigrationTaskFactory liveMigrationTaskFactory) { + this.taskManager = taskManager; this.instancesMetadata = instancesMetadata; this.sidecarConfiguration = sidecarConfiguration; this.liveMigrationMap = liveMigrationMap; @@ -77,16 +79,17 @@ public DataCopyTaskManager(InstancesMetadata instancesMetadata, * * @param request live migration request containing task parameters * @param currentHost the destination host where sidecar is running - * @return Future containing the created LiveMigrationTask - * @throws LiveMigrationDataCopyInProgressException if another task is already running - * @throws LiveMigrationInvalidRequestException if request validation fails + * @return Future containing the created LiveMigrationTask, or a failed Future with: + *
    + *
  • {@link LiveMigrationInvalidRequestException} if max concurrency exceeds the configured limit
    • + *
  • {@link LiveMigrationTaskInProgressException} if another task is already running for this instance
    • + *
*/ - public Future createTask(@NotNull LiveMigrationDataCopyRequest request, - @NotNull String currentHost) - throws LiveMigrationDataCopyInProgressException, LiveMigrationInvalidRequestException + public Future> createTask(@NotNull LiveMigrationDataCopyRequest request, + @NotNull String currentHost) { int maxPossibleConcurrency = Objects.requireNonNull(sidecarConfiguration.liveMigrationConfiguration()) - .maxConcurrentDownloads(); + .maxConcurrentFileRequests(); if (request.maxConcurrency > maxPossibleConcurrency) { return Future.failedFuture( @@ -99,34 +102,26 @@ public Future createTask(@NotNull LiveMigrationDataCopyReques .compose(source -> createDataCopyTask(request, source, localInstanceMetadata)); } - Future createDataCopyTask(LiveMigrationDataCopyRequest request, - String source, - InstanceMetadata localInstanceMetadata) + private Future> createDataCopyTask(LiveMigrationDataCopyRequest request, + String source, + InstanceMetadata localInstanceMetadata) { // Fast local JMX check before creating task - prevents task creation if Cassandra is running return verifyCassandraNotRunning(localInstanceMetadata) .compose(v -> { - LiveMigrationTask newTask = createTask(request, - source, - sidecarConfiguration.serviceConfiguration().port(), - localInstanceMetadata); + LiveMigrationTask newTask = createTask(request, + source, + sidecarConfiguration.serviceConfiguration().port(), + localInstanceMetadata); // It is possible to serve only one live migration data copy request per instance at a time. // Checking if there is another migration is in progress before accepting new one. - boolean accepted = newTask == currentTasks.compute(localInstanceMetadata.id(), (integer, taskInMap) -> { - if (taskInMap == null) - { - return newTask; - } - - // Accept new task if and only if the existing task has completed. - return taskInMap.isCompleted() ? newTask : taskInMap; - }); + boolean accepted = taskManager.submitTask(localInstanceMetadata.id(), newTask); if (!accepted) { return Future.failedFuture( - new LiveMigrationDataCopyInProgressException("Another task is already under progress. Cannot accept new task.")); + new LiveMigrationTaskInProgressException("Another task is already under progress. Cannot accept new task.")); } LOGGER.info("Starting data copy task with id={}, source={}, destination={}", newTask.id(), source, localInstanceMetadata.host()); @@ -175,72 +170,87 @@ private Future verifyCassandraNotRunning(InstanceMetadata localInstance) } } - LiveMigrationTask createTask(LiveMigrationDataCopyRequest request, - String source, - int port, - InstanceMetadata localInstanceMetadata) + LiveMigrationTask createTask(LiveMigrationDataCopyRequest request, + String source, + int port, + InstanceMetadata localInstanceMetadata) { - return liveMigrationTaskFactory.create(UUIDs.timeBased().toString(), request, source, port, localInstanceMetadata); } /** - * Returns all live migration tasks for the current host. + * Returns all live migration data copy tasks for the current host. * * @param currentHost the host where sidecar is running - * @return list containing at most one task (empty if no active task) + * @return list containing at most one task (empty if no active task or if task is not a data copy task) */ - public List getAllTasks(@NotNull String currentHost) + public List> getAllTasks(@NotNull String currentHost) { - InstanceMetadata localInstance = instancesMetadata.instanceFromHost(currentHost); - if (currentTasks.isEmpty() || !currentTasks.containsKey(localInstance.id())) + List> tasks = taskManager.getAllTasks(currentHost); + if (tasks.isEmpty()) { return Collections.emptyList(); } - return Collections.singletonList(currentTasks.get(localInstance.id())); + + LiveMigrationTask task = tasks.get(0); + if (isDataCopyTask(task)) + { + return List.of(castToDataCopyTask(task)); + } + + return Collections.emptyList(); } /** - * Returns the live migration task with the specified task ID. + * Returns the live migration data copy task with the specified task ID. * * @param taskId ID of the task to retrieve * @param currentHost the host where sidecar is running * @return the LiveMigrationTask matching the given taskId - * @throws LiveMigrationTaskNotFoundException if no task found with the given ID + * @throws LiveMigrationTaskNotFoundException if no task found with the given ID or if task is not a data copy task */ - public LiveMigrationTask getTask(@NotNull String taskId, - @NotNull String currentHost) throws LiveMigrationTaskNotFoundException + public LiveMigrationTask getTask(@NotNull String taskId, + @NotNull String currentHost) throws LiveMigrationTaskNotFoundException { - return getLiveMigrationTask(taskId, currentHost); + LiveMigrationTask task = taskManager.getTask(taskId, currentHost); + + if (isDataCopyTask(task)) + { + return castToDataCopyTask(task); + } + + throw new LiveMigrationTaskNotFoundException("Task " + taskId + " is not a data copy task"); } /** - * Cancels the live migration task with the specified task ID. + * Cancels the live migration data copy task with the specified task ID. * * @param taskId ID of the task to cancel * @param currentHost the host where sidecar is running * @return the cancelled LiveMigrationTask - * @throws LiveMigrationTaskNotFoundException if no task found with the given ID + * @throws LiveMigrationTaskNotFoundException if no task found with the given ID or if task is not a data copy task */ - public LiveMigrationTask cancelTask(@NotNull String taskId, - @NotNull String currentHost) throws LiveMigrationTaskNotFoundException + public LiveMigrationTask cancelTask(@NotNull String taskId, + @NotNull String currentHost) throws LiveMigrationTaskNotFoundException { - LiveMigrationTask taskInProgress = getLiveMigrationTask(taskId, currentHost); + LiveMigrationTask task = taskManager.getTask(taskId, currentHost); - // Cancelling the task - taskInProgress.cancel(); + if (isDataCopyTask(task)) + { + return castToDataCopyTask(taskManager.cancelTask(taskId, currentHost)); + } - return taskInProgress; + throw new LiveMigrationTaskNotFoundException("Task " + taskId + " is not a data copy task"); } - private LiveMigrationTask getLiveMigrationTask(@NotNull String taskId, @NotNull String currentHost) + private boolean isDataCopyTask(LiveMigrationTask task) { - InstanceMetadata localInstance = instancesMetadata.instanceFromHost(currentHost); - LiveMigrationTask taskInProgress = currentTasks.get(localInstance.id()); - if (null == taskInProgress || !taskId.equals(taskInProgress.id())) - { - throw new LiveMigrationTaskNotFoundException("No data copy task found with given id " + taskId); - } - return taskInProgress; + return DATA_COPY_TASK_TYPE.equals(task.type()); + } + + @SuppressWarnings("unchecked") + private LiveMigrationTask castToDataCopyTask(LiveMigrationTask task) + { + return (LiveMigrationTask) task; } } diff --git a/server/src/main/java/org/apache/cassandra/sidecar/livemigration/FilesVerificationTaskManager.java b/server/src/main/java/org/apache/cassandra/sidecar/livemigration/FilesVerificationTaskManager.java new file mode 100644 index 000000000..7ede963df --- /dev/null +++ b/server/src/main/java/org/apache/cassandra/sidecar/livemigration/FilesVerificationTaskManager.java @@ -0,0 +1,194 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.sidecar.livemigration; + +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.datastax.driver.core.utils.UUIDs; +import com.google.inject.Inject; +import com.google.inject.Singleton; +import io.vertx.core.Future; +import org.apache.cassandra.sidecar.cluster.instance.InstanceMetadata; +import org.apache.cassandra.sidecar.common.request.LiveMigrationFilesVerificationRequest; +import org.apache.cassandra.sidecar.common.response.LiveMigrationFilesVerificationResponse; +import org.apache.cassandra.sidecar.config.SidecarConfiguration; +import org.apache.cassandra.sidecar.exceptions.LiveMigrationExceptions.LiveMigrationInvalidRequestException; +import org.apache.cassandra.sidecar.exceptions.LiveMigrationExceptions.LiveMigrationTaskInProgressException; +import org.apache.cassandra.sidecar.exceptions.LiveMigrationExceptions.LiveMigrationTaskNotFoundException; +import org.apache.cassandra.sidecar.utils.DigestAlgorithmFactory; +import org.jetbrains.annotations.NotNull; + +import static org.apache.cassandra.sidecar.livemigration.LiveMigrationFilesVerificationTask.FILES_VERIFICATION_TASK_TYPE; + +/** + * Manages the lifecycle of file digest verification tasks during live migration operations. + * This manager ensures that only one {@link LiveMigrationTask} can be active per instance at a time, + * preventing concurrent migration operations that could impact system resources. + * Tasks are created using the {@link LiveMigrationFilesVerificationTaskFactory} and + * executed asynchronously to validate file integrity between source and destination nodes. + */ +@Singleton +public class FilesVerificationTaskManager +{ + private static final Logger LOGGER = LoggerFactory.getLogger(FilesVerificationTaskManager.class); + + private final LiveMigrationTaskManager taskManager; + private final LiveMigrationFilesVerificationTaskFactory taskFactory; + private final SidecarConfiguration sidecarConfiguration; + + @Inject + public FilesVerificationTaskManager(LiveMigrationTaskManager taskManager, + SidecarConfiguration sidecarConfiguration, + LiveMigrationFilesVerificationTaskFactory taskFactory) + { + this.taskManager = taskManager; + this.sidecarConfiguration = sidecarConfiguration; + this.taskFactory = taskFactory; + } + + /** + * Creates and submits a new file digest verification task for the specified instance. + * Only one task (of any type) can be active per instance at a time. If a task is already running, + * this method returns a failed future with {@link LiveMigrationTaskInProgressException}. + * + * @param request the file verification request containing digest information + * @param source the source identifier for the verification request + * @param localInstanceMetadata metadata of the local Cassandra instance + * @return a future that succeeds if the task is created and started, or fails with + * {@link LiveMigrationTaskInProgressException} if another task is already in progress + */ + public Future> createTask(LiveMigrationFilesVerificationRequest request, + String source, + InstanceMetadata localInstanceMetadata) + { + int maxPossibleConcurrency = sidecarConfiguration.liveMigrationConfiguration().maxConcurrentFileRequests(); + if (request.maxConcurrency() > maxPossibleConcurrency) + { + return Future.failedFuture( + new LiveMigrationInvalidRequestException("max concurrency can not be more than " + maxPossibleConcurrency)); + } + try + { + DigestAlgorithmFactory.validateAlgorithmName(request.digestAlgorithm()); + } + catch (IllegalArgumentException iae) + { + return Future.failedFuture(new LiveMigrationInvalidRequestException(iae.getMessage(), iae)); + } + + String timeUuid = UUIDs.timeBased().toString(); + int sidecarPort = sidecarConfiguration.serviceConfiguration().port(); + LiveMigrationTask newTask = + taskFactory.create(timeUuid, source, sidecarPort, request, localInstanceMetadata); + + boolean accepted = taskManager.submitTask(localInstanceMetadata.id(), newTask); + + if (accepted) + { + newTask.start(); + LOGGER.info("Accepted new files digest verification task for instance={} taskId={}", + localInstanceMetadata.id(), newTask.id()); + return Future.succeededFuture(newTask); + } + else + { + return Future.failedFuture(new LiveMigrationTaskInProgressException( + "Another files digest verification is in progress for instance=" + localInstanceMetadata.id())); + } + } + + /** + * Returns all live migration files verification tasks for the current host. + * + * @param currentHost the host where sidecar is running + * @return list containing at most one task (empty if no active task or if task is not a files verification task) + */ + public List> getAllTasks(@NotNull String currentHost) + { + List> tasks = taskManager.getAllTasks(currentHost); + if (tasks.isEmpty()) + { + return List.of(); + } + + LiveMigrationTask task = tasks.get(0); + if (isFilesVerificationTask(task)) + { + return List.of(castToFilesVerificationTask(task)); + } + + return List.of(); + } + + /** + * Returns the files verification task with the specified task ID. + * + * @param taskId ID of the task to retrieve + * @param currentHost the host where sidecar is running + * @return the LiveMigrationTask matching the given taskId + * @throws LiveMigrationTaskNotFoundException if no task found with the given ID or if task is not a files verification task + */ + public LiveMigrationTask getTask(@NotNull String taskId, + @NotNull String currentHost) throws LiveMigrationTaskNotFoundException + { + LiveMigrationTask task = taskManager.getTask(taskId, currentHost); + + if (isFilesVerificationTask(task)) + { + return castToFilesVerificationTask(task); + } + + throw new LiveMigrationTaskNotFoundException("Task " + taskId + " is not a files verification task"); + } + + /** + * Cancels the files verification task with the specified task ID. + * + * @param taskId ID of the task to cancel + * @param currentHost the host where sidecar is running + * @return the cancelled LiveMigrationTask + * @throws LiveMigrationTaskNotFoundException if no task found with the given ID or if task is not a files verification task + */ + public LiveMigrationTask cancelTask(@NotNull String taskId, + @NotNull String currentHost) throws LiveMigrationTaskNotFoundException + { + LiveMigrationTask task = taskManager.getTask(taskId, currentHost); + + if (isFilesVerificationTask(task)) + { + return castToFilesVerificationTask(taskManager.cancelTask(taskId, currentHost)); + } + + throw new LiveMigrationTaskNotFoundException("Task " + taskId + " is not a files verification task"); + } + + private boolean isFilesVerificationTask(LiveMigrationTask task) + { + return FILES_VERIFICATION_TASK_TYPE.equals(task.type()); + } + + @SuppressWarnings("unchecked") + private LiveMigrationTask castToFilesVerificationTask(LiveMigrationTask task) + { + return (LiveMigrationTask) task; + } +} diff --git a/server/src/main/java/org/apache/cassandra/sidecar/livemigration/LiveMigrationTaskImpl.java b/server/src/main/java/org/apache/cassandra/sidecar/livemigration/LiveMigrationDataCopyTask.java similarity index 75% rename from server/src/main/java/org/apache/cassandra/sidecar/livemigration/LiveMigrationTaskImpl.java rename to server/src/main/java/org/apache/cassandra/sidecar/livemigration/LiveMigrationDataCopyTask.java index c64b14be4..6a4d9d628 100644 --- a/server/src/main/java/org/apache/cassandra/sidecar/livemigration/LiveMigrationTaskImpl.java +++ b/server/src/main/java/org/apache/cassandra/sidecar/livemigration/LiveMigrationDataCopyTask.java @@ -31,7 +31,7 @@ import io.vertx.core.Vertx; import org.apache.cassandra.sidecar.cluster.instance.InstanceMetadata; import org.apache.cassandra.sidecar.common.request.LiveMigrationDataCopyRequest; -import org.apache.cassandra.sidecar.common.response.LiveMigrationTaskResponse; +import org.apache.cassandra.sidecar.common.response.LiveMigrationDataCopyResponse; import org.apache.cassandra.sidecar.concurrent.ExecutorPools; import org.apache.cassandra.sidecar.config.LiveMigrationConfiguration; import org.apache.cassandra.sidecar.utils.SidecarClientProvider; @@ -40,8 +40,10 @@ * Implementation of live migration task that handles file downloading with retry logic. * Manages the lifecycle of data copy operations from source to destination instances. */ -public class LiveMigrationTaskImpl implements LiveMigrationTask +public class LiveMigrationDataCopyTask implements LiveMigrationTask { + public static final String DATA_COPY_TASK_TYPE = "data-copy-task"; + private final String id; private final LiveMigrationDataCopyRequest request; private final Map statusMap = new TreeMap<>(); @@ -62,15 +64,16 @@ public class LiveMigrationTaskImpl implements LiveMigrationTask volatile LiveMigrationFileDownloader downloader; private volatile boolean cancelled = false; - public LiveMigrationTaskImpl(Vertx vertx, - ExecutorPools executorPools, - SidecarClientProvider sidecarClientProvider, - LiveMigrationConfiguration liveMigrationConfiguration, - String id, - LiveMigrationDataCopyRequest request, - String source, - int port, - InstanceMetadata instanceMetadata, LiveMigrationFileDownloadPreCheck preCheck) + public LiveMigrationDataCopyTask(Vertx vertx, + ExecutorPools executorPools, + SidecarClientProvider sidecarClientProvider, + LiveMigrationConfiguration liveMigrationConfiguration, + String id, + LiveMigrationDataCopyRequest request, + String source, + int port, + InstanceMetadata instanceMetadata, + LiveMigrationFileDownloadPreCheck preCheck) { this.vertx = vertx; this.executorPools = executorPools; @@ -93,6 +96,12 @@ public String id() return id; } + @Override + public String type() + { + return DATA_COPY_TASK_TYPE; + } + /** * {@inheritDoc} */ @@ -184,9 +193,9 @@ public void cancel() * {@inheritDoc} */ @Override - public LiveMigrationTaskResponse getResponse() + public LiveMigrationDataCopyResponse getResponse() { - return new LiveMigrationTaskResponse(id, source, port, request, getStatusResponse()); + return new LiveMigrationDataCopyResponse(id, source, port, request, getStatusResponse()); } Consumer statusUpdater(int iteration) @@ -194,23 +203,23 @@ Consumer statusUpdater(int iteration) return (operationStatus) -> statusMap.put(iteration, operationStatus); } - private List getStatusResponse() + private List getStatusResponse() { return statusMap.entrySet().stream() .map(entry -> toStatusResponse(entry.getValue(), entry.getKey())) .collect(Collectors.toList()); } - private LiveMigrationTaskResponse.Status toStatusResponse(OperationStatus operationStatus, int iteration) + private LiveMigrationDataCopyResponse.Status toStatusResponse(OperationStatus operationStatus, int iteration) { - return new LiveMigrationTaskResponse.Status(iteration, - operationStatus.state().toString(), - operationStatus.totalSize(), - operationStatus.totalFiles(), - operationStatus.bytesToDownload(), - operationStatus.filesToDownload(), - operationStatus.filesDownloaded(), - operationStatus.downloadFailures(), - operationStatus.bytesDownloaded()); + return new LiveMigrationDataCopyResponse.Status(iteration, + operationStatus.state().toString(), + operationStatus.totalSize(), + operationStatus.totalFiles(), + operationStatus.bytesToDownload(), + operationStatus.filesToDownload(), + operationStatus.filesDownloaded(), + operationStatus.downloadFailures(), + operationStatus.bytesDownloaded()); } } diff --git a/server/src/main/java/org/apache/cassandra/sidecar/livemigration/LiveMigrationTaskFactoryImpl.java b/server/src/main/java/org/apache/cassandra/sidecar/livemigration/LiveMigrationDataCopyTaskFactoryImpl.java similarity index 62% rename from server/src/main/java/org/apache/cassandra/sidecar/livemigration/LiveMigrationTaskFactoryImpl.java rename to server/src/main/java/org/apache/cassandra/sidecar/livemigration/LiveMigrationDataCopyTaskFactoryImpl.java index 4c254f20c..f8b661a9e 100644 --- a/server/src/main/java/org/apache/cassandra/sidecar/livemigration/LiveMigrationTaskFactoryImpl.java +++ b/server/src/main/java/org/apache/cassandra/sidecar/livemigration/LiveMigrationDataCopyTaskFactoryImpl.java @@ -23,6 +23,7 @@ import io.vertx.core.Vertx; import org.apache.cassandra.sidecar.cluster.instance.InstanceMetadata; import org.apache.cassandra.sidecar.common.request.LiveMigrationDataCopyRequest; +import org.apache.cassandra.sidecar.common.response.LiveMigrationDataCopyResponse; import org.apache.cassandra.sidecar.concurrent.ExecutorPools; import org.apache.cassandra.sidecar.config.LiveMigrationConfiguration; import org.apache.cassandra.sidecar.config.SidecarConfiguration; @@ -32,7 +33,7 @@ * Factory implementation which produces {@link LiveMigrationTask} instances. */ @Singleton -public class LiveMigrationTaskFactoryImpl implements LiveMigrationTaskFactory +public class LiveMigrationDataCopyTaskFactoryImpl implements LiveMigrationTaskFactory { private final Vertx vertx; @@ -42,11 +43,11 @@ public class LiveMigrationTaskFactoryImpl implements LiveMigrationTaskFactory private final LiveMigrationFileDownloadPreCheck preCheck; @Inject - public LiveMigrationTaskFactoryImpl(Vertx vertx, - ExecutorPools executorPools, - SidecarClientProvider sidecarClientProvider, - SidecarConfiguration sidecarConfiguration, - LiveMigrationFileDownloadPreCheck preCheck) + public LiveMigrationDataCopyTaskFactoryImpl(Vertx vertx, + ExecutorPools executorPools, + SidecarClientProvider sidecarClientProvider, + SidecarConfiguration sidecarConfiguration, + LiveMigrationFileDownloadPreCheck preCheck) { this.vertx = vertx; this.executorPools = executorPools; @@ -59,13 +60,13 @@ public LiveMigrationTaskFactoryImpl(Vertx vertx, * {@inheritDoc} */ @Override - public LiveMigrationTask create(String id, - LiveMigrationDataCopyRequest request, - String source, - int port, - InstanceMetadata instanceMetadata) + public LiveMigrationTask create(String id, + LiveMigrationDataCopyRequest request, + String source, + int port, + InstanceMetadata instanceMetadata) { - return new LiveMigrationTaskImpl(vertx, executorPools, sidecarClientProvider, liveMigrationConfiguration, - id, request, source, port, instanceMetadata, preCheck); + return new LiveMigrationDataCopyTask(vertx, executorPools, sidecarClientProvider, liveMigrationConfiguration, + id, request, source, port, instanceMetadata, preCheck); } } diff --git a/server/src/main/java/org/apache/cassandra/sidecar/livemigration/LiveMigrationFilesVerificationTask.java b/server/src/main/java/org/apache/cassandra/sidecar/livemigration/LiveMigrationFilesVerificationTask.java new file mode 100644 index 000000000..950754259 --- /dev/null +++ b/server/src/main/java/org/apache/cassandra/sidecar/livemigration/LiveMigrationFilesVerificationTask.java @@ -0,0 +1,619 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.sidecar.livemigration; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.Callable; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import io.vertx.core.Future; +import io.vertx.core.MultiMap; +import io.vertx.core.Promise; +import io.vertx.core.Vertx; +import org.apache.cassandra.sidecar.client.SidecarClient; +import org.apache.cassandra.sidecar.client.SidecarInstanceImpl; +import org.apache.cassandra.sidecar.cluster.instance.InstanceMetadata; +import org.apache.cassandra.sidecar.common.DataObjectBuilder; +import org.apache.cassandra.sidecar.common.request.LiveMigrationFilesVerificationRequest; +import org.apache.cassandra.sidecar.common.request.data.Digest; +import org.apache.cassandra.sidecar.common.response.DigestResponse; +import org.apache.cassandra.sidecar.common.response.InstanceFileInfo; +import org.apache.cassandra.sidecar.common.response.InstanceFilesListResponse; +import org.apache.cassandra.sidecar.common.response.LiveMigrationFilesVerificationResponse; +import org.apache.cassandra.sidecar.concurrent.AsyncConcurrentTaskExecutor; +import org.apache.cassandra.sidecar.concurrent.ExecutorPools; +import org.apache.cassandra.sidecar.config.LiveMigrationConfiguration; +import org.apache.cassandra.sidecar.exceptions.LiveMigrationExceptions.DigestMismatchException; +import org.apache.cassandra.sidecar.exceptions.LiveMigrationExceptions.FileVerificationFailureException; +import org.apache.cassandra.sidecar.utils.DigestVerifierFactory; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.VisibleForTesting; + +import static org.apache.cassandra.sidecar.common.response.InstanceFileInfo.FileType.DIRECTORY; +import static org.apache.cassandra.sidecar.livemigration.LiveMigrationInstanceMetadataUtil.localPath; + +/** + * Verifies file integrity between source and destination instances during live migration + * by comparing file lists and validating file digests using configurable digest algorithms. + * + *

The verification process consists of three stages: + *

    + *
  1. Fetch file lists from both source and destination instances concurrently
    2. + *
  2. Compare file metadata (size, type, modification time) - fails when there are mismatches
    3. + *
  3. Verify cryptographic digests (MD5 or XXHash32) with configurable concurrency
    4. + *
+ * + *

The task supports cancellation, and tracks detailed metrics including + * metadata matches/mismatches, digest verification results, and failure counts. + */ +public class LiveMigrationFilesVerificationTask implements LiveMigrationTask +{ + public static final String FILES_VERIFICATION_TASK_TYPE = "files-verification-task"; + private static final Logger LOGGER = LoggerFactory.getLogger(LiveMigrationFilesVerificationTask.class); + private final Vertx vertx; + private final String id; + private final String source; + private final int port; + // Instance metadata of current instance + private final InstanceMetadata instanceMetadata; + private final ExecutorPools executorPools; + private final LiveMigrationFilesVerificationRequest request; + private final SidecarClient sidecarClient; + private final LiveMigrationConfiguration liveMigrationConfiguration; + + @VisibleForTesting + final DigestVerifierFactory digestVerifierFactory; + + private final AtomicReference state; + private final Promise completionPromise = Promise.promise(); + private final AtomicInteger filesNotFoundAtSource = new AtomicInteger(0); + private final AtomicInteger filesNotFoundAtDestination = new AtomicInteger(0); + private final AtomicInteger metadataMatched = new AtomicInteger(0); + private final AtomicInteger metadataMismatches = new AtomicInteger(0); + private final AtomicInteger digestMismatches = new AtomicInteger(0); + private final AtomicInteger digestVerificationFailures = new AtomicInteger(0); + private final AtomicInteger filesMatched = new AtomicInteger(0); + private final String logPrefix; + private final AtomicBoolean cancelled = new AtomicBoolean(false); + private volatile AsyncConcurrentTaskExecutor asyncConcurrentTaskExecutor; + + @VisibleForTesting + protected LiveMigrationFilesVerificationTask(Builder builder) + { + this.vertx = builder.vertx; + this.id = builder.id; + this.source = builder.source; + this.port = builder.port; + this.instanceMetadata = builder.instanceMetadata; + this.executorPools = builder.executorPools; + this.request = builder.request; + this.sidecarClient = builder.sidecarClient; + this.liveMigrationConfiguration = builder.liveMigrationConfiguration; + this.digestVerifierFactory = builder.digestVerifierFactory; + this.logPrefix = "fileVerifyTaskId=" + id; + this.state = new AtomicReference<>(State.NOT_STARTED); + } + + public static Builder builder() + { + return new Builder(); + } + + public Future validate() + { + return abortIfCancelled(new Object()) + // Get files info from both source and destination + .compose(v -> Future.join(listFilesInLocal(), fetchSourceFileList())) + + // Compare the files information before calculating digests + .compose(this::abortIfCancelled) + .compose(cf -> compareFilesMetadata(cf.resultAt(0), cf.resultAt(1))) + + // Verify digests of each local file with source file + .compose(this::abortIfCancelled) + .compose(this::verifyDigests) + + .andThen(ar -> { + if (ar.failed()) + { + LOGGER.error("{} Files verification task failed during validation", logPrefix, ar.cause()); + // Fail the task if it is in IN_PROGRESS state. + state.compareAndSet(State.IN_PROGRESS, State.FAILED); + } + else + { + State currentState = state.compareAndExchange(State.IN_PROGRESS, State.COMPLETED); + if (currentState == State.CANCELLED) + { + LOGGER.debug("{} Task was cancelled before completion", logPrefix); + } + else if (currentState != State.IN_PROGRESS) + { + LOGGER.warn("{} Unexpected state transition failure. State was {}", logPrefix, currentState); + } + else + { + LOGGER.info("{} Files verification task completed successfully", logPrefix); + } + } + }); + } + + private Future> listFilesInLocal() + { + return executorPools.internal() + .executeBlocking(() -> new CassandraInstanceFilesImpl(instanceMetadata, + liveMigrationConfiguration) + .files()); + } + + private Future fetchSourceFileList() + { + return Future.fromCompletionStage( + sidecarClient.liveMigrationListInstanceFilesAsync(new SidecarInstanceImpl(source, port))) + .onFailure(cause -> LOGGER.error("{} Failed to obtain list of files from source {}:{}", + logPrefix, source, port, cause)); + } + + private @NotNull Future> compareFilesMetadata(List localFiles, + InstanceFilesListResponse sourceFiles) + { + Map filesAtLocal = + localFiles.stream() + .collect(Collectors.toMap(fileInfo -> fileInfo.fileUrl, fileInfo -> fileInfo)); + + Map filesAtSource = + sourceFiles.getFiles().stream() + .collect(Collectors.toMap(fileInfo -> fileInfo.fileUrl, fileInfo -> fileInfo)); + + for (Map.Entry localFileEntry : filesAtLocal.entrySet()) + { + String fileUrl = localFileEntry.getKey(); + InstanceFileInfo localFile = localFileEntry.getValue(); + if (filesAtSource.containsKey(fileUrl)) + { + InstanceFileInfo sourceFile = filesAtSource.get(fileUrl); + // compare files metadata + if (localFile.equals(sourceFile) || + (localFile.fileType == DIRECTORY && sourceFile.fileType == DIRECTORY)) + { + // File info matches if either of them are directories or + // their size, last modified time etc... matches + LOGGER.debug("{} {} file metadata matched with source", logPrefix, fileUrl); + metadataMatched.incrementAndGet(); + } + else + { + // Files metadata did not match + metadataMismatches.incrementAndGet(); + LOGGER.error("{} {} did not match with source. Local file info={}, source file info={}", + logPrefix, fileUrl, localFile, sourceFile); + } + + // done with processing current local file, remove it + filesAtSource.remove(fileUrl); + } + else + { + // File is not available at source, report it + LOGGER.error("{} File {} exists at destination but not found at source", logPrefix, fileUrl); + filesNotFoundAtSource.incrementAndGet(); + } + } + + for (Map.Entry sourceFileEntry : filesAtSource.entrySet()) + { + // Remaining files are not present at destination, report error for them + LOGGER.error("{} File {} exists at source but not found at destination", logPrefix, sourceFileEntry.getKey()); + filesNotFoundAtDestination.incrementAndGet(); + } + + if (filesNotFoundAtDestination.get() > 0 || + filesNotFoundAtSource.get() > 0 || + metadataMismatches.get() > 0) + { + FileVerificationFailureException exception = + new FileVerificationFailureException("Files list did not match between source and destination"); + + return Future.failedFuture(exception); + } + + return Future.succeededFuture(localFiles); + } + + private @NotNull Future verifyDigests(List localFiles) + { + // now verify digests of each local file with source file + List>> tasks = new ArrayList<>(localFiles.size()); + localFiles.stream() + .filter(fileInfo -> fileInfo.fileType != DIRECTORY) + .forEach(fileInfo -> tasks.add(() -> verifyDigest(fileInfo))); + asyncConcurrentTaskExecutor = new AsyncConcurrentTaskExecutor<>(vertx, tasks, request.maxConcurrency()); + List> verifyTaskResults = asyncConcurrentTaskExecutor.start(); + return Future.join(verifyTaskResults) + .mapEmpty() + .recover(cause -> Future.failedFuture( + new FileVerificationFailureException("File digests did not match between source and destination"))); + } + + @VisibleForTesting + Future verifyDigest(InstanceFileInfo fileInfo) + { + return getSourceFileDigest(fileInfo) + .compose(digest -> { + String path = localPath(fileInfo.fileUrl, instanceMetadata).toAbsolutePath().toString(); + return digestVerifierFactory.verifier(MultiMap.caseInsensitiveMultiMap().addAll(digest.headers())) + .verify(path) + .compose(verified -> Future.succeededFuture(path)) + .recover(cause -> Future.failedFuture( + new DigestMismatchException(path, fileInfo.fileUrl, cause))); + }) + .transform(ar -> { + if (ar.succeeded()) + { + filesMatched.incrementAndGet(); + LOGGER.debug("{} Verified file {}", logPrefix, fileInfo.fileUrl); + return Future.succeededFuture(ar.result()); + } + Throwable cause = ar.cause(); + if (cause instanceof DigestMismatchException) + { + digestMismatches.incrementAndGet(); + LOGGER.error("{} File digests did not match for {}.", + logPrefix, ((DigestMismatchException) cause).fileUrl(), cause); + } + else + { + digestVerificationFailures.incrementAndGet(); + LOGGER.error("{} Failed to verify file {}", logPrefix, fileInfo.fileUrl, cause); + } + return Future.failedFuture(cause); + }); + } + + private Future getSourceFileDigest(InstanceFileInfo fileInfo) + { + return Future.fromCompletionStage(sidecarClient.liveMigrationFileDigestAsync(new SidecarInstanceImpl(source, port), + fileInfo.fileUrl, + request.digestAlgorithm())) + .compose(this::toDigest); + } + + Future toDigest(DigestResponse digestResponse) + { + try + { + return Future.succeededFuture(digestResponse.toDigest()); + } + catch (Exception e) + { + return Future.failedFuture(e); + } + } + + @Override + public String id() + { + return id; + } + + @Override + public String type() + { + return FILES_VERIFICATION_TASK_TYPE; + } + + @Override + public void start() + { + if (state.compareAndSet(State.NOT_STARTED, State.IN_PROGRESS)) + { + validate().onComplete(ar -> { + if (ar.failed()) + { + completionPromise.tryFail(ar.cause()); + } + else + { + completionPromise.tryComplete(ar.result()); + } + }); + } + else + { + LOGGER.warn("{} Cannot start task. Current state: {}", logPrefix, state.get()); + } + } + + @VisibleForTesting + Future future() + { + return completionPromise.future(); + } + + public boolean hasStarted() + { + return state.get() != State.NOT_STARTED; + } + + /** + * Checks if the task has been cancelled and aborts the validation pipeline if so. + * Updates task state to {@link State#CANCELLED} if currently + * {@link State#NOT_STARTED} or {@link State#IN_PROGRESS}. + * + * @param t the value to pass through + * @param the type of the value + * @return succeeded future with the input if not cancelled, failed future otherwise + */ + @VisibleForTesting + Future abortIfCancelled(T t) + { + if (cancelled.get()) + { + State latestState = state.updateAndGet(currentState -> { + if (currentState == State.NOT_STARTED || currentState == State.IN_PROGRESS) + { + return State.CANCELLED; + } + return currentState; + }); + + if (latestState == State.CANCELLED) + { + LOGGER.info("{} Files verification task successfully cancelled.", logPrefix); + } + else + { + LOGGER.warn("{} could not update files verification task status to cancelled. Current state={}", + logPrefix, state.get()); + } + return Future.failedFuture("Task got cancelled"); + } + + return Future.succeededFuture(t); + } + + @Override + public LiveMigrationFilesVerificationResponse getResponse() + { + return new LiveMigrationFilesVerificationResponse(id, + request.digestAlgorithm(), + this.state.get().name(), + source, + port, + filesNotFoundAtSource.get(), + filesNotFoundAtDestination.get(), + metadataMatched.get(), + metadataMismatches.get(), + digestMismatches.get(), + digestVerificationFailures.get(), + filesMatched.get()); + } + + @Override + public void cancel() + { + if (cancelled.compareAndSet(false, true)) + { + LOGGER.info("{} Cancelling files verification task", logPrefix); + abortIfCancelled(null); + AsyncConcurrentTaskExecutor executor = this.asyncConcurrentTaskExecutor; + if (executor != null) + { + executor.cancelTasks(); + } + + if (!completionPromise.future().isComplete()) + { + completionPromise.tryFail("Task was cancelled."); + } + } + else + { + LOGGER.info("{} Files verification task already cancelled", logPrefix); + } + } + + public boolean isCancelled() + { + return cancelled.get(); + } + + @Override + public boolean isCompleted() + { + return completionPromise.future().isComplete(); + } + + /** + * Represents the current state of the file verification task + */ + public enum State + { + NOT_STARTED, IN_PROGRESS, COMPLETED, FAILED, CANCELLED + } + + /** + * {@code LiveMigrationFilesVerificationTask} builder static inner class. + */ + public static class Builder implements DataObjectBuilder + { + private Vertx vertx; + private String id; + private String source; + private int port; + private InstanceMetadata instanceMetadata; + private ExecutorPools executorPools; + private LiveMigrationFilesVerificationRequest request; + private SidecarClient sidecarClient; + private LiveMigrationConfiguration liveMigrationConfiguration; + private DigestVerifierFactory digestVerifierFactory; + + protected Builder() + { + } + + @Override + public Builder self() + { + return this; + } + + /** + * Sets the {@code vertx} and returns a reference to this Builder enabling method chaining. + * + * @param vertx the {@code vertx} to set + * @return a reference to this Builder + */ + public Builder vertx(Vertx vertx) + { + return update(b -> b.vertx = vertx); + } + + /** + * Sets the {@code id} and returns a reference to this Builder enabling method chaining. + * + * @param id the {@code id} to set + * @return a reference to this Builder + */ + public Builder id(String id) + { + return update(b -> b.id = id); + } + + /** + * Sets the {@code source} and returns a reference to this Builder enabling method chaining. + * + * @param source the {@code source} to set + * @return a reference to this Builder + */ + public Builder source(String source) + { + return update(b -> b.source = source); + } + + /** + * Sets the {@code port} and returns a reference to this Builder enabling method chaining. + * + * @param port the {@code port} to set + * @return a reference to this Builder + */ + public Builder port(int port) + { + return update(b -> b.port = port); + } + + /** + * Sets the {@code instanceMetadata} and returns a reference to this Builder enabling method chaining. + * + * @param instanceMetadata the {@code instanceMetadata} to set + * @return a reference to this Builder + */ + public Builder instanceMetadata(InstanceMetadata instanceMetadata) + { + return update(b -> b.instanceMetadata = instanceMetadata); + } + + /** + * Sets the {@code executorPools} and returns a reference to this Builder enabling method chaining. + * + * @param executorPools the {@code executorPools} to set + * @return a reference to this Builder + */ + public Builder executorPools(ExecutorPools executorPools) + { + return update(b -> b.executorPools = executorPools); + } + + /** + * Sets the {@code request} and returns a reference to this Builder enabling method chaining. + * + * @param request the {@code request} to set + * @return a reference to this Builder + */ + public Builder request(LiveMigrationFilesVerificationRequest request) + { + return update(b -> b.request = request); + } + + /** + * Sets the {@code sidecarClient} and returns a reference to this Builder enabling method chaining. + * + * @param sidecarClient the {@code sidecarClient} to set + * @return a reference to this Builder + */ + public Builder sidecarClient(SidecarClient sidecarClient) + { + return update(b -> b.sidecarClient = sidecarClient); + } + + /** + * Sets the {@code liveMigrationConfiguration} and returns a reference to this Builder enabling method chaining. + * + * @param liveMigrationConfiguration the {@code liveMigrationConfiguration} to set + * @return a reference to this Builder + */ + public Builder liveMigrationConfiguration(LiveMigrationConfiguration liveMigrationConfiguration) + { + return update(b -> b.liveMigrationConfiguration = liveMigrationConfiguration); + } + + /** + * Sets the {@code digestVerifierFactory} and returns a reference to this Builder enabling method chaining. + * + * @param digestVerifierFactory the {@code digestVerifierFactory} to set + * @return a reference to this Builder + */ + public Builder digestVerifierFactory(DigestVerifierFactory digestVerifierFactory) + { + return update(b -> b.digestVerifierFactory = digestVerifierFactory); + } + + /** + * Returns a {@code LiveMigrationFilesVerificationTask} built from the parameters previously set. + * + * @return a {@code LiveMigrationFilesVerificationTask} built with parameters of this + * {@code LiveMigrationFilesVerificationTask.Builder} + */ + @Override + public LiveMigrationFilesVerificationTask build() + { + Objects.requireNonNull(vertx, "vertx is required"); + Objects.requireNonNull(id, "id is required"); + Objects.requireNonNull(source, "source is required"); + Objects.requireNonNull(instanceMetadata, "instanceMetadata is required"); + Objects.requireNonNull(executorPools, "executorPools is required"); + Objects.requireNonNull(request, "request is required"); + Objects.requireNonNull(sidecarClient, "sidecarClient is required"); + Objects.requireNonNull(liveMigrationConfiguration, "liveMigrationConfiguration is required"); + Objects.requireNonNull(digestVerifierFactory, "digestVerifierFactory is required"); + + return new LiveMigrationFilesVerificationTask(this); + } + } +} diff --git a/server/src/main/java/org/apache/cassandra/sidecar/livemigration/LiveMigrationFilesVerificationTaskFactory.java b/server/src/main/java/org/apache/cassandra/sidecar/livemigration/LiveMigrationFilesVerificationTaskFactory.java new file mode 100644 index 000000000..e89fdfb8f --- /dev/null +++ b/server/src/main/java/org/apache/cassandra/sidecar/livemigration/LiveMigrationFilesVerificationTaskFactory.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.sidecar.livemigration; + +import com.google.inject.Inject; +import com.google.inject.Singleton; +import io.vertx.core.Vertx; +import org.apache.cassandra.sidecar.cluster.instance.InstanceMetadata; +import org.apache.cassandra.sidecar.common.request.LiveMigrationFilesVerificationRequest; +import org.apache.cassandra.sidecar.common.response.LiveMigrationFilesVerificationResponse; +import org.apache.cassandra.sidecar.concurrent.ExecutorPools; +import org.apache.cassandra.sidecar.config.SidecarConfiguration; +import org.apache.cassandra.sidecar.utils.DigestVerifierFactory; +import org.apache.cassandra.sidecar.utils.SidecarClientProvider; + +/** + * Factory for creating {@link LiveMigrationFilesVerificationTask} instances used to verify file digests + * between source and destination instances during live migration operation. + */ +@Singleton +public class LiveMigrationFilesVerificationTaskFactory +{ + private final Vertx vertx; + private final SidecarClientProvider sidecarClientProvider; + private final ExecutorPools executorPools; + private final DigestVerifierFactory digestVerifierFactory; + private final SidecarConfiguration sidecarConfiguration; + + @Inject + public LiveMigrationFilesVerificationTaskFactory(Vertx vertx, + ExecutorPools executorPools, + SidecarConfiguration sidecarConfiguration, + SidecarClientProvider sidecarClientProvider, + DigestVerifierFactory digestVerifierFactory) + { + this.vertx = vertx; + this.sidecarClientProvider = sidecarClientProvider; + this.executorPools = executorPools; + this.digestVerifierFactory = digestVerifierFactory; + this.sidecarConfiguration = sidecarConfiguration; + } + + public LiveMigrationTask create(String id, + String source, + int port, + LiveMigrationFilesVerificationRequest request, + InstanceMetadata localInstanceMetadata) + { + return LiveMigrationFilesVerificationTask.builder() + .id(id) + .source(source) + .port(port) + .vertx(vertx) + .executorPools(executorPools) + .sidecarClient(sidecarClientProvider.get()) + .digestVerifierFactory(digestVerifierFactory) + .liveMigrationConfiguration(sidecarConfiguration.liveMigrationConfiguration()) + .request(request) + .instanceMetadata(localInstanceMetadata) + .build(); + } +} diff --git a/server/src/main/java/org/apache/cassandra/sidecar/livemigration/LiveMigrationInstanceMetadataUtil.java b/server/src/main/java/org/apache/cassandra/sidecar/livemigration/LiveMigrationInstanceMetadataUtil.java index 2d20daba4..382fffdfc 100644 --- a/server/src/main/java/org/apache/cassandra/sidecar/livemigration/LiveMigrationInstanceMetadataUtil.java +++ b/server/src/main/java/org/apache/cassandra/sidecar/livemigration/LiveMigrationInstanceMetadataUtil.java @@ -23,6 +23,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; @@ -33,6 +34,7 @@ import org.apache.cassandra.sidecar.cluster.instance.InstanceMetadata; import org.apache.cassandra.sidecar.common.ApiEndpointsV1; +import org.apache.cassandra.sidecar.handlers.livemigration.LiveMigrationDirType; import org.jetbrains.annotations.NotNull; import static org.apache.cassandra.sidecar.handlers.livemigration.LiveMigrationDirType.CDC_RAW_DIR; @@ -52,149 +54,214 @@ * Utility class for having all {@link InstanceMetadata} related helper functions related to * Live Migration in one place. */ -@SuppressWarnings("ConstantValue") public class LiveMigrationInstanceMetadataUtil { - - public static final String LIVE_MIGRATION_CDC_RAW_DIR_PATH = ApiEndpointsV1.LIVE_MIGRATION_FILES_ROUTE + "/" + CDC_RAW_DIR.dirType; - public static final String LIVE_MIGRATION_COMMITLOG_DIR_PATH = ApiEndpointsV1.LIVE_MIGRATION_FILES_ROUTE + "/" + COMMIT_LOG_DIR.dirType; - public static final String LIVE_MIGRATION_DATA_FILE_DIR_PATH = ApiEndpointsV1.LIVE_MIGRATION_FILES_ROUTE + "/" + DATA_FILE_DIR.dirType; - public static final String LIVE_MIGRATION_HINTS_DIR_PATH = ApiEndpointsV1.LIVE_MIGRATION_FILES_ROUTE + "/" + HINTS_DIR.dirType; - public static final String LIVE_MIGRATION_LOCAL_SYSTEM_DATA_FILE_DIR_PATH = ApiEndpointsV1.LIVE_MIGRATION_FILES_ROUTE - + "/" + LOCAL_SYSTEM_DATA_FILE_DIR.dirType; - public static final String LIVE_MIGRATION_SAVED_CACHES_DIR_PATH = ApiEndpointsV1.LIVE_MIGRATION_FILES_ROUTE + "/" + SAVED_CACHES_DIR.dirType; private static final Logger LOGGER = LoggerFactory.getLogger(LiveMigrationInstanceMetadataUtil.class); /** - * Returns all directories that need to be copied during live migration. - * Includes hints, commit log, saved caches, CDC, local system data, and data directories. - * - * @param instanceMetadata the Cassandra instance metadata containing directory paths - * @return an unmodifiable list of directory paths to be copied during live migration + * Encapsulates all metadata for a specific directory instance in live migration. + * Each descriptor represents one physical directory with its associated index, URL route, and placeholder. + * + *

Example mapping for a data directory at index 0: + *

    + *
  • dirType: DATA_FILE_DIR
    • + *
  • placeholder: "DATA_FILE_DIR"
    • + *
  • localDir: "/var/lib/cassandra/data"
    • + *
  • index: 0
    • + *
  • fileTransferUrl: "/api/v1/live-migration/data/0"
    • + *
+ * + *

The descriptor is used to map between: + *

    + *
  • URL paths (e.g., /api/v1/live-migration/data/0/keyspace/table/file.db)
    • + *
  • Local file paths (e.g., /var/lib/cassandra/data/keyspace/table/file.db)
    • + *
  • Configuration placeholders (e.g., ${DATA_FILE_DIR}/keyspace/table/file.db)
    • + *
*/ - public static List dirsToCopy(InstanceMetadata instanceMetadata) + private static class DirectoryDescriptor { - List dirsToCopy = new ArrayList<>(); - dirsToCopy.add(instanceMetadata.hintsDir()); - dirsToCopy.add(instanceMetadata.commitlogDir()); - if (instanceMetadata.savedCachesDir() != null) - { - dirsToCopy.add(instanceMetadata.savedCachesDir()); - } - if (instanceMetadata.cdcDir() != null) - { - dirsToCopy.add(instanceMetadata.cdcDir()); - } - if (instanceMetadata.localSystemDataFileDir() != null) - { - dirsToCopy.add(instanceMetadata.localSystemDataFileDir()); - } - dirsToCopy.addAll(instanceMetadata.dataDirs()); - return Collections.unmodifiableList(dirsToCopy); - } + /** The type of directory (e.g., DATA_FILE_DIR, HINTS_DIR, COMMIT_LOG_DIR) */ + final LiveMigrationDirType dirType; - /** - * Returns map of directory that can be copied and the index to use while constructing - * url for file transfer. - */ - public static Map dirPathPrefixMap(InstanceMetadata instanceMetadata) - { - Map dirIndexMap = new HashMap<>(); - dirIndexMap.put(instanceMetadata.hintsDir(), LIVE_MIGRATION_HINTS_DIR_PATH + "/0"); - dirIndexMap.put(instanceMetadata.commitlogDir(), LIVE_MIGRATION_COMMITLOG_DIR_PATH + "/0"); - if (instanceMetadata.savedCachesDir() != null) - { - dirIndexMap.put(instanceMetadata.savedCachesDir(), LIVE_MIGRATION_SAVED_CACHES_DIR_PATH + "/0"); - } - if (instanceMetadata.cdcDir() != null) - { - dirIndexMap.put(instanceMetadata.cdcDir(), LIVE_MIGRATION_CDC_RAW_DIR_PATH + "/0"); - } - if (instanceMetadata.localSystemDataFileDir() != null) + /** The placeholder name used in configuration patterns (e.g., "DATA_FILE_DIR") */ + final String placeholder; + + /** The absolute path to the local directory on the file system */ + final String localDir; + + /** The index of this directory within directories of the same type (0-based) */ + final int index; + + /** The URL prefix for file transfer operations (e.g., "/api/v1/live-migration/data/0") */ + final String fileTransferUrl; + + /** + * Creates a directory descriptor with the specified metadata. + * + * @param dirType the type of directory + * @param placeholder the placeholder name for configuration patterns + * @param localDir the absolute path to the local directory + * @param index the index of this directory (0 for single directories, 0-N for data directories) + */ + DirectoryDescriptor(LiveMigrationDirType dirType, + String placeholder, + String localDir, + int index) { - dirIndexMap.put(instanceMetadata.localSystemDataFileDir(), LIVE_MIGRATION_LOCAL_SYSTEM_DATA_FILE_DIR_PATH + "/0"); + this.dirType = dirType; + this.placeholder = placeholder; + this.localDir = localDir; + this.index = index; + + String urlBase = ApiEndpointsV1.LIVE_MIGRATION_FILES_ROUTE + "/" + dirType.dirType; + this.fileTransferUrl = urlBase + "/" + index; } - for (int i = 0; i < instanceMetadata.dataDirs().size(); i++) + + /** + * Returns the set of placeholders that can be used to reference this directory. + * + *

Most directory types return a single placeholder (e.g., "HINTS_DIR"). + * DATA_FILE_DIR returns both a generic placeholder and an indexed one to support + * multiple data directories: + *

    + *
  • Generic: "DATA_FILE_DIR" - resolves to all data directories
    • + *
  • Indexed: "DATA_FILE_DIR_0", "DATA_FILE_DIR_1", etc. - resolves to specific data directory
    • + *
+ * + * @return immutable set of placeholder strings for this directory + */ + Set getPlaceholders() { - dirIndexMap.put(instanceMetadata.dataDirs().get(i), LIVE_MIGRATION_DATA_FILE_DIR_PATH + "/" + i); + if (dirType == DATA_FILE_DIR) + { + return Set.of(placeholder, placeholder + "_" + index); + } + return Set.of(placeholder); } - return Collections.unmodifiableMap(dirIndexMap); } /** - * Returns map of directory and set of placeholders that represent the directory. + * Builds all directory descriptors for the given instance metadata. + * Each descriptor represents one specific directory with its index. + * This is the single source of truth for all directory-to-URL mappings. + * + * @param instanceMetadata the Cassandra instance metadata + * @return list of directory descriptors for all directories in the instance */ - public static Map> dirPlaceHoldersMap(InstanceMetadata instanceMetadata) + @SuppressWarnings("ConstantValue") + private static List buildDescriptors(InstanceMetadata instanceMetadata) { - Map> placeholderMap = new HashMap<>(); + List descriptors = new ArrayList<>(); + + // Hints directory - always has index 0 + descriptors.add(new DirectoryDescriptor( + HINTS_DIR, HINTS_DIR_PLACEHOLDER, instanceMetadata.hintsDir(), 0)); - placeholderMap.put(instanceMetadata.hintsDir(), Collections.singleton(HINTS_DIR_PLACEHOLDER)); - placeholderMap.put(instanceMetadata.commitlogDir(), Collections.singleton(COMMITLOG_DIR_PLACEHOLDER)); + // Commit log directory - always has index 0 + descriptors.add(new DirectoryDescriptor( + COMMIT_LOG_DIR, COMMITLOG_DIR_PLACEHOLDER, instanceMetadata.commitlogDir(), 0)); + + // Saved caches directory - always has index 0 if (instanceMetadata.savedCachesDir() != null) { - placeholderMap.put(instanceMetadata.savedCachesDir(), Collections.singleton(SAVED_CACHES_DIR_PLACEHOLDER)); + descriptors.add(new DirectoryDescriptor( + SAVED_CACHES_DIR, SAVED_CACHES_DIR_PLACEHOLDER, instanceMetadata.savedCachesDir(), 0)); } + // CDC directory - always has index 0 if (instanceMetadata.cdcDir() != null) { - placeholderMap.put(instanceMetadata.cdcDir(), Collections.singleton(CDC_RAW_DIR_PLACEHOLDER)); + descriptors.add(new DirectoryDescriptor( + CDC_RAW_DIR, CDC_RAW_DIR_PLACEHOLDER, instanceMetadata.cdcDir(), 0)); } + // Local system data directory - always has index 0 if (instanceMetadata.localSystemDataFileDir() != null) { - placeholderMap.put(instanceMetadata.localSystemDataFileDir(), - Collections.singleton(LOCAL_SYSTEM_DATA_FILE_DIR_PLACEHOLDER)); + descriptors.add(new DirectoryDescriptor( + LOCAL_SYSTEM_DATA_FILE_DIR, LOCAL_SYSTEM_DATA_FILE_DIR_PLACEHOLDER, + instanceMetadata.localSystemDataFileDir(), 0)); } + // Data directories - each gets its own index List dataDirs = instanceMetadata.dataDirs(); for (int i = 0; i < dataDirs.size(); i++) { - String dir = dataDirs.get(i); - Set placeholders = Set.of(DATA_FILE_DIR_PLACEHOLDER, DATA_FILE_DIR_PLACEHOLDER + "_" + i); - placeholderMap.put(dir, placeholders); + descriptors.add(new DirectoryDescriptor( + DATA_FILE_DIR, DATA_FILE_DIR_PLACEHOLDER, dataDirs.get(i), i)); } - return Collections.unmodifiableMap(placeholderMap); + return descriptors; } /** - * Returns a map of placeholder and its directories based on given {@link InstanceMetadata}. + * Returns all directories that need to be copied during live migration. + * Includes hints, commit log, saved caches, CDC, local system data, and data directories. + * + * @param instanceMetadata the Cassandra instance metadata containing directory paths + * @return an unmodifiable list of directory paths to be copied during live migration */ - public static Map> placeholderDirsMap(InstanceMetadata instanceMetadata) + public static List dirsToCopy(InstanceMetadata instanceMetadata) { - Map> placeholderDirsMap = new HashMap<>(); - - placeholderDirsMap.put(HINTS_DIR_PLACEHOLDER, Collections.singleton(instanceMetadata.hintsDir())); - placeholderDirsMap.put(COMMITLOG_DIR_PLACEHOLDER, Collections.singleton(instanceMetadata.commitlogDir())); - - if (instanceMetadata.savedCachesDir() != null) + List dirsToCopy = new ArrayList<>(); + for (DirectoryDescriptor desc : buildDescriptors(instanceMetadata)) { - placeholderDirsMap.put(SAVED_CACHES_DIR_PLACEHOLDER, Collections.singleton(instanceMetadata.savedCachesDir())); + dirsToCopy.add(desc.localDir); } + return Collections.unmodifiableList(dirsToCopy); + } - if (instanceMetadata.cdcDir() != null) + /** + * Returns a map of local directory paths to their corresponding file transfer URL prefixes. + *

+ * Example: For a data directory "/var/lib/cassandra/data", the map contains: + * "/var/lib/cassandra/data" -> "/api/v1/live-migration/data/0" + * + * @param instanceMetadata the Cassandra instance metadata containing directory paths + * @return unmodifiable map from local directory path to URL prefix + */ + public static Map dirPathPrefixMap(InstanceMetadata instanceMetadata) + { + Map dirIndexMap = new HashMap<>(); + for (DirectoryDescriptor desc : buildDescriptors(instanceMetadata)) { - placeholderDirsMap.put(CDC_RAW_DIR_PLACEHOLDER, Collections.singleton(instanceMetadata.cdcDir())); + dirIndexMap.put(desc.localDir, desc.fileTransferUrl); } + return Collections.unmodifiableMap(dirIndexMap); + } - if (instanceMetadata.localSystemDataFileDir() != null) + /** + * Returns map of directory and set of placeholders that represent the directory. + */ + public static Map> dirPlaceHoldersMap(InstanceMetadata instanceMetadata) + { + Map> placeholderMap = new HashMap<>(); + for (DirectoryDescriptor desc : buildDescriptors(instanceMetadata)) { - placeholderDirsMap.put(LOCAL_SYSTEM_DATA_FILE_DIR_PLACEHOLDER, - Collections.singleton(instanceMetadata.localSystemDataFileDir())); + placeholderMap.put(desc.localDir, desc.getPlaceholders()); } + return Collections.unmodifiableMap(placeholderMap); + } - placeholderDirsMap.put(DATA_FILE_DIR_PLACEHOLDER, Set.copyOf(instanceMetadata.dataDirs())); + /** + * Returns a map of placeholder and its directories based on given {@link InstanceMetadata}. + */ + public static Map> placeholderDirsMap(InstanceMetadata instanceMetadata) + { + Map> placeholderDirsMap = new HashMap<>(); - List dataDirs = instanceMetadata.dataDirs(); - for (int i = 0; i < dataDirs.size(); i++) + for (DirectoryDescriptor desc : buildDescriptors(instanceMetadata)) { - placeholderDirsMap.put(DATA_FILE_DIR_PLACEHOLDER + "_" + i, Collections.singleton(dataDirs.get(i))); + for (String placeholder : desc.getPlaceholders()) + { + placeholderDirsMap.computeIfAbsent(placeholder, k -> new HashSet<>()) + .add(desc.localDir); + } } - return placeholderDirsMap; + return Collections.unmodifiableMap(placeholderDirsMap); } - /** * Converts given live migration file download URL to local path. * @@ -205,6 +272,8 @@ public static Map> placeholderDirsMap(InstanceMetadata insta public static Path localPath(@NotNull String fileUrl, @NotNull InstanceMetadata metadata) { + Objects.requireNonNull(fileUrl, "fileUrl cannot be null"); + Objects.requireNonNull(metadata, "metadata cannot be null"); if (fileUrl.contains("/../") || fileUrl.endsWith("/..")) { @@ -220,33 +289,33 @@ public static Path localPath(@NotNull String fileUrl, { Objects.requireNonNull(entry.getValue(), () -> "No local path found for url " + fileUrl); String relativePath = fileUrl.substring(entry.getKey().length()); - return Paths.get(entry.getValue(), relativePath).toAbsolutePath(); + Path baseDir = Paths.get(entry.getValue()).toAbsolutePath().normalize(); + Path resolvedPath = Paths.get(entry.getValue(), relativePath).toAbsolutePath().normalize(); + + if (!resolvedPath.startsWith(baseDir)) + { + String errorMessage = "Resolved path escapes base directory for url " + fileUrl; + LOGGER.error(errorMessage); + throw new IllegalArgumentException(errorMessage); + } + + return resolvedPath; } } throw new IllegalArgumentException("File url " + fileUrl + " is unknown."); } + private static Map migrationUrlLocalDirMap(InstanceMetadata instanceMetadata) { Map urlToLocalDirMap = new HashMap<>(); - urlToLocalDirMap.put(LIVE_MIGRATION_COMMITLOG_DIR_PATH + "/0/", instanceMetadata.commitlogDir()); - urlToLocalDirMap.put(LIVE_MIGRATION_HINTS_DIR_PATH + "/0/", instanceMetadata.hintsDir()); - urlToLocalDirMap.put(LIVE_MIGRATION_SAVED_CACHES_DIR_PATH + "/0/", instanceMetadata.savedCachesDir()); - - if (instanceMetadata.cdcDir() != null) - { - urlToLocalDirMap.put(LIVE_MIGRATION_CDC_RAW_DIR_PATH + "/0/", instanceMetadata.cdcDir()); - } - if (instanceMetadata.localSystemDataFileDir() != null) - { - urlToLocalDirMap.put(LIVE_MIGRATION_LOCAL_SYSTEM_DATA_FILE_DIR_PATH + "/0/", instanceMetadata.localSystemDataFileDir()); - } - for (int i = 0; i < instanceMetadata.dataDirs().size(); i++) + for (DirectoryDescriptor desc : buildDescriptors(instanceMetadata)) { - urlToLocalDirMap.put(LIVE_MIGRATION_DATA_FILE_DIR_PATH + "/" + i + "/", instanceMetadata.dataDirs().get(i)); + // Add file transfer URL mapping + urlToLocalDirMap.put(desc.fileTransferUrl + "/", desc.localDir); } - return urlToLocalDirMap; + return Collections.unmodifiableMap(urlToLocalDirMap); } } diff --git a/server/src/main/java/org/apache/cassandra/sidecar/livemigration/LiveMigrationTask.java b/server/src/main/java/org/apache/cassandra/sidecar/livemigration/LiveMigrationTask.java index 796504104..ceb7d817e 100644 --- a/server/src/main/java/org/apache/cassandra/sidecar/livemigration/LiveMigrationTask.java +++ b/server/src/main/java/org/apache/cassandra/sidecar/livemigration/LiveMigrationTask.java @@ -18,18 +18,13 @@ package org.apache.cassandra.sidecar.livemigration; -import org.apache.cassandra.sidecar.common.request.LiveMigrationDataCopyRequest; -import org.apache.cassandra.sidecar.common.response.LiveMigrationTaskResponse; - /** - * Represents a live migration data copy task. During live migration, the source Cassandra instance - * may continue running while the destination copies files, meaning new SSTables can be created and - * existing ones may disappear due to compaction. To handle these changes, the task performs multiple - * iterations to synchronize data until the success threshold criteria in - * {@link LiveMigrationDataCopyRequest#successThreshold} is met, up to a maximum number of iterations - * specified by {@link LiveMigrationDataCopyRequest#maxIterations}. + * Represents a live migration task. This interface provides lifecycle management + * for asynchronous live migration operations. + * + * @param the type of response returned by this task */ -public interface LiveMigrationTask +public interface LiveMigrationTask { /** * ID of live migration task. @@ -39,7 +34,12 @@ public interface LiveMigrationTask String id(); /** - * Starts live migration. + * Type of task + */ + String type(); + + /** + * Starts the live migration task. */ void start(); @@ -49,15 +49,17 @@ public interface LiveMigrationTask * * @return current live migration task's response */ - LiveMigrationTaskResponse getResponse(); + T getResponse(); /** * Cancels the current live migration task if not finished already. + * + *

Note: Cancellation is best-effort; ongoing operations may not stop immediately. */ void cancel(); /** - * Tells whether current live migration has completed or not. + * Tells whether current live migration task has completed or not. * * @return true if completed otherwise false. */ diff --git a/server/src/main/java/org/apache/cassandra/sidecar/livemigration/LiveMigrationTaskFactory.java b/server/src/main/java/org/apache/cassandra/sidecar/livemigration/LiveMigrationTaskFactory.java index fd698ced3..52371d18b 100644 --- a/server/src/main/java/org/apache/cassandra/sidecar/livemigration/LiveMigrationTaskFactory.java +++ b/server/src/main/java/org/apache/cassandra/sidecar/livemigration/LiveMigrationTaskFactory.java @@ -20,6 +20,7 @@ import org.apache.cassandra.sidecar.cluster.instance.InstanceMetadata; import org.apache.cassandra.sidecar.common.request.LiveMigrationDataCopyRequest; +import org.apache.cassandra.sidecar.common.response.LiveMigrationDataCopyResponse; /** @@ -41,9 +42,9 @@ public interface LiveMigrationTaskFactory * @param instanceMetadata Instance metadata for which data copy should be initiated. * @return Live migration task */ - LiveMigrationTask create(String id, - LiveMigrationDataCopyRequest request, - String source, - int port, - InstanceMetadata instanceMetadata); + LiveMigrationTask create(String id, + LiveMigrationDataCopyRequest request, + String source, + int port, + InstanceMetadata instanceMetadata); } diff --git a/server/src/main/java/org/apache/cassandra/sidecar/livemigration/LiveMigrationTaskManager.java b/server/src/main/java/org/apache/cassandra/sidecar/livemigration/LiveMigrationTaskManager.java new file mode 100644 index 000000000..c92f7360c --- /dev/null +++ b/server/src/main/java/org/apache/cassandra/sidecar/livemigration/LiveMigrationTaskManager.java @@ -0,0 +1,152 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.sidecar.livemigration; + +import java.util.Collections; +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; + +import com.google.common.annotations.VisibleForTesting; + +import com.google.inject.Inject; +import com.google.inject.Singleton; +import org.apache.cassandra.sidecar.cluster.InstancesMetadata; +import org.apache.cassandra.sidecar.cluster.instance.InstanceMetadata; +import org.apache.cassandra.sidecar.exceptions.LiveMigrationExceptions; +import org.jetbrains.annotations.NotNull; + +/** + * Centralized singleton manager for all live migration tasks across Cassandra instances. + * Enforces the constraint that only one {@link LiveMigrationTask} (of any type) can be active per instance at a time, + * preventing concurrent data copy and files verification operations that could lead to resource conflicts or data integrity issues. + * This singleton is shared across {@link DataCopyTaskManager} and {@link FilesVerificationTaskManager} to ensure + * proper coordination and mutual exclusion of tasks per instance. + */ +@Singleton +public class LiveMigrationTaskManager +{ + @VisibleForTesting + final ConcurrentHashMap> currentTasks = new ConcurrentHashMap<>(); + + private final InstancesMetadata instancesMetadata; + + @Inject + public LiveMigrationTaskManager(InstancesMetadata instancesMetadata) + { + this.instancesMetadata = instancesMetadata; + } + + /** + * Attempts to submit a new task for the specified instance. + * Only one task (of any type) can be active per instance at a time. + * + * @param instanceId the instance ID + * @param newTask the task to submit + * @return true if the task was accepted, false if another task is already in progress + */ + public boolean submitTask(int instanceId, LiveMigrationTask newTask) + { + return currentTasks.compute(instanceId, (integer, taskInMap) -> { + if (taskInMap == null) + { + return newTask; + } + + if (!taskInMap.isCompleted()) + { + // Reject new task if existing task is still in progress + return taskInMap; + } + else + { + // Accept new task if existing task has completed + return newTask; + } + }) == newTask; + } + + /** + * Returns all live migration tasks for given currentHost. + * This includes both active and completed tasks that haven't been replaced. + * + * @param currentHost the host where sidecar is running + * @return list containing at most one task (empty if no task has ever been submitted for this host) + */ + @SuppressWarnings("ConstantValue") + public List> getAllTasks(@NotNull String currentHost) + { + InstanceMetadata localInstance = instancesMetadata.instanceFromHost(currentHost); + if (localInstance == null) + { + throw new IllegalStateException("No instance found for host: " + currentHost); + } + + LiveMigrationTask task = currentTasks.get(localInstance.id()); + return task == null ? Collections.emptyList() : Collections.singletonList(task); + } + + /** + * Returns the live migration task with the specified task ID. + * + * @param taskId ID of the task to retrieve + * @param currentHost the host where sidecar is running + * @return the LiveMigrationTask matching the given taskId + * @throws LiveMigrationExceptions.LiveMigrationTaskNotFoundException if no task found with the given ID + */ + public LiveMigrationTask getTask(@NotNull String taskId, + @NotNull String currentHost) throws LiveMigrationExceptions.LiveMigrationTaskNotFoundException + { + return getLiveMigrationTask(taskId, currentHost); + } + + /** + * Cancels the live migration task with the specified task ID. + * + * @param taskId ID of the task to cancel + * @param currentHost the host where sidecar is running + * @return the cancelled LiveMigrationTask + * @throws LiveMigrationExceptions.LiveMigrationTaskNotFoundException if no task found with the given ID + */ + public LiveMigrationTask cancelTask(@NotNull String taskId, + @NotNull String currentHost) throws LiveMigrationExceptions.LiveMigrationTaskNotFoundException + { + LiveMigrationTask taskInProgress = getLiveMigrationTask(taskId, currentHost); + + // Cancelling the task + taskInProgress.cancel(); + + return taskInProgress; + } + + @SuppressWarnings("ConstantValue") + private LiveMigrationTask getLiveMigrationTask(@NotNull String taskId, @NotNull String currentHost) + { + InstanceMetadata localInstance = instancesMetadata.instanceFromHost(currentHost); + if (localInstance == null) + { + throw new IllegalStateException("No instance found for host: " + currentHost); + } + LiveMigrationTask taskInProgress = currentTasks.get(localInstance.id()); + if (taskInProgress == null || !taskId.equals(taskInProgress.id())) + { + throw new LiveMigrationExceptions.LiveMigrationTaskNotFoundException("No task found with given id " + taskId); + } + return taskInProgress; + } +} diff --git a/server/src/main/java/org/apache/cassandra/sidecar/modules/LiveMigrationModule.java b/server/src/main/java/org/apache/cassandra/sidecar/modules/LiveMigrationModule.java index 4c10c5180..48ea12cde 100644 --- a/server/src/main/java/org/apache/cassandra/sidecar/modules/LiveMigrationModule.java +++ b/server/src/main/java/org/apache/cassandra/sidecar/modules/LiveMigrationModule.java @@ -26,27 +26,37 @@ import jakarta.ws.rs.POST; import jakarta.ws.rs.Path; import org.apache.cassandra.sidecar.common.ApiEndpointsV1; +import org.apache.cassandra.sidecar.common.response.DigestResponse; import org.apache.cassandra.sidecar.common.response.InstanceFilesListResponse; +import org.apache.cassandra.sidecar.common.response.LiveMigrationDataCopyResponse; +import org.apache.cassandra.sidecar.common.response.LiveMigrationFilesVerificationResponse; import org.apache.cassandra.sidecar.common.response.LiveMigrationStatus; -import org.apache.cassandra.sidecar.common.response.LiveMigrationTaskResponse; +import org.apache.cassandra.sidecar.common.response.LiveMigrationTaskCreationResponse; import org.apache.cassandra.sidecar.handlers.FileStreamHandler; import org.apache.cassandra.sidecar.handlers.livemigration.LiveMigrationApiEnableDisableHandler; import org.apache.cassandra.sidecar.handlers.livemigration.LiveMigrationCancelDataCopyTaskHandler; +import org.apache.cassandra.sidecar.handlers.livemigration.LiveMigrationCancelFilesVerificationTaskHandler; +import org.apache.cassandra.sidecar.handlers.livemigration.LiveMigrationConcurrencyLimitHandler; import org.apache.cassandra.sidecar.handlers.livemigration.LiveMigrationCreateDataCopyTaskHandler; -import org.apache.cassandra.sidecar.handlers.livemigration.LiveMigrationFileStreamHandler; +import org.apache.cassandra.sidecar.handlers.livemigration.LiveMigrationCreateFilesVerificationTaskHandler; +import org.apache.cassandra.sidecar.handlers.livemigration.LiveMigrationDigestHandlerWrapper; +import org.apache.cassandra.sidecar.handlers.livemigration.LiveMigrationFileResolveHandler; import org.apache.cassandra.sidecar.handlers.livemigration.LiveMigrationGetAllDataCopyTasksHandler; +import org.apache.cassandra.sidecar.handlers.livemigration.LiveMigrationGetAllFilesVerificationTasksHandler; import org.apache.cassandra.sidecar.handlers.livemigration.LiveMigrationGetDataCopyTaskHandler; +import org.apache.cassandra.sidecar.handlers.livemigration.LiveMigrationGetFilesVerificationTaskHandler; import org.apache.cassandra.sidecar.handlers.livemigration.LiveMigrationListInstanceFilesHandler; import org.apache.cassandra.sidecar.handlers.livemigration.LiveMigrationMap; import org.apache.cassandra.sidecar.handlers.livemigration.LiveMigrationMapSidecarConfigImpl; import org.apache.cassandra.sidecar.handlers.livemigration.LiveMigrationStatusClearHandler; import org.apache.cassandra.sidecar.handlers.livemigration.LiveMigrationStatusCompleteHandler; import org.apache.cassandra.sidecar.handlers.livemigration.LiveMigrationStatusGetHandler; +import org.apache.cassandra.sidecar.livemigration.LiveMigrationDataCopyTaskFactoryImpl; import org.apache.cassandra.sidecar.livemigration.LiveMigrationFileDownloadPreCheck; +import org.apache.cassandra.sidecar.livemigration.LiveMigrationFilesVerificationTaskFactory; import org.apache.cassandra.sidecar.livemigration.LiveMigrationStatusTracker; import org.apache.cassandra.sidecar.livemigration.LiveMigrationStatusTrackerImpl; import org.apache.cassandra.sidecar.livemigration.LiveMigrationTaskFactory; -import org.apache.cassandra.sidecar.livemigration.LiveMigrationTaskFactoryImpl; import org.apache.cassandra.sidecar.modules.multibindings.KeyClassMapKey; import org.apache.cassandra.sidecar.modules.multibindings.VertxRouteMapKeys; import org.apache.cassandra.sidecar.routes.RouteBuilder; @@ -68,28 +78,33 @@ public class LiveMigrationModule extends AbstractModule protected void configure() { bind(LiveMigrationMap.class).to(LiveMigrationMapSidecarConfigImpl.class); - bind(LiveMigrationTaskFactory.class).to(LiveMigrationTaskFactoryImpl.class); + bind(LiveMigrationTaskFactory.class).to(LiveMigrationDataCopyTaskFactoryImpl.class); + bind(LiveMigrationFilesVerificationTaskFactory.class); bind(LiveMigrationStatusTracker.class).to(LiveMigrationStatusTrackerImpl.class); bind(LiveMigrationFileDownloadPreCheck.class).toInstance(LiveMigrationFileDownloadPreCheck.DEFAULT); } - @GET + @POST @Path(ApiEndpointsV1.LIVE_MIGRATION_DATA_COPY_TASKS_ROUTE) @Operation(summary = "Create data copy task", - description = "Creates a new data copy task for live migration") + description = "Creates a new data copy task for live migration") @APIResponse(description = "Data copy task created successfully", - responseCode = "202", - content = @Content(mediaType = "application/json", - schema = @Schema(type = SchemaType.OBJECT))) - @APIResponse(responseCode = "403", - description = "Live migration not enabled or node not configured as destination", - content = @Content(mediaType = "application/json", - schema = @Schema(type = SchemaType.OBJECT))) + responseCode = "202", + content = @Content(mediaType = "application/json", + schema = @Schema(type = SchemaType.OBJECT))) + @APIResponse(responseCode = "404", + description = "Live migration not enabled or node not configured as destination", + content = @Content(mediaType = "application/json", + schema = @Schema(type = SchemaType.OBJECT))) + @APIResponse(responseCode = "409", + description = "Cannot accept data copy task as another task is in progress", + content = @Content(mediaType = "application/json", + schema = @Schema(type = SchemaType.OBJECT))) @ProvidesIntoMap @KeyClassMapKey(VertxRouteMapKeys.LiveMigrationCreateDataCopyTaskRouteKey.class) - public VertxRoute createDataCopyTaskRoute(RouteBuilder.Factory factory, - LiveMigrationApiEnableDisableHandler liveMigrationApiEnableDisableHandler, - LiveMigrationCreateDataCopyTaskHandler liveMigrationCreateDataCopyTaskHandler) + VertxRoute createDataCopyTaskRoute(RouteBuilder.Factory factory, + LiveMigrationApiEnableDisableHandler liveMigrationApiEnableDisableHandler, + LiveMigrationCreateDataCopyTaskHandler liveMigrationCreateDataCopyTaskHandler) { return factory.builderForRoute() .setBodyHandler(true) @@ -103,22 +118,18 @@ public VertxRoute createDataCopyTaskRoute(RouteBuilder.Factory factory, @Operation(summary = "Cancel data copy task", description = "Cancels an existing data copy task for live migration") @APIResponse(description = "Data copy task cancelled successfully", - responseCode = "200", - content = @Content(mediaType = "application/json", - schema = @Schema(implementation = LiveMigrationTaskResponse.class))) - @APIResponse(responseCode = "403", - description = "Live migration not enabled or node not configured as destination", - content = @Content(mediaType = "application/json", - schema = @Schema(type = SchemaType.OBJECT))) + responseCode = "200", + content = @Content(mediaType = "application/json", + schema = @Schema(implementation = LiveMigrationDataCopyResponse.class))) @APIResponse(responseCode = "404", - description = "Data copy task not found", - content = @Content(mediaType = "application/json", - schema = @Schema(type = SchemaType.OBJECT))) + description = "Live migration not enabled, node not configured as destination, or data copy task not found", + content = @Content(mediaType = "application/json", + schema = @Schema(type = SchemaType.OBJECT))) @ProvidesIntoMap @KeyClassMapKey(VertxRouteMapKeys.LiveMigrationCancelDataCopyTaskRouteKey.class) - public VertxRoute cancelDataCopyTaskRoute(RouteBuilder.Factory factory, - LiveMigrationApiEnableDisableHandler liveMigrationApiEnableDisableHandler, - LiveMigrationCancelDataCopyTaskHandler liveMigrationCancelDataCopyTaskHandler) + VertxRoute cancelDataCopyTaskRoute(RouteBuilder.Factory factory, + LiveMigrationApiEnableDisableHandler liveMigrationApiEnableDisableHandler, + LiveMigrationCancelDataCopyTaskHandler liveMigrationCancelDataCopyTaskHandler) { return factory.builderForRoute() .handler(liveMigrationApiEnableDisableHandler::isDestination) @@ -128,24 +139,20 @@ public VertxRoute cancelDataCopyTaskRoute(RouteBuilder.Factory factory, @GET @Operation(summary = "Get data copy task", - description = "Retrieves the status and details of a specific data copy task by task ID") + description = "Retrieves the status and details of a specific data copy task by task ID") @APIResponse(description = "Data copy task retrieved successfully", - responseCode = "200", - content = @Content(mediaType = "application/json", - schema = @Schema(implementation = LiveMigrationTaskResponse.class))) - @APIResponse(responseCode = "403", - description = "Live migration not enabled or node not configured as destination", - content = @Content(mediaType = "application/json", - schema = @Schema(type = SchemaType.OBJECT))) + responseCode = "200", + content = @Content(mediaType = "application/json", + schema = @Schema(implementation = LiveMigrationDataCopyResponse.class))) @APIResponse(responseCode = "404", - description = "Data copy task not found", - content = @Content(mediaType = "application/json", - schema = @Schema(type = SchemaType.OBJECT))) + description = "Live migration not enabled, node not configured as destination, or data copy task not found", + content = @Content(mediaType = "application/json", + schema = @Schema(type = SchemaType.OBJECT))) @ProvidesIntoMap @KeyClassMapKey(VertxRouteMapKeys.LiveMigrationGetDataCopyTaskRouteKey.class) - public VertxRoute getDataCopyTaskRoute(RouteBuilder.Factory factory, - LiveMigrationApiEnableDisableHandler liveMigrationApiEnableDisableHandler, - LiveMigrationGetDataCopyTaskHandler liveMigrationGetDataCopyTaskHandler) + VertxRoute getDataCopyTaskRoute(RouteBuilder.Factory factory, + LiveMigrationApiEnableDisableHandler liveMigrationApiEnableDisableHandler, + LiveMigrationGetDataCopyTaskHandler liveMigrationGetDataCopyTaskHandler) { return factory.builderForRoute() .handler(liveMigrationApiEnableDisableHandler::isDestination) @@ -157,18 +164,18 @@ public VertxRoute getDataCopyTaskRoute(RouteBuilder.Factory factory, @Operation(summary = "Get all data copy tasks", description = "Retrieves all data copy tasks for live migration on the current node") @APIResponse(description = "Data copy tasks retrieved successfully", - responseCode = "200", - content = @Content(mediaType = "application/json", - schema = @Schema(type = SchemaType.ARRAY))) - @APIResponse(responseCode = "403", - description = "Live migration not enabled or node not configured as destination", - content = @Content(mediaType = "application/json", - schema = @Schema(type = SchemaType.OBJECT))) + responseCode = "200", + content = @Content(mediaType = "application/json", + schema = @Schema(type = SchemaType.ARRAY))) + @APIResponse(responseCode = "404", + description = "Live migration not enabled or node not configured as destination", + content = @Content(mediaType = "application/json", + schema = @Schema(type = SchemaType.OBJECT))) @ProvidesIntoMap @KeyClassMapKey(VertxRouteMapKeys.LiveMigrationGetAllDataCopyTasksRouteKey.class) - public VertxRoute getAllDataCopyTasksRoute(RouteBuilder.Factory factory, - LiveMigrationApiEnableDisableHandler liveMigrationApiEnableDisableHandler, - LiveMigrationGetAllDataCopyTasksHandler liveMigrationGetAllDataCopyTasksHandler) + VertxRoute getAllDataCopyTasksRoute(RouteBuilder.Factory factory, + LiveMigrationApiEnableDisableHandler liveMigrationApiEnableDisableHandler, + LiveMigrationGetAllDataCopyTasksHandler liveMigrationGetAllDataCopyTasksHandler) { return factory.builderForRoute() .handler(liveMigrationApiEnableDisableHandler::isDestination) @@ -179,26 +186,43 @@ public VertxRoute getAllDataCopyTasksRoute(RouteBuilder.Factory factory, @GET @Path(ApiEndpointsV1.LIVE_MIGRATION_FILE_TRANSFER_ROUTE) @Operation(summary = "Stream file for live migration", - description = "Streams a file for live migration data transfer") - @APIResponse(description = "File stream for live migration initiated successfully", - responseCode = "200", - content = @Content(mediaType = "application/octet-stream", - schema = @Schema(type = SchemaType.STRING))) - @APIResponse(responseCode = "403", - description = "Live migration not enabled or file access denied", - content = @Content(mediaType = "application/json", - schema = @Schema(type = SchemaType.OBJECT))) + description = "Streams a file for live migration data transfer. " + + "Optionally returns file digest when digestAlgorithm query parameter is provided") + @APIResponse(description = "File stream for live migration initiated successfully (when digestAlgorithm param is absent)", + responseCode = "200", + content = @Content(mediaType = "application/octet-stream", + schema = @Schema(type = SchemaType.STRING))) + @APIResponse(description = "File digest calculated successfully (when digestAlgorithm param is present)", + responseCode = "200", + content = @Content(mediaType = "application/json", + schema = @Schema(implementation = DigestResponse.class))) + @APIResponse(responseCode = "404", + description = "Live migration not enabled or node not configured as source", + content = @Content(mediaType = "application/json", + schema = @Schema(type = SchemaType.OBJECT))) + @APIResponse(responseCode = "503", + description = "Concurrency limit reached for file requests", + content = @Content(mediaType = "application/json", + schema = @Schema(type = SchemaType.OBJECT))) + @APIResponse(responseCode = "500", + description = "Failed to calculate digest", + content = @Content(mediaType = "application/json", + schema = @Schema(type = SchemaType.OBJECT))) @ProvidesIntoMap @KeyClassMapKey(VertxRouteMapKeys.LiveMigrationFileStreamHandlerRouteKey.class) - public VertxRoute downloadFileRoute(RouteBuilder.Factory factory, - LiveMigrationApiEnableDisableHandler liveMigrationApiEnableDisableHandler, - LiveMigrationFileStreamHandler liveMigrationFileStreamHandler, - FileStreamHandler fileStreamHandler) + VertxRoute liveMigrationFileRoute(RouteBuilder.Factory factory, + LiveMigrationApiEnableDisableHandler liveMigrationApiEnableDisableHandler, + LiveMigrationConcurrencyLimitHandler concurrencyLimitHandler, + LiveMigrationFileResolveHandler liveMigrationFileResolveHandler, + FileStreamHandler fileStreamHandler, + LiveMigrationDigestHandlerWrapper liveMigrationDigestHandlerWrapper) { return factory.builderForRoute() .handler(liveMigrationApiEnableDisableHandler::isSource) .handler(liveMigrationApiEnableDisableHandler::allowIfMigrationNotComplete) - .handler(liveMigrationFileStreamHandler) + .handler(concurrencyLimitHandler) + .handler(liveMigrationFileResolveHandler) + .handler(liveMigrationDigestHandlerWrapper) .handler(fileStreamHandler) .build(); } @@ -206,15 +230,15 @@ public VertxRoute downloadFileRoute(RouteBuilder.Factory factory, @GET @Path(ApiEndpointsV1.LIVE_MIGRATION_FILES_ROUTE) @Operation(summary = "List instance files", - description = "Lists files available on an instance for live migration purposes") + description = "Lists files available on an instance for live migration purposes") @APIResponse(description = "Instance files listed successfully", - responseCode = "200", - content = @Content(mediaType = "application/json", - schema = @Schema(implementation = InstanceFilesListResponse.class))) - @APIResponse(responseCode = "403", - description = "Live migration not enabled or node not configured for migration", - content = @Content(mediaType = "application/json", - schema = @Schema(type = SchemaType.OBJECT))) + responseCode = "200", + content = @Content(mediaType = "application/json", + schema = @Schema(implementation = InstanceFilesListResponse.class))) + @APIResponse(responseCode = "404", + description = "Live migration not enabled or node not configured for migration", + content = @Content(mediaType = "application/json", + schema = @Schema(type = SchemaType.OBJECT))) @ProvidesIntoMap @KeyClassMapKey(VertxRouteMapKeys.LiveMigrationListInstanceFilesRouteKey.class) public VertxRoute listInstanceFiles(RouteBuilder.Factory factory, @@ -230,19 +254,19 @@ public VertxRoute listInstanceFiles(RouteBuilder.Factory factory, @POST @Path(ApiEndpointsV1.LIVE_MIGRATION_STATUS_ROUTE) @Operation(summary = "Updates live migration status", - description = "Updates live migration status as COMPLETED for requested instance") + description = "Updates live migration status as COMPLETED for requested instance") @APIResponse(description = "Live migration status updated successfully", - responseCode = "200", - content = @Content(mediaType = "application/json", - schema = @Schema(implementation = LiveMigrationStatus.class))) + responseCode = "200", + content = @Content(mediaType = "application/json", + schema = @Schema(implementation = LiveMigrationStatus.class))) @APIResponse(responseCode = "400", - description = "When tried to update live migration status when it is already marked as COMPLETED", - content = @Content(mediaType = "application/json", - schema = @Schema(type = SchemaType.OBJECT))) + description = "When tried to update live migration status when it is already marked as COMPLETED", + content = @Content(mediaType = "application/json", + schema = @Schema(type = SchemaType.OBJECT))) @APIResponse(responseCode = "503", - description = "When could not update live migration status as COMPLETED", - content = @Content(mediaType = "application/json", - schema = @Schema(type = SchemaType.OBJECT))) + description = "When could not update live migration status as COMPLETED", + content = @Content(mediaType = "application/json", + schema = @Schema(type = SchemaType.OBJECT))) @ProvidesIntoMap @KeyClassMapKey(VertxRouteMapKeys.LiveMigrationStatusUpdateRouteKey.class) VertxRoute getStatusUpdateRoute(RouteBuilder.Factory factory, @@ -250,19 +274,19 @@ VertxRoute getStatusUpdateRoute(RouteBuilder.Factory factory, LiveMigrationStatusCompleteHandler statusCompleteHandler) { return factory.builderForRoute() - .handler(liveMigrationApiEnableDisableHandler::isSourceOrDestination) - .handler(statusCompleteHandler) - .build(); + .handler(liveMigrationApiEnableDisableHandler::isSourceOrDestination) + .handler(statusCompleteHandler) + .build(); } @GET @Path(ApiEndpointsV1.LIVE_MIGRATION_STATUS_ROUTE) @Operation(summary = "Get live migration status", - description = "Get the status of the live migration") + description = "Get the status of the live migration") @APIResponse(description = "Live migration status retrieved successfully", - responseCode = "200", - content = @Content(mediaType = "application/json", - schema = @Schema(implementation = LiveMigrationStatus.class))) + responseCode = "200", + content = @Content(mediaType = "application/json", + schema = @Schema(implementation = LiveMigrationStatus.class))) @ProvidesIntoMap @KeyClassMapKey(VertxRouteMapKeys.LiveMigrationStatusRouteKey.class) VertxRoute getStatusRoute(RouteBuilder.Factory factory, @@ -270,32 +294,32 @@ VertxRoute getStatusRoute(RouteBuilder.Factory factory, LiveMigrationStatusGetHandler statusHandler) { return factory.builderForRoute() - .handler(liveMigrationApiEnableDisableHandler::isSourceOrDestination) - .handler(statusHandler) - .build(); + .handler(liveMigrationApiEnableDisableHandler::isSourceOrDestination) + .handler(statusHandler) + .build(); } @DELETE @Path(ApiEndpointsV1.LIVE_MIGRATION_STATUS_ROUTE) @Operation(summary = "Deletes live migration status", - description = "Deletes live migration status for requested instance. " + - "It should be called after clearing the live migration map configuration only.") + description = "Deletes live migration status for requested instance. " + + "It should be called after clearing the live migration map configuration only.") @APIResponse(description = "Live migration status deleted successfully", - responseCode = "200", - content = @Content(mediaType = "application/json", - schema = @Schema(implementation = LiveMigrationStatus.class))) + responseCode = "200", + content = @Content(mediaType = "application/json", + schema = @Schema(implementation = LiveMigrationStatus.class))) @APIResponse(responseCode = "403", - description = "When tried to delete live migration status before clearing the live migration map", - content = @Content(mediaType = "application/json", - schema = @Schema(type = SchemaType.OBJECT))) + description = "When tried to delete live migration status before clearing the live migration map", + content = @Content(mediaType = "application/json", + schema = @Schema(type = SchemaType.OBJECT))) @APIResponse(responseCode = "400", - description = "When tried to delete Live migration status before without updating the status as COMPLETED", - content = @Content(mediaType = "application/json", - schema = @Schema(type = SchemaType.OBJECT))) + description = "When tried to delete Live migration status before without updating the status as COMPLETED", + content = @Content(mediaType = "application/json", + schema = @Schema(type = SchemaType.OBJECT))) @APIResponse(responseCode = "503", - description = "When faced some issue while deleting the live migration status", - content = @Content(mediaType = "application/json", - schema = @Schema(type = SchemaType.OBJECT))) + description = "When faced some issue while deleting the live migration status", + content = @Content(mediaType = "application/json", + schema = @Schema(type = SchemaType.OBJECT))) @ProvidesIntoMap @KeyClassMapKey(VertxRouteMapKeys.LiveMigrationStatusDeleteRouteKey.class) VertxRoute deleteStatusRoute(RouteBuilder.Factory factory, @@ -303,8 +327,110 @@ VertxRoute deleteStatusRoute(RouteBuilder.Factory factory, LiveMigrationStatusClearHandler statusDeleteHandler) { return factory.builderForRoute() - .handler(liveMigrationApiEnableDisableHandler::neitherSourceNorDestination) - .handler(statusDeleteHandler) - .build(); + .handler(liveMigrationApiEnableDisableHandler::neitherSourceNorDestination) + .handler(statusDeleteHandler) + .build(); + } + + @POST + @Path(ApiEndpointsV1.LIVE_MIGRATION_FILES_VERIFICATION_TASKS_ROUTE) + @Operation(summary = "Create files verification task", + description = "Creates a new files verification task") + @APIResponse(description = "Files verification task created successfully", + responseCode = "202", + content = @Content(mediaType = "application/json", + schema = @Schema(implementation = LiveMigrationTaskCreationResponse.class))) + @APIResponse(responseCode = "404", + description = "Live migration not enabled or node not configured as destination", + content = @Content(mediaType = "application/json", + schema = @Schema(type = SchemaType.OBJECT))) + @APIResponse(responseCode = "409", + description = "Cannot accept files verification task as another task is in progress", + content = @Content(mediaType = "application/json", + schema = @Schema(type = SchemaType.OBJECT))) + @ProvidesIntoMap + @KeyClassMapKey(VertxRouteMapKeys.LiveMigrationCreateFilesDigestVerificationTaskRouteKey.class) + public VertxRoute createFilesVerificationTask(RouteBuilder.Factory factory, + LiveMigrationApiEnableDisableHandler liveMigrationApiEnableDisableHandler, + LiveMigrationCreateFilesVerificationTaskHandler verificationTaskHandler) + { + return factory.builderForRoute() + .setBodyHandler(true) + .handler(liveMigrationApiEnableDisableHandler::isDestination) + .handler(liveMigrationApiEnableDisableHandler::allowIfMigrationNotComplete) + .handler(verificationTaskHandler) + .build(); + } + + @GET + @Path(value = ApiEndpointsV1.LIVE_MIGRATION_FILES_VERIFICATION_TASK_ROUTE) + @Operation(summary = "Get files verification task", + description = "Retrieves the files verification task by task ID") + @APIResponse(description = "Files verification task retrieved successfully", + responseCode = "200", + content = @Content(mediaType = "application/json", + schema = @Schema(implementation = LiveMigrationFilesVerificationResponse.class))) + @APIResponse(responseCode = "404", + description = "Live migration not enabled, node not configured as destination, or files verification task not found", + content = @Content(mediaType = "application/json", + schema = @Schema(type = SchemaType.OBJECT))) + @ProvidesIntoMap + @KeyClassMapKey(VertxRouteMapKeys.LiveMigrationGetFilesVerificationTaskRouteKey.class) + VertxRoute getFilesVerificationTask(RouteBuilder.Factory factory, + LiveMigrationApiEnableDisableHandler liveMigrationApiEnableDisableHandler, + LiveMigrationGetFilesVerificationTaskHandler getVerificationTaskHandler) + { + return factory.builderForRoute() + .handler(liveMigrationApiEnableDisableHandler::isDestination) + .handler(getVerificationTaskHandler) + .build(); + } + + @GET + @Path(value = ApiEndpointsV1.LIVE_MIGRATION_FILES_VERIFICATION_TASKS_ROUTE) + @Operation(summary = "Get all files verification tasks", + description = "Retrieves all live migration file verification tasks of the current node") + @APIResponse(description = "File verification tasks retrieved successfully", + responseCode = "200", + content = @Content(mediaType = "application/json", + schema = @Schema(type = SchemaType.ARRAY, implementation = LiveMigrationFilesVerificationResponse.class))) + @APIResponse(responseCode = "404", + description = "Live migration not enabled or node not configured as destination", + content = @Content(mediaType = "application/json", + schema = @Schema(type = SchemaType.OBJECT))) + @ProvidesIntoMap + @KeyClassMapKey(VertxRouteMapKeys.LiveMigrationGetAllFilesVerificationTasksRouteKey.class) + VertxRoute getAllFilesVerificationTask(RouteBuilder.Factory factory, + LiveMigrationApiEnableDisableHandler liveMigrationApiEnableDisableHandler, + LiveMigrationGetAllFilesVerificationTasksHandler getAllVerificationTaskHandler) + { + return factory.builderForRoute() + .handler(liveMigrationApiEnableDisableHandler::isDestination) + .handler(getAllVerificationTaskHandler) + .build(); + } + + @PATCH + @Path(value = ApiEndpointsV1.LIVE_MIGRATION_FILES_VERIFICATION_TASK_ROUTE) + @Operation(summary = "Cancel files verification task", + description = "Cancels an existing live migration files verification task") + @APIResponse(description = "Files verification task cancelled successfully", + responseCode = "200", + content = @Content(mediaType = "application/json", + schema = @Schema(implementation = LiveMigrationFilesVerificationResponse.class))) + @APIResponse(responseCode = "404", + description = "Live migration not enabled, node not configured as destination, or files verification task not found", + content = @Content(mediaType = "application/json", + schema = @Schema(type = SchemaType.OBJECT))) + @ProvidesIntoMap + @KeyClassMapKey(VertxRouteMapKeys.LiveMigrationCancelFilesVerificationTaskRouteKey.class) + VertxRoute cancelFilesVerificationTaskRoute(RouteBuilder.Factory factory, + LiveMigrationApiEnableDisableHandler liveMigrationApiEnableDisableHandler, + LiveMigrationCancelFilesVerificationTaskHandler cancelFilesVerificationTaskHandler) + { + return factory.builderForRoute() + .handler(liveMigrationApiEnableDisableHandler::isDestination) + .handler(cancelFilesVerificationTaskHandler) + .build(); } } diff --git a/server/src/main/java/org/apache/cassandra/sidecar/modules/multibindings/VertxRouteMapKeys.java b/server/src/main/java/org/apache/cassandra/sidecar/modules/multibindings/VertxRouteMapKeys.java index df5a37288..5ccbf32a7 100644 --- a/server/src/main/java/org/apache/cassandra/sidecar/modules/multibindings/VertxRouteMapKeys.java +++ b/server/src/main/java/org/apache/cassandra/sidecar/modules/multibindings/VertxRouteMapKeys.java @@ -254,6 +254,11 @@ interface LiveMigrationCancelDataCopyTaskRouteKey extends RouteClassKey HttpMethod HTTP_METHOD = HttpMethod.PATCH; String ROUTE_URI = ApiEndpointsV1.LIVE_MIGRATION_DATA_COPY_TASK_ROUTE; } + interface LiveMigrationCancelFilesVerificationTaskRouteKey extends RouteClassKey + { + HttpMethod HTTP_METHOD = HttpMethod.PATCH; + String ROUTE_URI = ApiEndpointsV1.LIVE_MIGRATION_FILES_VERIFICATION_TASK_ROUTE; + } interface LiveMigrationCreateDataCopyTaskRouteKey extends RouteClassKey { HttpMethod HTTP_METHOD = HttpMethod.POST; @@ -264,16 +269,31 @@ interface LiveMigrationGetAllDataCopyTasksRouteKey extends RouteClassKey HttpMethod HTTP_METHOD = HttpMethod.GET; String ROUTE_URI = ApiEndpointsV1.LIVE_MIGRATION_DATA_COPY_TASKS_ROUTE; } + interface LiveMigrationGetAllFilesVerificationTasksRouteKey extends RouteClassKey + { + HttpMethod HTTP_METHOD = HttpMethod.GET; + String ROUTE_URI = ApiEndpointsV1.LIVE_MIGRATION_FILES_VERIFICATION_TASKS_ROUTE; + } interface LiveMigrationGetDataCopyTaskRouteKey extends RouteClassKey { HttpMethod HTTP_METHOD = HttpMethod.GET; String ROUTE_URI = ApiEndpointsV1.LIVE_MIGRATION_DATA_COPY_TASK_ROUTE; } + interface LiveMigrationGetFilesVerificationTaskRouteKey extends RouteClassKey + { + HttpMethod HTTP_METHOD = HttpMethod.GET; + String ROUTE_URI = ApiEndpointsV1.LIVE_MIGRATION_FILES_VERIFICATION_TASK_ROUTE; + } interface LiveMigrationFileStreamHandlerRouteKey extends RouteClassKey { HttpMethod HTTP_METHOD = HttpMethod.GET; String ROUTE_URI = ApiEndpointsV1.LIVE_MIGRATION_FILE_TRANSFER_ROUTE; } + interface LiveMigrationCreateFilesDigestVerificationTaskRouteKey extends RouteClassKey + { + HttpMethod HTTP_METHOD = HttpMethod.POST; + String ROUTE_URI = ApiEndpointsV1.LIVE_MIGRATION_FILES_VERIFICATION_TASKS_ROUTE; + } interface LiveMigrationListInstanceFilesRouteKey extends RouteClassKey { HttpMethod HTTP_METHOD = HttpMethod.GET; @@ -284,12 +304,12 @@ interface LiveMigrationStatusRouteKey extends RouteClassKey HttpMethod HTTP_METHOD = HttpMethod.GET; String ROUTE_URI = ApiEndpointsV1.LIVE_MIGRATION_STATUS_ROUTE; } - public interface LiveMigrationStatusUpdateRouteKey extends RouteClassKey + interface LiveMigrationStatusUpdateRouteKey extends RouteClassKey { HttpMethod HTTP_METHOD = HttpMethod.POST; String ROUTE_URI = ApiEndpointsV1.LIVE_MIGRATION_STATUS_ROUTE; } - public interface LiveMigrationStatusDeleteRouteKey extends RouteClassKey + interface LiveMigrationStatusDeleteRouteKey extends RouteClassKey { HttpMethod HTTP_METHOD = HttpMethod.DELETE; String ROUTE_URI = ApiEndpointsV1.LIVE_MIGRATION_STATUS_ROUTE; diff --git a/server/src/main/java/org/apache/cassandra/sidecar/utils/AsyncFileDigestCalculator.java b/server/src/main/java/org/apache/cassandra/sidecar/utils/AsyncFileDigestCalculator.java new file mode 100644 index 000000000..1385a2b9e --- /dev/null +++ b/server/src/main/java/org/apache/cassandra/sidecar/utils/AsyncFileDigestCalculator.java @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.sidecar.utils; + +import java.io.IOException; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import io.vertx.core.Future; +import io.vertx.core.Handler; +import io.vertx.core.Promise; +import io.vertx.core.Vertx; +import io.vertx.core.buffer.Buffer; +import io.vertx.core.file.AsyncFile; +import io.vertx.core.file.OpenOptions; + +/** + * Utility class for asynchronously calculating file digests using Vert.x. + * Reads files in chunks to avoid loading entire files into memory, making it suitable for large files. + */ +public class AsyncFileDigestCalculator +{ + public static final int DEFAULT_READ_BUFFER_SIZE = 512 * 1024; // 512KiB + private static final Logger LOGGER = LoggerFactory.getLogger(AsyncFileDigestCalculator.class); + + /** + * Returns a future with the calculated digest for the file at the provided path. + * + * @param vertx the Vertx instance + * @param filePath the path to the file to use for digest calculation + * @param digestAlgorithm the digest algorithm to use + * @return a future with the computed digest for the file + */ + public static Future calculateDigest(Vertx vertx, String filePath, DigestAlgorithm digestAlgorithm) + { + return vertx.fileSystem() + .open(filePath, new OpenOptions().setRead(true).setCreate(false)) + .compose(asyncFile -> calculateDigest(asyncFile, digestAlgorithm)); + } + + /** + * Returns a future with the calculated digest for the provided {@link AsyncFile file}. + * + * @param asyncFile the async file to use for digest calculation + * @return a future with the computed digest for the provided {@link AsyncFile file} + */ + public static Future calculateDigest(AsyncFile asyncFile, DigestAlgorithm digestAlgorithm) + { + Promise result = Promise.promise(); + + readFile(asyncFile, result, + buf -> { + byte[] bytes = buf.getBytes(); + digestAlgorithm.update(bytes, 0, bytes.length); + }, + onReadComplete -> { + result.complete(digestAlgorithm.digest()); + try + { + digestAlgorithm.close(); + } + catch (IOException e) + { + LOGGER.warn("Potential memory leak due to failed to close hasher {}", + digestAlgorithm.getClass().getSimpleName()); + } + }); + + return result.future(); + } + + private static void readFile(AsyncFile file, + Promise result, + Handler onBufferAvailable, + Handler onReadComplete) + { + // Make sure to close the file when complete + result.future().onComplete(ignored -> file.end()); + file.pause() + .setReadBufferSize(DEFAULT_READ_BUFFER_SIZE) + .handler(onBufferAvailable) + .endHandler(onReadComplete) + .exceptionHandler(cause -> { + LOGGER.error("Could not read file", cause); + result.fail(cause); + }) + .resume(); + } +} diff --git a/server/src/main/java/org/apache/cassandra/sidecar/utils/AsyncFileDigestVerifier.java b/server/src/main/java/org/apache/cassandra/sidecar/utils/AsyncFileDigestVerifier.java index 881722b7b..d617045e1 100644 --- a/server/src/main/java/org/apache/cassandra/sidecar/utils/AsyncFileDigestVerifier.java +++ b/server/src/main/java/org/apache/cassandra/sidecar/utils/AsyncFileDigestVerifier.java @@ -18,16 +18,12 @@ package org.apache.cassandra.sidecar.utils; -import java.io.IOException; import java.util.Objects; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import io.vertx.core.Future; -import io.vertx.core.Handler; -import io.vertx.core.Promise; -import io.vertx.core.buffer.Buffer; import io.vertx.core.file.AsyncFile; import io.vertx.core.file.FileSystem; import io.vertx.core.file.OpenOptions; @@ -43,7 +39,6 @@ */ public abstract class AsyncFileDigestVerifier implements DigestVerifier { - public static final int DEFAULT_READ_BUFFER_SIZE = 512 * 1024; // 512KiB protected final Logger logger = LoggerFactory.getLogger(this.getClass()); protected final FileSystem fs; protected final D digest; @@ -91,42 +86,6 @@ public Future verify(String filePath) */ protected Future calculateDigest(AsyncFile asyncFile) { - Promise result = Promise.promise(); - - readFile(asyncFile, result, - buf -> { - byte[] bytes = buf.getBytes(); - digestAlgorithm.update(bytes, 0, bytes.length); - }, - onReadComplete -> { - result.complete(digestAlgorithm.digest()); - try - { - digestAlgorithm.close(); - } - catch (IOException e) - { - logger.warn("Potential memory leak due to failed to close hasher {}", - digestAlgorithm.getClass().getSimpleName()); - } - }); - - return result.future(); - } - - protected void readFile(AsyncFile file, Promise result, Handler onBufferAvailable, - Handler onReadComplete) - { - // Make sure to close the file when complete - result.future().onComplete(ignored -> file.end()); - file.pause() - .setReadBufferSize(DEFAULT_READ_BUFFER_SIZE) - .handler(onBufferAvailable) - .endHandler(onReadComplete) - .exceptionHandler(cause -> { - logger.error("Error while calculating the {} digest", digest.algorithm(), cause); - result.fail(cause); - }) - .resume(); + return AsyncFileDigestCalculator.calculateDigest(asyncFile, digestAlgorithm); } } diff --git a/server/src/main/java/org/apache/cassandra/sidecar/utils/DigestAlgorithmFactory.java b/server/src/main/java/org/apache/cassandra/sidecar/utils/DigestAlgorithmFactory.java new file mode 100644 index 000000000..c6063b7f9 --- /dev/null +++ b/server/src/main/java/org/apache/cassandra/sidecar/utils/DigestAlgorithmFactory.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.sidecar.utils; + +import com.google.inject.Inject; +import com.google.inject.name.Named; +import org.jetbrains.annotations.Nullable; + +import static org.apache.cassandra.sidecar.common.request.data.MD5Digest.MD5_ALGORITHM; +import static org.apache.cassandra.sidecar.common.request.data.XXHash32Digest.XXHASH_32_ALGORITHM; + +/** + * Factory for creating digest algorithm instances based on algorithm name and optional seed. + * Supports MD5 and XXHash32 digest algorithms. + */ +public class DigestAlgorithmFactory +{ + private final DigestAlgorithmProvider md5DigestAlgorithmProvider; + private final DigestAlgorithmProvider xxHash32DigestAlgorithmProvider; + + @Inject + public DigestAlgorithmFactory(@Named("md5") DigestAlgorithmProvider md5DigestAlgorithmProvider, + @Named("xxhash32") DigestAlgorithmProvider xxHash32DigestAlgorithmProvider) + { + this.md5DigestAlgorithmProvider = md5DigestAlgorithmProvider; + this.xxHash32DigestAlgorithmProvider = xxHash32DigestAlgorithmProvider; + } + + /** + * Validates whether the given digest algorithm name is supported. + * This method performs validation without creating a DigestAlgorithm instance. + * + * @param algorithmName the digest algorithm name to validate + * @throws IllegalArgumentException if the algorithm name is null, empty, or unsupported + */ + public static void validateAlgorithmName(String algorithmName) + { + if (null == algorithmName || algorithmName.isBlank()) + { + throw new IllegalArgumentException("Digest algorithm name cannot be null or empty"); + } + if (!algorithmName.equalsIgnoreCase(MD5_ALGORITHM) && !algorithmName.equalsIgnoreCase(XXHASH_32_ALGORITHM)) + { + throw new IllegalArgumentException("Unsupported digest algorithm " + algorithmName); + } + } + + /** + * Creates a digest algorithm instance based on the specified algorithm name and optional seed. + * + * @param algorithmName the digest algorithm name (MD5 or XXHash32, case-insensitive) + * @param seed optional seed value for the digest algorithm (maybe null) + * @return a DigestAlgorithm instance + * @throws IllegalArgumentException if algorithmName is null, empty, or unsupported + */ + public DigestAlgorithm getDigestAlgorithm(String algorithmName, @Nullable Integer seed) + { + if (null == algorithmName || algorithmName.isBlank()) + { + throw new IllegalArgumentException("Digest algorithm name cannot be null or empty"); + } + if (algorithmName.equalsIgnoreCase(MD5_ALGORITHM)) + { + return seed == null + ? md5DigestAlgorithmProvider.get() + : md5DigestAlgorithmProvider.get(seed); + } + else if (algorithmName.equalsIgnoreCase(XXHASH_32_ALGORITHM)) + { + return seed == null + ? xxHash32DigestAlgorithmProvider.get() + : xxHash32DigestAlgorithmProvider.get(seed); + } + else + { + throw new IllegalArgumentException("Unsupported digest algorithm " + algorithmName); + } + } +} diff --git a/server/src/main/java/org/apache/cassandra/sidecar/utils/JdkMd5DigestProvider.java b/server/src/main/java/org/apache/cassandra/sidecar/utils/JdkMd5DigestProvider.java index 6f213cb20..4f5fa9a58 100644 --- a/server/src/main/java/org/apache/cassandra/sidecar/utils/JdkMd5DigestProvider.java +++ b/server/src/main/java/org/apache/cassandra/sidecar/utils/JdkMd5DigestProvider.java @@ -30,17 +30,17 @@ public class JdkMd5DigestProvider implements DigestAlgorithmProvider @Override public DigestAlgorithm get(int seed) { - return new JdkMD5Digest(); + return new JdkMD5DigestAlgorithm(); } /** * MD5 implementation from JDK */ - public static class JdkMD5Digest implements DigestAlgorithm + public static class JdkMD5DigestAlgorithm implements DigestAlgorithm { private final MessageDigest md5; - public JdkMD5Digest() + public JdkMD5DigestAlgorithm() { try { diff --git a/server/src/main/java/org/apache/cassandra/sidecar/utils/XXHash32Provider.java b/server/src/main/java/org/apache/cassandra/sidecar/utils/XXHash32Provider.java index 7c45e9fc7..e39d422d6 100644 --- a/server/src/main/java/org/apache/cassandra/sidecar/utils/XXHash32Provider.java +++ b/server/src/main/java/org/apache/cassandra/sidecar/utils/XXHash32Provider.java @@ -28,17 +28,17 @@ public class XXHash32Provider implements DigestAlgorithmProvider @Override public DigestAlgorithm get(int seed) { - return new Lz4XXHash32(seed); + return new Lz4XXHash32DigestAlgorithm(seed); } /** * XXHash32 implementation from LZ4 */ - public static class Lz4XXHash32 implements DigestAlgorithm + public static class Lz4XXHash32DigestAlgorithm implements DigestAlgorithm { private final XXHash32 xxHash32; - Lz4XXHash32(int seed) + Lz4XXHash32DigestAlgorithm(int seed) { this.xxHash32 = new XXHash32(seed); } diff --git a/server/src/test/java/org/apache/cassandra/sidecar/HelperTestModules.java b/server/src/test/java/org/apache/cassandra/sidecar/HelperTestModules.java index 87674d82e..760e596fe 100644 --- a/server/src/test/java/org/apache/cassandra/sidecar/HelperTestModules.java +++ b/server/src/test/java/org/apache/cassandra/sidecar/HelperTestModules.java @@ -22,6 +22,9 @@ import java.util.Objects; import com.google.inject.AbstractModule; +import com.google.inject.Provides; +import com.google.inject.Singleton; +import com.google.inject.name.Named; import io.vertx.core.http.HttpServerRequest; import io.vertx.core.http.HttpServerResponse; import io.vertx.core.net.SocketAddress; @@ -29,10 +32,14 @@ import org.apache.cassandra.sidecar.cluster.InstancesMetadata; import org.apache.cassandra.sidecar.cluster.instance.InstanceMetadata; import org.apache.cassandra.sidecar.exceptions.NoSuchCassandraInstanceException; +import org.apache.cassandra.sidecar.utils.DigestAlgorithmProvider; +import org.apache.cassandra.sidecar.utils.JdkMd5DigestProvider; +import org.apache.cassandra.sidecar.utils.XXHash32Provider; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; /** @@ -101,4 +108,26 @@ protected void configure() bind(InstancesMetadata.class).toInstance(mockInstancesMetadata); } } + + /** + * Test module that provides {@link DigestAlgorithmProvider} + */ + public static class DigestAlgorithmProviderTestModule extends AbstractModule + { + @Provides + @Singleton + @Named("xxhash32") + DigestAlgorithmProvider xxHash32Provider() + { + return spy(new XXHash32Provider()); + } + + @Provides + @Singleton + @Named("md5") + DigestAlgorithmProvider md5Provider() + { + return spy(new JdkMd5DigestProvider()); + } + } } diff --git a/server/src/test/java/org/apache/cassandra/sidecar/handlers/livemigration/FakeLiveMigrationTask.java b/server/src/test/java/org/apache/cassandra/sidecar/handlers/livemigration/FakeLiveMigrationTask.java index 5a09cd1d9..f43734a1c 100644 --- a/server/src/test/java/org/apache/cassandra/sidecar/handlers/livemigration/FakeLiveMigrationTask.java +++ b/server/src/test/java/org/apache/cassandra/sidecar/handlers/livemigration/FakeLiveMigrationTask.java @@ -19,18 +19,20 @@ package org.apache.cassandra.sidecar.handlers.livemigration; -import org.apache.cassandra.sidecar.common.response.LiveMigrationTaskResponse; +import org.apache.cassandra.sidecar.common.response.LiveMigrationDataCopyResponse; import org.apache.cassandra.sidecar.livemigration.LiveMigrationTask; +import static org.apache.cassandra.sidecar.livemigration.LiveMigrationDataCopyTask.DATA_COPY_TASK_TYPE; + /** * Test implementation of LiveMigrationTask that returns predefined responses for testing purposes. */ -public class FakeLiveMigrationTask implements LiveMigrationTask +public class FakeLiveMigrationTask implements LiveMigrationTask { - private final LiveMigrationTaskResponse taskResponse; + private final LiveMigrationDataCopyResponse taskResponse; private boolean cancelled = false; - public FakeLiveMigrationTask(LiveMigrationTaskResponse taskResponse) + public FakeLiveMigrationTask(LiveMigrationDataCopyResponse taskResponse) { this.taskResponse = taskResponse; } @@ -41,13 +43,19 @@ public String id() return taskResponse.taskId(); } + @Override + public String type() + { + return DATA_COPY_TASK_TYPE; + } + @Override public void start() { } @Override - public LiveMigrationTaskResponse getResponse() + public LiveMigrationDataCopyResponse getResponse() { return taskResponse; } diff --git a/server/src/test/java/org/apache/cassandra/sidecar/handlers/livemigration/InstanceFetcherTestModule.java b/server/src/test/java/org/apache/cassandra/sidecar/handlers/livemigration/InstanceFetcherTestModule.java index 915bfc030..b7e444d43 100644 --- a/server/src/test/java/org/apache/cassandra/sidecar/handlers/livemigration/InstanceFetcherTestModule.java +++ b/server/src/test/java/org/apache/cassandra/sidecar/handlers/livemigration/InstanceFetcherTestModule.java @@ -51,7 +51,8 @@ protected void configure() List.of( getMockInstance(1, "localhost"), getMockInstance(2, "localhost2"), - getMockInstance(3, "localhost3")), + getMockInstance(3, "localhost3"), + getMockInstance(4, "localhost4")), new DnsResolver() { @Override diff --git a/server/src/test/java/org/apache/cassandra/sidecar/handlers/livemigration/InstanceMetadataTestUtil.java b/server/src/test/java/org/apache/cassandra/sidecar/handlers/livemigration/InstanceMetadataTestUtil.java new file mode 100644 index 000000000..82f75b3ae --- /dev/null +++ b/server/src/test/java/org/apache/cassandra/sidecar/handlers/livemigration/InstanceMetadataTestUtil.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.sidecar.handlers.livemigration; + +import java.nio.file.Path; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import com.codahale.metrics.MetricRegistry; +import org.apache.cassandra.sidecar.cluster.instance.InstanceMetadata; +import org.apache.cassandra.sidecar.cluster.instance.InstanceMetadataImpl; +import org.apache.cassandra.sidecar.metrics.MetricRegistryFactory; + +import static org.apache.cassandra.sidecar.common.ApiEndpointsV1.LIVE_MIGRATION_FILES_ROUTE; +import static org.apache.cassandra.sidecar.handlers.livemigration.LiveMigrationDirType.CDC_RAW_DIR; +import static org.apache.cassandra.sidecar.handlers.livemigration.LiveMigrationDirType.COMMIT_LOG_DIR; +import static org.apache.cassandra.sidecar.handlers.livemigration.LiveMigrationDirType.DATA_FILE_DIR; +import static org.apache.cassandra.sidecar.handlers.livemigration.LiveMigrationDirType.HINTS_DIR; +import static org.apache.cassandra.sidecar.handlers.livemigration.LiveMigrationDirType.LOCAL_SYSTEM_DATA_FILE_DIR; +import static org.apache.cassandra.sidecar.handlers.livemigration.LiveMigrationDirType.SAVED_CACHES_DIR; + +/** + * Utility class for creating test InstanceMetadata objects and live migration route paths for testing. + */ +public class InstanceMetadataTestUtil +{ + public static final String LIVE_MIGRATION_CDC_RAW_DIR_PATH = LIVE_MIGRATION_FILES_ROUTE + "/" + CDC_RAW_DIR.dirType; + public static final String LIVE_MIGRATION_COMMITLOG_DIR_PATH = LIVE_MIGRATION_FILES_ROUTE + "/" + COMMIT_LOG_DIR.dirType; + public static final String LIVE_MIGRATION_DATA_FILE_DIR_PATH = LIVE_MIGRATION_FILES_ROUTE + "/" + DATA_FILE_DIR.dirType; + public static final String LIVE_MIGRATION_HINTS_DIR_PATH = LIVE_MIGRATION_FILES_ROUTE + "/" + HINTS_DIR.dirType; + public static final String LIVE_MIGRATION_LOCAL_SYSTEM_DATA_FILE_DIR_PATH = LIVE_MIGRATION_FILES_ROUTE + + "/" + LOCAL_SYSTEM_DATA_FILE_DIR.dirType; + public static final String LIVE_MIGRATION_SAVED_CACHES_DIR_PATH = LIVE_MIGRATION_FILES_ROUTE + "/" + SAVED_CACHES_DIR.dirType; + private static final MetricRegistryFactory REGISTRY_FACTORY = + new MetricRegistryFactory("test_metric_registry", Collections.emptyList(), Collections.emptyList()); + + private InstanceMetadataTestUtil() + { + throw new AssertionError("Test utility class, no need to instantiate it"); + } + + public static InstanceMetadata getInstanceMetadata(String instanceIp, + int instanceId, + Path tempDir) + { + String root = tempDir.resolve(String.valueOf(instanceId)).toString(); + List dataDirs = Arrays.asList(root + "/d1/data", root + "/d2/data"); + MetricRegistry instanceSpecificRegistry = REGISTRY_FACTORY.getOrCreate(instanceId); + + return InstanceMetadataImpl.builder() + .id(instanceId) + .host(instanceIp) + .port(9042) + .storagePort(7000) + .dataDirs(dataDirs) + .hintsDir(root + "/hints") + .commitlogDir(root + "/commitlog") + .savedCachesDir(root + "/saved_caches") + .stagingDir(root + "/staging") + .cdcDir(root + "/cdc") + .localSystemDataFileDir(root + "/local_system_data") + .metricRegistry(instanceSpecificRegistry) + .build(); + } +} diff --git a/server/src/test/java/org/apache/cassandra/sidecar/handlers/livemigration/LiveMigrationConcurrencyLimitHandlerTest.java b/server/src/test/java/org/apache/cassandra/sidecar/handlers/livemigration/LiveMigrationConcurrencyLimitHandlerTest.java new file mode 100644 index 000000000..aeec349c7 --- /dev/null +++ b/server/src/test/java/org/apache/cassandra/sidecar/handlers/livemigration/LiveMigrationConcurrencyLimitHandlerTest.java @@ -0,0 +1,154 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.sidecar.handlers.livemigration; + +import org.junit.jupiter.api.Test; + +import com.google.inject.AbstractModule; +import com.google.inject.Guice; +import com.google.inject.Injector; +import io.netty.handler.codec.http.HttpResponseStatus; +import io.vertx.core.AsyncResult; +import io.vertx.core.Handler; +import io.vertx.ext.web.RoutingContext; +import io.vertx.ext.web.handler.HttpException; +import org.apache.cassandra.sidecar.HelperTestModules.RoutingContextTestModule; +import org.apache.cassandra.sidecar.config.LiveMigrationConfiguration; +import org.apache.cassandra.sidecar.config.SidecarConfiguration; +import org.mockito.ArgumentCaptor; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +class LiveMigrationConcurrencyLimitHandlerTest +{ + private Injector createInjector(int maxConcurrentFileRequests) + { + return Guice.createInjector(new TestModule(maxConcurrentFileRequests)); + } + + @Test + public void testPermitAcquiredWhenUnderLimit() + { + Injector injector = createInjector(2); + LiveMigrationConcurrencyLimitHandler handler = injector.getInstance(LiveMigrationConcurrencyLimitHandler.class); + RoutingContext rc = injector.getInstance(RoutingContext.class); + + handler.handle(rc); + + verify(rc, times(1)).next(); + verify(rc, times(1)).addEndHandler(any()); + verify(rc, never()).fail(any(Throwable.class)); + } + + @Test + public void testRequestRejectedWhenLimitExceeded() + { + Injector injector = createInjector(1); + LiveMigrationConcurrencyLimitHandler handler = injector.getInstance(LiveMigrationConcurrencyLimitHandler.class); + RoutingContext rc = injector.getInstance(RoutingContext.class); + + // First request acquires the only available permit + handler.handle(rc); + verify(rc, times(1)).next(); + + // Second request exceeds the limit and should be rejected + handler.handle(rc); + ArgumentCaptor captor = ArgumentCaptor.forClass(Throwable.class); + verify(rc, times(1)).fail(captor.capture()); + assertThat(captor.getValue()).isInstanceOf(HttpException.class); + assertThat(((HttpException) captor.getValue()).getStatusCode()).isEqualTo(HttpResponseStatus.SERVICE_UNAVAILABLE.code()); + // next() was only called for the first request + verify(rc, times(1)).next(); + } + + @Test + @SuppressWarnings("unchecked") + public void testPermitReleasedAfterEndHandler() + { + Injector injector = createInjector(1); + LiveMigrationConcurrencyLimitHandler handler = injector.getInstance(LiveMigrationConcurrencyLimitHandler.class); + RoutingContext rc = injector.getInstance(RoutingContext.class); + + // Acquire the single permit + handler.handle(rc); + verify(rc, times(1)).next(); + + // Capture the registered end handler and fire it to simulate request completion + ArgumentCaptor>> endHandlerCaptor = ArgumentCaptor.forClass(Handler.class); + verify(rc, times(1)).addEndHandler(endHandlerCaptor.capture()); + endHandlerCaptor.getValue().handle(null); + + // Permit was released; the next request should succeed + handler.handle(rc); + verify(rc, times(2)).next(); + verify(rc, never()).fail(any(Throwable.class)); + } + + @Test + public void testAllPermitsFilledThenRejected() + { + int limit = 3; + Injector injector = createInjector(limit); + LiveMigrationConcurrencyLimitHandler handler = injector.getInstance(LiveMigrationConcurrencyLimitHandler.class); + RoutingContext rc = injector.getInstance(RoutingContext.class); + + // Fill all permits up to the limit - each should succeed + for (int i = 0; i < limit; i++) + { + handler.handle(rc); + } + verify(rc, times(limit)).next(); + + // One more request beyond the limit should be rejected + handler.handle(rc); + ArgumentCaptor captor = ArgumentCaptor.forClass(Throwable.class); + verify(rc, times(1)).fail(captor.capture()); + assertThat(captor.getValue()).isInstanceOf(HttpException.class); + assertThat(((HttpException) captor.getValue()).getStatusCode()).isEqualTo(HttpResponseStatus.SERVICE_UNAVAILABLE.code()); + } + + private static class TestModule extends AbstractModule + { + private final int maxConcurrentFileRequests; + + TestModule(int maxConcurrentFileRequests) + { + this.maxConcurrentFileRequests = maxConcurrentFileRequests; + } + + @Override + protected void configure() + { + LiveMigrationConfiguration liveMigrationConfig = mock(LiveMigrationConfiguration.class); + when(liveMigrationConfig.maxConcurrentFileRequests()).thenReturn(maxConcurrentFileRequests); + + SidecarConfiguration sidecarConfiguration = mock(SidecarConfiguration.class); + when(sidecarConfiguration.liveMigrationConfiguration()).thenReturn(liveMigrationConfig); + + bind(SidecarConfiguration.class).toInstance(sidecarConfiguration); + install(new RoutingContextTestModule()); + } + } +} diff --git a/server/src/test/java/org/apache/cassandra/sidecar/handlers/livemigration/LiveMigrationDataCopyTaskHandlerTest.java b/server/src/test/java/org/apache/cassandra/sidecar/handlers/livemigration/LiveMigrationDataCopyTaskHandlerTest.java index c39948bb6..b087ee71a 100644 --- a/server/src/test/java/org/apache/cassandra/sidecar/handlers/livemigration/LiveMigrationDataCopyTaskHandlerTest.java +++ b/server/src/test/java/org/apache/cassandra/sidecar/handlers/livemigration/LiveMigrationDataCopyTaskHandlerTest.java @@ -50,8 +50,8 @@ import org.apache.cassandra.sidecar.TestModule; import org.apache.cassandra.sidecar.cluster.instance.InstanceMetadata; import org.apache.cassandra.sidecar.common.request.LiveMigrationDataCopyRequest; -import org.apache.cassandra.sidecar.common.response.LiveMigrationTaskResponse; -import org.apache.cassandra.sidecar.common.response.LiveMigrationTaskResponse.Status; +import org.apache.cassandra.sidecar.common.response.LiveMigrationDataCopyResponse; +import org.apache.cassandra.sidecar.common.response.LiveMigrationDataCopyResponse.Status; import org.apache.cassandra.sidecar.config.LiveMigrationConfiguration; import org.apache.cassandra.sidecar.config.SidecarConfiguration; import org.apache.cassandra.sidecar.config.yaml.SidecarConfigurationImpl; @@ -134,9 +134,9 @@ void testTaskSubmission(VertxTestContext context) when(taskFactory.create(anyString(), any(LiveMigrationDataCopyRequest.class), anyString(), anyInt(), any(InstanceMetadata.class))) .thenAnswer(invocation -> { return new FakeLiveMigrationTask( - new LiveMigrationTaskResponse(invocation.getArgument(0), - "dummysource", 9043, invocation.getArgument(1, LiveMigrationDataCopyRequest.class), - List.of(new Status(0, "SUCCESS", 1000, 10, 1000, 10, 10, 0, 1000)) + new LiveMigrationDataCopyResponse(invocation.getArgument(0), + "dummysource", 9043, invocation.getArgument(1, LiveMigrationDataCopyRequest.class), + List.of(new Status(0, "SUCCESS", 1000, 10, 1000, 10, 10, 0, 1000)) )); }); @@ -194,9 +194,9 @@ void testTaskSubmissionWhenAnotherTaskIsInProgress(VertxTestContext context) when(taskFactory.create(anyString(), any(LiveMigrationDataCopyRequest.class), anyString(), anyInt(), any(InstanceMetadata.class))) .thenAnswer(invocation -> { return new FakeLiveMigrationTask( - new LiveMigrationTaskResponse(invocation.getArgument(0), - "dummysource", 9043, invocation.getArgument(1, LiveMigrationDataCopyRequest.class), - List.of(new Status(0, "STARTING", -1, -1, -1, -1, 0, 0, 0)) + new LiveMigrationDataCopyResponse(invocation.getArgument(0), + "dummysource", 9043, invocation.getArgument(1, LiveMigrationDataCopyRequest.class), + List.of(new Status(0, "STARTING", -1, -1, -1, -1, 0, 0, 0)) )); }); // Task in starting state only. @@ -212,7 +212,7 @@ void testTaskSubmissionWhenAnotherTaskIsInProgress(VertxTestContext context) .as(BodyCodec.jsonObject()) .sendJsonObject(dataCopyTaskPayload)) .onSuccess(result -> context.verify(() -> { - assertThat(result.statusCode()).isEqualTo(HttpResponseStatus.FORBIDDEN.code()); + assertThat(result.statusCode()).isEqualTo(HttpResponseStatus.CONFLICT.code()); JsonObject task = result.body(); assertThat(task).isNotNull(); assertThat(task.getString("message")).isNotNull(); @@ -228,7 +228,7 @@ public void testTaskSubmissionWhenLiveMigrationAlreadyMarkedAsCompleted(VertxTes LiveMigrationTaskFactory taskFactory = injector.getInstance(LiveMigrationTaskFactory.class); when(taskFactory.create(anyString(), any(LiveMigrationDataCopyRequest.class), anyString(), anyInt(), any(InstanceMetadata.class))) - .thenAnswer(invocation -> new FakeLiveMigrationTask(new LiveMigrationTaskResponse( + .thenAnswer(invocation -> new FakeLiveMigrationTask(new LiveMigrationDataCopyResponse( invocation.getArgument(0), "dummysource", 9043, invocation.getArgument(1, LiveMigrationDataCopyRequest.class), List.of(new Status(0, "STARTING", -1, -1, -1, -1, 0, 0, 0)) @@ -265,9 +265,9 @@ void testCreateTaskCancelAndCreateAnotherTask(VertxTestContext context) when(taskFactory.create(anyString(), any(LiveMigrationDataCopyRequest.class), anyString(), anyInt(), any(InstanceMetadata.class))) .thenAnswer(invocation -> { return new FakeLiveMigrationTask( - new LiveMigrationTaskResponse(invocation.getArgument(0), - "dummysource", 9043, invocation.getArgument(1, LiveMigrationDataCopyRequest.class), - List.of(new Status(0, "STARTING", -1, -1, -1, -1, 0, 0, 0)) + new LiveMigrationDataCopyResponse(invocation.getArgument(0), + "dummysource", 9043, invocation.getArgument(1, LiveMigrationDataCopyRequest.class), + List.of(new Status(0, "STARTING", -1, -1, -1, -1, 0, 0, 0)) )); }); // Task in starting state only. @@ -312,9 +312,9 @@ void testCreateTaskBadRequestSubmission(VertxTestContext context) when(taskFactory.create(anyString(), any(LiveMigrationDataCopyRequest.class), anyString(), anyInt(), any(InstanceMetadata.class))) .thenAnswer(invocation -> { return new FakeLiveMigrationTask( - new LiveMigrationTaskResponse(invocation.getArgument(0), - "dummysource", 9043, invocation.getArgument(1, LiveMigrationDataCopyRequest.class), - List.of(new Status(0, "SUCCESS", 1000, 10, 1000, 10, 10, 0, 1000)) + new LiveMigrationDataCopyResponse(invocation.getArgument(0), + "dummysource", 9043, invocation.getArgument(1, LiveMigrationDataCopyRequest.class), + List.of(new Status(0, "SUCCESS", 1000, 10, 1000, 10, 10, 0, 1000)) )); }); @@ -341,16 +341,16 @@ void testCreateDataCopyTaskInvalidMaxConcurrency(VertxTestContext context) when(taskFactory.create(anyString(), any(LiveMigrationDataCopyRequest.class), anyString(), anyInt(), any(InstanceMetadata.class))) .thenAnswer(invocation -> { return new FakeLiveMigrationTask( - new LiveMigrationTaskResponse(invocation.getArgument(0), - "dummysource", 9043, invocation.getArgument(1, LiveMigrationDataCopyRequest.class), - List.of(new Status(0, "SUCCESS", 1000, 10, 1000, 10, 10, 0, 1000)) + new LiveMigrationDataCopyResponse(invocation.getArgument(0), + "dummysource", 9043, invocation.getArgument(1, LiveMigrationDataCopyRequest.class), + List.of(new Status(0, "SUCCESS", 1000, 10, 1000, 10, 10, 0, 1000)) )); }); // Here, we are trying to trigger a data copy request with max concurrency greater than the allowed limit. // Thus, this should throw validation errors and this test case is trying to test this particular scenario. final JsonObject dataCopyTaskPayload = getDataCopyTaskPayload(); - JsonObject badRequest = dataCopyTaskPayload.copy().put("maxConcurrency", sidecarConfiguration.liveMigrationConfiguration().maxConcurrentDownloads() + 1); + JsonObject badRequest = dataCopyTaskPayload.copy().put("maxConcurrency", sidecarConfiguration.liveMigrationConfiguration().maxConcurrentFileRequests() + 1); // Data copy task request can only be submitted from a destination host (since it follows a pull model) client.post(server.actualPort(), FIRST_DESTINATION_HOST, LIVE_MIGRATION_DATA_COPY_TASKS_ROUTE) @@ -401,9 +401,9 @@ void testCancelSucceededTask(VertxTestContext context) when(taskFactory.create(anyString(), any(LiveMigrationDataCopyRequest.class), anyString(), anyInt(), any(InstanceMetadata.class))) .thenAnswer(invocation -> { return new FakeLiveMigrationTask( - new LiveMigrationTaskResponse(invocation.getArgument(0), - "dummysource", 9043, invocation.getArgument(1, LiveMigrationDataCopyRequest.class), - List.of(new Status(0, "SUCCESS", 1000, 2, 1000, 2, 2, 0, 1000)) + new LiveMigrationDataCopyResponse(invocation.getArgument(0), + "dummysource", 9043, invocation.getArgument(1, LiveMigrationDataCopyRequest.class), + List.of(new Status(0, "SUCCESS", 1000, 2, 1000, 2, 2, 0, 1000)) )); }); @@ -447,9 +447,9 @@ void testCancelCancelledTask(VertxTestContext context) when(taskFactory.create(anyString(), any(LiveMigrationDataCopyRequest.class), anyString(), anyInt(), any(InstanceMetadata.class))) .thenAnswer(invocation -> { return new FakeLiveMigrationTask( - new LiveMigrationTaskResponse(invocation.getArgument(0), - "dummysource", 9043, invocation.getArgument(1, LiveMigrationDataCopyRequest.class), - List.of(new Status(0, "CANCELLED", 1000, 2, 1000, 2, 1, 0, 100)) + new LiveMigrationDataCopyResponse(invocation.getArgument(0), + "dummysource", 9043, invocation.getArgument(1, LiveMigrationDataCopyRequest.class), + List.of(new Status(0, "CANCELLED", 1000, 2, 1000, 2, 1, 0, 100)) )); }); @@ -510,7 +510,7 @@ protected void configure() .thenReturn(Collections.singleton("glob:${DATA_FILE_DIR}/*/*/snapshots")); when(mockLiveMigrationConfiguration.migrationMap()) .thenReturn(migrationMap); - when(mockLiveMigrationConfiguration.maxConcurrentDownloads()).thenReturn(10); + when(mockLiveMigrationConfiguration.maxConcurrentFileRequests()).thenReturn(10); SidecarConfiguration sidecarConfiguration = SidecarConfigurationImpl.builder() .liveMigrationConfiguration(mockLiveMigrationConfiguration) diff --git a/server/src/test/java/org/apache/cassandra/sidecar/handlers/livemigration/LiveMigrationFileDigestHandlerTest.java b/server/src/test/java/org/apache/cassandra/sidecar/handlers/livemigration/LiveMigrationFileDigestHandlerTest.java new file mode 100644 index 000000000..5d083891c --- /dev/null +++ b/server/src/test/java/org/apache/cassandra/sidecar/handlers/livemigration/LiveMigrationFileDigestHandlerTest.java @@ -0,0 +1,489 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.sidecar.handlers.livemigration; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.Arrays; +import java.util.Base64; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.io.TempDir; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.inject.AbstractModule; +import com.google.inject.Guice; +import com.google.inject.Injector; +import com.google.inject.util.Modules; +import io.netty.handler.codec.http.HttpResponseStatus; +import io.vertx.core.Vertx; +import io.vertx.core.json.JsonObject; +import io.vertx.ext.web.client.WebClient; +import io.vertx.ext.web.codec.BodyCodec; +import io.vertx.junit5.VertxExtension; +import io.vertx.junit5.VertxTestContext; +import org.apache.cassandra.sidecar.HelperTestModules.InstanceMetadataTestModule; +import org.apache.cassandra.sidecar.TestModule; +import org.apache.cassandra.sidecar.cluster.instance.InstanceMetadata; +import org.apache.cassandra.sidecar.config.LiveMigrationConfiguration; +import org.apache.cassandra.sidecar.config.SidecarConfiguration; +import org.apache.cassandra.sidecar.config.yaml.LiveMigrationConfigurationImpl; +import org.apache.cassandra.sidecar.config.yaml.SidecarConfigurationImpl; +import org.apache.cassandra.sidecar.modules.SidecarModules; +import org.apache.cassandra.sidecar.server.Server; +import org.apache.cassandra.sidecar.utils.DigestAlgorithm; +import org.apache.cassandra.sidecar.utils.XXHash32Provider; + +import static org.apache.cassandra.sidecar.common.ApiEndpointsV1.LIVE_MIGRATION_FILES_ROUTE; +import static org.apache.cassandra.sidecar.handlers.livemigration.InstanceMetadataTestUtil.getInstanceMetadata; +import static org.apache.cassandra.sidecar.utils.TestFileUtils.createFile; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; + +/** + * Tests for the LiveMigrationFileDigestHandler that validates file digest computation across different directory types and digest algorithms. + */ +@SuppressWarnings("SameParameterValue") +@ExtendWith(VertxExtension.class) +public class LiveMigrationFileDigestHandlerTest +{ + @SuppressWarnings("SpellCheckingInspection") + public static final String CHARACTER_SET = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"; + public static final Random RANDOM = new Random(); + private static final Logger LOGGER = LoggerFactory.getLogger(LiveMigrationFileDigestHandlerTest.class); + private static final int FIRST_ID = 1000110; + private static final String FIRST_INSTANCE_IP = "127.0.0.1"; + private static final int SECOND_ID = 1000111; + private static final String SECOND_INSTANCE_IP = "127.0.0.2"; + private static final int THIRD_ID = 1000112; + private static final String THIRD_INSTANCE_IP = "127.0.0.3"; + private static final int MAX_CONCURRENT_FILE_REQUESTS = 2; + private final Vertx vertx = Vertx.vertx(); + @TempDir + Path tempDir; + InstanceMetadata firstInstanceMeta; + List firstInstanceDataDirs; + List secondInstanceDataDirs; + List thirdInstanceDataDirs; + MessageDigest md5; + private Server server; + private Injector injector; + + public LiveMigrationFileDigestHandlerTest() throws NoSuchAlgorithmException + { + md5 = MessageDigest.getInstance("MD5"); + } + + @BeforeEach + void setup() throws InterruptedException + { + firstInstanceMeta = getInstanceMetadata(FIRST_INSTANCE_IP, FIRST_ID, tempDir); + firstInstanceDataDirs = firstInstanceMeta.dataDirs(); + InstanceMetadata secondInstanceMeta = getInstanceMetadata(SECOND_INSTANCE_IP, SECOND_ID, tempDir); + secondInstanceDataDirs = secondInstanceMeta.dataDirs(); + InstanceMetadata thirdInstanceMeta = getInstanceMetadata(THIRD_INSTANCE_IP, THIRD_ID, tempDir); + thirdInstanceDataDirs = thirdInstanceMeta.dataDirs(); + + LiveMigrationFileDigestHandlerTestModule handlerTestModule = new LiveMigrationFileDigestHandlerTestModule( + Arrays.asList(firstInstanceMeta, secondInstanceMeta, thirdInstanceMeta)); + + injector = Guice.createInjector(Modules.override(SidecarModules.all()) + .with(Modules.override(new TestModule()) + .with(handlerTestModule))); + server = injector.getInstance(Server.class); + VertxTestContext context = new VertxTestContext(); + + server.start() + .onSuccess(s -> context.completeNow()) + .onFailure(context::failNow); + context.awaitCompletion(15, TimeUnit.SECONDS); + } + + @AfterEach + void after() throws InterruptedException + { + CountDownLatch closeLatch = new CountDownLatch(1); + server.close().onSuccess(res -> closeLatch.countDown()); + if (closeLatch.await(60, TimeUnit.SECONDS)) + LOGGER.info("Close event received before timeout."); + else + LOGGER.error("Close event timed out."); + } + + @Test + public void testRouteSucceedsForFirstDataDirForMd5DigestAlgo(VertxTestContext context) throws IOException + { + String filePath = "/ks/tb-1234/ks-tb-1234-Data.db"; + String dummyText = getDummyData(RANDOM.nextInt(128)); + createFile(dummyText, firstInstanceDataDirs.get(0) + filePath); + + String testRoute = LIVE_MIGRATION_FILES_ROUTE + "/data/0" + filePath + "?digestAlgorithm=md5"; + shouldSucceed(context, testRoute, FIRST_INSTANCE_IP, SECOND_INSTANCE_IP, FIRST_INSTANCE_IP, md5Sum(dummyText)); + } + + @Test + public void testRouteSucceedsForSecondDataDir(VertxTestContext context) throws IOException + { + String filePath = "/ks/tb-1234/ks-tb-1234-Data.db"; + String dummyText = getDummyData(RANDOM.nextInt(128)); + createFile(dummyText, firstInstanceDataDirs.get(1) + filePath); + + // This time using XXHash32 as the digest algorithm + String testRoute = LIVE_MIGRATION_FILES_ROUTE + "/data/1" + filePath + "?digestAlgorithm=xxhash32"; + shouldSucceed(context, testRoute, FIRST_INSTANCE_IP, SECOND_INSTANCE_IP, FIRST_INSTANCE_IP, xxhash32(dummyText)); + } + + @Test + public void testRouteSucceedsForCommitLogDir(VertxTestContext context) throws IOException + { + String filePath = "/commit-1.db"; + String dummyText = getDummyData(RANDOM.nextInt(128)); + createFile(dummyText, firstInstanceMeta.commitlogDir() + filePath); + + String testRoute = LIVE_MIGRATION_FILES_ROUTE + "/commitlog/0" + filePath + "?digestAlgorithm=xxhash32"; + shouldSucceed(context, testRoute, FIRST_INSTANCE_IP, SECOND_INSTANCE_IP, FIRST_INSTANCE_IP, xxhash32(dummyText)); + } + + @Test + public void testRouteFailsForInvalidDir(VertxTestContext context) throws IOException + { + String filePath = "/commit-1.db"; + String dummyText = getDummyData(RANDOM.nextInt(128)); + createFile(dummyText, firstInstanceMeta.commitlogDir() + filePath); + + String testRoute = LIVE_MIGRATION_FILES_ROUTE + "/commitlog/1" + filePath + "?digestAlgorithm=xxhash32"; + shouldFail(context, testRoute, FIRST_INSTANCE_IP, SECOND_INSTANCE_IP, FIRST_INSTANCE_IP, 404); + } + + @Test + public void testRouteSucceedsForHintsDir(VertxTestContext context) throws IOException + { + String filePath = "/hints-1.db"; + String dummyText = getDummyData(RANDOM.nextInt(128)); + createFile(dummyText, firstInstanceMeta.hintsDir() + filePath); + + String testRoute = LIVE_MIGRATION_FILES_ROUTE + "/hints/0" + filePath + "?digestAlgorithm=xxhash32"; + shouldSucceed(context, testRoute, FIRST_INSTANCE_IP, SECOND_INSTANCE_IP, FIRST_INSTANCE_IP, xxhash32(dummyText)); + } + + @Test + public void testRouteSucceedsForSavedCachesDir(VertxTestContext context) throws IOException + { + String filePath = "/cache-101029383.txt"; + String dummyText = getDummyData(RANDOM.nextInt(128)); + createFile(dummyText, firstInstanceMeta.savedCachesDir() + filePath); + + String testRoute = LIVE_MIGRATION_FILES_ROUTE + "/saved_caches/0" + filePath + + "?digestAlgorithm=md5"; + shouldSucceed(context, testRoute, FIRST_INSTANCE_IP, SECOND_INSTANCE_IP, FIRST_INSTANCE_IP, md5Sum(dummyText)); + } + + @Test + public void testRouteSucceedsForCdcDir(VertxTestContext context) throws IOException + { + String filePath = "/Commitlog-8.db"; + String dummyText = getDummyData(RANDOM.nextInt(128)); + createFile(dummyText, firstInstanceMeta.cdcDir() + filePath); + + String testRoute = LIVE_MIGRATION_FILES_ROUTE + "/cdc_raw/0" + filePath + + "?digestAlgorithm=md5"; + shouldSucceed(context, testRoute, FIRST_INSTANCE_IP, SECOND_INSTANCE_IP, FIRST_INSTANCE_IP, md5Sum(dummyText)); + } + + @Test + public void testRouteFailsForDestination(VertxTestContext context) throws IOException + { + String filePath = "/Commitlog-8.db"; + String dummyText = getDummyData(RANDOM.nextInt(128)); + createFile(dummyText, firstInstanceMeta.cdcDir() + filePath); + + String testRoute = LIVE_MIGRATION_FILES_ROUTE + "/cdc_raw/0" + filePath + + "?digestAlgorithm=md5"; + shouldFail(context, testRoute, SECOND_INSTANCE_IP, FIRST_INSTANCE_IP, FIRST_INSTANCE_IP, 404); + } + + @Test + public void testRouteFailsForInstanceNotRelatedToLiveMigration(VertxTestContext context) throws IOException + { + String filePath = "/Commitlog-8.db"; + String dummyText = getDummyData(RANDOM.nextInt(128)); + createFile(dummyText, firstInstanceMeta.cdcDir() + filePath); + + String testRoute = LIVE_MIGRATION_FILES_ROUTE + "/cdc_raw/0" + filePath + + "?digestAlgorithm=md5"; + shouldFail(context, testRoute, SECOND_INSTANCE_IP, THIRD_INSTANCE_IP, FIRST_INSTANCE_IP, 404); + } + + @Test + public void testRequestWithEmptyDigestAlgoShouldFail(VertxTestContext context) throws IOException + { + String filePath = "/ks/tb-1234/ks-tb-1234-Data.db"; + String dummyText = getDummyData(RANDOM.nextInt(128)); + createFile(dummyText, firstInstanceDataDirs.get(0) + filePath); + + // Digest algo is just a white space + String testRoute = LIVE_MIGRATION_FILES_ROUTE + "/data/0" + filePath + "?digestAlgorithm= \t\n"; + shouldFail(context, testRoute, FIRST_INSTANCE_IP, SECOND_INSTANCE_IP, FIRST_INSTANCE_IP, 400); + } + + @Test + public void testRequestWithUnSupportedDigestAlgoShouldFail(VertxTestContext context) throws IOException + { + String filePath = "/ks/tb-1234/ks-tb-1234-Data.db"; + String dummyText = getDummyData(RANDOM.nextInt(128)); + createFile(dummyText, firstInstanceDataDirs.get(0) + filePath); + + String testRoute = LIVE_MIGRATION_FILES_ROUTE + "/data/0" + filePath + "?digestAlgorithm=UNKnown$"; + shouldFail(context, testRoute, FIRST_INSTANCE_IP, SECOND_INSTANCE_IP, FIRST_INSTANCE_IP, 400); + } + + @Test + public void testRouteFailsForNonExistentFile(VertxTestContext context) + { + String filePath = "/ks/tb-1234/ks-tb-1234-NonExistent.db"; + // Don't create the file - it should not exist + + String testRoute = LIVE_MIGRATION_FILES_ROUTE + "/data/0" + filePath + "?digestAlgorithm=md5"; + shouldFail(context, testRoute, FIRST_INSTANCE_IP, SECOND_INSTANCE_IP, FIRST_INSTANCE_IP, 404); + } + + @Test + public void testRouteSucceedsForEmptyFile(VertxTestContext context) throws IOException + { + String filePath = "/ks/tb-1234/ks-tb-1234-Empty.db"; + createFile("", firstInstanceDataDirs.get(0) + filePath); + + String testRoute = LIVE_MIGRATION_FILES_ROUTE + "/data/0" + filePath + "?digestAlgorithm=md5"; + shouldSucceed(context, testRoute, FIRST_INSTANCE_IP, SECOND_INSTANCE_IP, FIRST_INSTANCE_IP, md5Sum("")); + } + + @Test + public void testRouteSucceedsForEmptyFileWithXxhash32(VertxTestContext context) throws IOException + { + String filePath = "/ks/tb-1234/ks-tb-1234-Empty.db"; + createFile("", firstInstanceDataDirs.get(0) + filePath); + + String testRoute = LIVE_MIGRATION_FILES_ROUTE + "/data/0" + filePath + "?digestAlgorithm=xxhash32"; + shouldSucceed(context, testRoute, FIRST_INSTANCE_IP, SECOND_INSTANCE_IP, FIRST_INSTANCE_IP, xxhash32("")); + } + + @Test + public void testRequestWithUppercaseDigestAlgorithm(VertxTestContext context) throws IOException + { + String filePath = "/ks/tb-1234/ks-tb-1234-Data.db"; + String dummyText = getDummyData(RANDOM.nextInt(1024 * 1024)); + createFile(dummyText, firstInstanceDataDirs.get(0) + filePath); + + String testRoute = LIVE_MIGRATION_FILES_ROUTE + "/data/0" + filePath + "?digestAlgorithm=MD5"; + shouldSucceed(context, testRoute, FIRST_INSTANCE_IP, SECOND_INSTANCE_IP, FIRST_INSTANCE_IP, md5Sum(dummyText)); + } + + @Test + public void testRequestWithMixedCaseDigestAlgorithm(VertxTestContext context) throws IOException + { + String filePath = "/ks/tb-1234/ks-tb-1234-Data.db"; + String dummyText = getDummyData(RANDOM.nextInt(1024 * 1024)); + createFile(dummyText, firstInstanceDataDirs.get(0) + filePath); + + String testRoute = LIVE_MIGRATION_FILES_ROUTE + "/data/0" + filePath + "?digestAlgorithm=XxHash32"; + shouldSucceed(context, testRoute, FIRST_INSTANCE_IP, SECOND_INSTANCE_IP, FIRST_INSTANCE_IP, xxhash32(dummyText)); + } + + @Test + public void testConcurrencyLimitRejectsExcessRequests(VertxTestContext context) throws IOException + { + // Configuration has MAX_CONCURRENT_FILE_REQUESTS=2, send 3 requests rapidly + // At least one should be rejected + String dummyText = getDummyData(1024 * 1024); // 1MB file for slower processing + List filePaths = Arrays.asList( + "/ks/tb-1234/ks-tb-1234-Concurrent1.db", + "/ks/tb-1234/ks-tb-1234-Concurrent2.db", + "/ks/tb-1234/ks-tb-1234-Concurrent3.db" + ); + + // Create all test files + for (String filePath : filePaths) + { + createFile(dummyText, firstInstanceDataDirs.get(0) + filePath); + } + + mockLiveMigrationMap(FIRST_INSTANCE_IP, SECOND_INSTANCE_IP); + + WebClient client = WebClient.create(vertx); + AtomicInteger okCount = new AtomicInteger(0); + AtomicInteger tooManyRequestsCount = new AtomicInteger(0); + AtomicInteger completedCount = new AtomicInteger(0); + + // Send 3 requests rapidly - with limit of 2, at least one should be rejected + for (String filePath : filePaths) + { + String testRoute = LIVE_MIGRATION_FILES_ROUTE + "/data/0" + filePath + "?digestAlgorithm=md5"; + client.get(server.actualPort(), FIRST_INSTANCE_IP, testRoute) + .as(BodyCodec.buffer()) + .send(response -> { + if (response.succeeded()) + { + int statusCode = response.result().statusCode(); + if (statusCode == HttpResponseStatus.OK.code()) + { + okCount.incrementAndGet(); + } + else if (statusCode == HttpResponseStatus.SERVICE_UNAVAILABLE.code()) + { + tooManyRequestsCount.incrementAndGet(); + } + + if (completedCount.incrementAndGet() == filePaths.size()) + { + // All requests completed, verify results + context.verify(() -> { + // At least one request should be rejected due to concurrency limit + assertThat(tooManyRequestsCount.get()).isGreaterThan(0); + // Total should equal number of requests + assertThat(okCount.get() + tooManyRequestsCount.get()).isEqualTo(filePaths.size()); + client.close(); + context.completeNow(); + }); + } + } + else + { + context.failNow(response.cause()); + } + }); + } + } + + void shouldFail(VertxTestContext context, + String testRoute, + String source, + String destination, + String requestHost, + int expectedStatusCode) + { + mockLiveMigrationMap(source, destination); + + WebClient client = WebClient.create(vertx); + String url = String.format("http://%s:%d%s", requestHost, server.actualPort(), testRoute); + client.getAbs(url) + .as(BodyCodec.buffer()) + .send(context.succeeding(resp -> context.verify(() -> { + assertThat(resp.statusCode()).isEqualTo(expectedStatusCode); + client.close(); + context.completeNow(); + }))); + } + + void shouldSucceed(VertxTestContext context, + String testRoute, + String source, + String destination, + String requestHost, + String expectedDigest) + { + mockLiveMigrationMap(source, destination); + + WebClient client = WebClient.create(vertx); + client.get(server.actualPort(), requestHost, testRoute) + .as(BodyCodec.buffer()) + .send(context.succeeding(resp -> context.verify(() -> { + assertThat(resp.statusCode()).isEqualTo(HttpResponseStatus.OK.code()); + + JsonObject jsonObject = resp.bodyAsJsonObject(); + assertThat(jsonObject).isNotNull(); + assertThat(jsonObject.getString("digest")).isEqualTo(expectedDigest); + + client.close(); + context.completeNow(); + }))); + } + + void mockLiveMigrationMap(String source, String destination) + { + LiveMigrationConfiguration liveMigrationConfig = injector.getInstance(SidecarConfiguration.class) + .liveMigrationConfiguration(); + when(liveMigrationConfig.migrationMap()).thenReturn(Map.of(source, destination)); + } + + String getDummyData(int size) + { + StringBuilder randomString = new StringBuilder(); + + for (int i = 0; i < size; i++) + { + int index = RANDOM.nextInt(CHARACTER_SET.length()); + randomString.append(CHARACTER_SET.charAt(index)); + } + return randomString.toString(); + } + + String md5Sum(String data) + { + return Base64.getEncoder() + .encodeToString(md5.digest(data.getBytes(StandardCharsets.UTF_8))); + } + + String xxhash32(String data) + { + byte[] bytes = data.getBytes(StandardCharsets.UTF_8); + DigestAlgorithm digestAlgorithm = new XXHash32Provider().get(0); + digestAlgorithm.update(bytes, 0, bytes.length); + return digestAlgorithm.digest(); + } + + private static class LiveMigrationFileDigestHandlerTestModule extends AbstractModule + { + private final List instanceMetaList; + + public LiveMigrationFileDigestHandlerTestModule(List instanceMetaList) + { + this.instanceMetaList = instanceMetaList; + } + + @Override + protected void configure() + { + LiveMigrationConfiguration liveMigrationConfigurationSpy = spy(new LiveMigrationConfigurationImpl( + Set.of(), Set.of("glob:${DATA_FILE_DIR}/*/*/snapshots"), Map.of(), MAX_CONCURRENT_FILE_REQUESTS)); + + SidecarConfiguration sidecarConfiguration = + SidecarConfigurationImpl.builder() + .liveMigrationConfiguration(liveMigrationConfigurationSpy) + .build(); + + bind(SidecarConfiguration.class).toInstance(sidecarConfiguration); + install(new InstanceMetadataTestModule(instanceMetaList)); + } + } +} diff --git a/server/src/test/java/org/apache/cassandra/sidecar/handlers/livemigration/LiveMigrationFileStreamHandlerTest.java b/server/src/test/java/org/apache/cassandra/sidecar/handlers/livemigration/LiveMigrationFileStreamTest.java similarity index 85% rename from server/src/test/java/org/apache/cassandra/sidecar/handlers/livemigration/LiveMigrationFileStreamHandlerTest.java rename to server/src/test/java/org/apache/cassandra/sidecar/handlers/livemigration/LiveMigrationFileStreamTest.java index 8213e454e..e2e401936 100644 --- a/server/src/test/java/org/apache/cassandra/sidecar/handlers/livemigration/LiveMigrationFileStreamHandlerTest.java +++ b/server/src/test/java/org/apache/cassandra/sidecar/handlers/livemigration/LiveMigrationFileStreamTest.java @@ -25,7 +25,6 @@ import java.util.List; import java.util.Map; import java.util.Set; -import java.util.UUID; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -37,7 +36,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.codahale.metrics.MetricRegistry; import com.google.inject.AbstractModule; import com.google.inject.Guice; import com.google.inject.Injector; @@ -52,27 +50,28 @@ import org.apache.cassandra.sidecar.HelperTestModules; import org.apache.cassandra.sidecar.TestModule; import org.apache.cassandra.sidecar.cluster.instance.InstanceMetadata; -import org.apache.cassandra.sidecar.cluster.instance.InstanceMetadataImpl; import org.apache.cassandra.sidecar.config.LiveMigrationConfiguration; import org.apache.cassandra.sidecar.config.SidecarConfiguration; +import org.apache.cassandra.sidecar.config.yaml.LiveMigrationConfigurationImpl; import org.apache.cassandra.sidecar.config.yaml.SidecarConfigurationImpl; import org.apache.cassandra.sidecar.livemigration.LiveMigrationStatusTracker; -import org.apache.cassandra.sidecar.metrics.MetricRegistryFactory; import org.apache.cassandra.sidecar.modules.SidecarModules; import org.apache.cassandra.sidecar.server.Server; -import static org.apache.cassandra.sidecar.livemigration.LiveMigrationInstanceMetadataUtil.LIVE_MIGRATION_CDC_RAW_DIR_PATH; -import static org.apache.cassandra.sidecar.livemigration.LiveMigrationInstanceMetadataUtil.LIVE_MIGRATION_DATA_FILE_DIR_PATH; +import static org.apache.cassandra.sidecar.handlers.livemigration.InstanceMetadataTestUtil.LIVE_MIGRATION_CDC_RAW_DIR_PATH; +import static org.apache.cassandra.sidecar.handlers.livemigration.InstanceMetadataTestUtil.LIVE_MIGRATION_DATA_FILE_DIR_PATH; +import static org.apache.cassandra.sidecar.handlers.livemigration.InstanceMetadataTestUtil.getInstanceMetadata; import static org.apache.cassandra.sidecar.utils.TestFileUtils.createFile; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; @ExtendWith(VertxExtension.class) -class LiveMigrationFileStreamHandlerTest +class LiveMigrationFileStreamTest { - private static final Logger LOGGER = LoggerFactory.getLogger(LiveMigrationFileStreamHandlerTest.class); + private static final Logger LOGGER = LoggerFactory.getLogger(LiveMigrationFileStreamTest.class); private static final int FIRST_ID = 1000110; private static final String FIRST_INSTANCE_IP = "127.0.0.1"; @@ -82,8 +81,6 @@ class LiveMigrationFileStreamHandlerTest private static final String THIRD_INSTANCE_IP = "127.0.0.3"; private static final String DUMMY_CONTENT = "data"; - private static final MetricRegistryFactory REGISTRY_FACTORY = - new MetricRegistryFactory("cassandra_sidecar_" + UUID.randomUUID(), Collections.emptyList(), Collections.emptyList()); private final Vertx vertx = Vertx.vertx(); @TempDir Path tempDir; @@ -96,11 +93,11 @@ class LiveMigrationFileStreamHandlerTest @BeforeEach public void setup() throws InterruptedException { - InstanceMetadata firstInstanceMeta = getInstanceMetadata(FIRST_INSTANCE_IP, FIRST_ID); + InstanceMetadata firstInstanceMeta = getInstanceMetadata(FIRST_INSTANCE_IP, FIRST_ID, tempDir); firstInstanceDataDirs = firstInstanceMeta.dataDirs(); - InstanceMetadata secondInstanceMeta = getInstanceMetadata(SECOND_INSTANCE_IP, SECOND_ID); + InstanceMetadata secondInstanceMeta = getInstanceMetadata(SECOND_INSTANCE_IP, SECOND_ID, tempDir); secondInstanceDataDirs = secondInstanceMeta.dataDirs(); - InstanceMetadata thirdInstanceMeta = getInstanceMetadata(THIRD_INSTANCE_IP, THIRD_ID); + InstanceMetadata thirdInstanceMeta = getInstanceMetadata(THIRD_INSTANCE_IP, THIRD_ID, tempDir); thirdInstanceDataDirs = thirdInstanceMeta.dataDirs(); FileStreamHandlerTestModule handlerTestModule = new FileStreamHandlerTestModule( Arrays.asList(firstInstanceMeta, secondInstanceMeta, thirdInstanceMeta)); @@ -116,27 +113,6 @@ public void setup() throws InterruptedException context.awaitCompletion(15, TimeUnit.SECONDS); } - private InstanceMetadata getInstanceMetadata(String instanceIp, - int instanceId) - { - String root = tempDir.resolve(String.valueOf(instanceId)).toString(); - List dataDirs = Arrays.asList(root + "/d1/data", root + "/d2/data"); - MetricRegistry instanceSpecificRegistry = REGISTRY_FACTORY.getOrCreate(instanceId); - - return InstanceMetadataImpl.builder() - .id(instanceId) - .host(instanceIp) - .port(9042) - .storagePort(7000) - .dataDirs(dataDirs) - .hintsDir(root + "/hints") - .commitlogDir(root + "/commitlog") - .savedCachesDir(root + "/saved_caches") - .stagingDir(root + "/staging") - .metricRegistry(instanceSpecificRegistry) - .build(); - } - @AfterEach public void tearDown() throws InterruptedException { @@ -159,7 +135,7 @@ public void testRouteSucceeds(VertxTestContext context) throws IOException } @Test - public void testDownloadFailsWhenLiveMigrationCompletedAsCompleted(VertxTestContext context) throws IOException + public void testDownloadFailsWhenLiveMigrationMarkedAsCompleted(VertxTestContext context) throws IOException { String filePath = "/ks/tb-1234/ks-tb-1234-Data.db"; createFile(DUMMY_CONTENT, firstInstanceDataDirs.get(0) + filePath); @@ -189,8 +165,9 @@ public void testRouteDoubleDotAtTheEndAfterDirIndex(VertxTestContext context) th String filePath = "/ks/tb-1234/ks-tb-1234-Data.db"; createFile(DUMMY_CONTENT, firstInstanceDataDirs.get(0) + filePath); - shouldThrowError(context, LIVE_MIGRATION_DATA_FILE_DIR_PATH + "/data/0/..", - FIRST_INSTANCE_IP, THIRD_INSTANCE_IP, FIRST_INSTANCE_IP, 400); + String url = LIVE_MIGRATION_DATA_FILE_DIR_PATH + "/0/.."; + shouldThrowError(context, url, // This url itself will not be identified by Vertx, 404 is expected + FIRST_INSTANCE_IP, THIRD_INSTANCE_IP, FIRST_INSTANCE_IP, 404); } @Test @@ -231,7 +208,7 @@ public void testRequestInvalidPathUsingEncodedDots(VertxTestContext context) thr String filePath = "/ks/tb-1234/ks-tb-1234-Data.db"; createFile(DUMMY_CONTENT, firstInstanceDataDirs.get(0) + filePath); - // Vertx is stopping routes having three "%2E%2E" in the path and returning 404 + // Vertx rejects 3+ consecutive %2E%2E (encoded ..) patterns shouldThrowError(context, LIVE_MIGRATION_DATA_FILE_DIR_PATH + "/0/%2E%2E/%2E%2E/%2E%2E/secrets", FIRST_INSTANCE_IP, THIRD_INSTANCE_IP, FIRST_INSTANCE_IP, 404); } @@ -438,18 +415,13 @@ public FileStreamHandlerTestModule(List instanceMetaList) @Override protected void configure() { + LiveMigrationConfiguration liveMigrationConfigurationSpy = spy(new LiveMigrationConfigurationImpl( + Set.of(), Set.of("glob:${DATA_FILE_DIR}/*/*/snapshots"), Map.of(), 10)); - LiveMigrationConfiguration mockLiveMigrationConfiguration = mock(LiveMigrationConfiguration.class); - when(mockLiveMigrationConfiguration.filesToExclude()) - .thenReturn(Collections.emptySet()); - when(mockLiveMigrationConfiguration.directoriesToExclude()) - .thenReturn(Collections.singleton("glob:${DATA_FILE_DIR}/*/*/snapshots")); - when(mockLiveMigrationConfiguration.migrationMap()) - .thenReturn(Collections.emptyMap()); - - SidecarConfiguration sidecarConfiguration = SidecarConfigurationImpl.builder() - .liveMigrationConfiguration(mockLiveMigrationConfiguration) - .build(); + SidecarConfiguration sidecarConfiguration = + SidecarConfigurationImpl.builder() + .liveMigrationConfiguration(liveMigrationConfigurationSpy) + .build(); bind(SidecarConfiguration.class).toInstance(sidecarConfiguration); install(new HelperTestModules.InstanceMetadataTestModule(instanceMetaList)); diff --git a/server/src/test/java/org/apache/cassandra/sidecar/handlers/livemigration/LiveMigrationFilesVerificationTaskHandlerTest.java b/server/src/test/java/org/apache/cassandra/sidecar/handlers/livemigration/LiveMigrationFilesVerificationTaskHandlerTest.java new file mode 100644 index 000000000..a2291d39f --- /dev/null +++ b/server/src/test/java/org/apache/cassandra/sidecar/handlers/livemigration/LiveMigrationFilesVerificationTaskHandlerTest.java @@ -0,0 +1,699 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.sidecar.handlers.livemigration; + +import java.nio.file.Path; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.io.TempDir; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.inject.AbstractModule; +import com.google.inject.Guice; +import com.google.inject.Injector; +import com.google.inject.util.Modules; +import io.netty.handler.codec.http.HttpResponseStatus; +import io.vertx.core.Future; +import io.vertx.core.Vertx; +import io.vertx.core.json.JsonArray; +import io.vertx.core.json.JsonObject; +import io.vertx.ext.web.client.WebClient; +import io.vertx.ext.web.codec.BodyCodec; +import io.vertx.junit5.VertxExtension; +import io.vertx.junit5.VertxTestContext; +import org.apache.cassandra.sidecar.TestModule; +import org.apache.cassandra.sidecar.cluster.instance.InstanceMetadata; +import org.apache.cassandra.sidecar.common.request.LiveMigrationFilesVerificationRequest; +import org.apache.cassandra.sidecar.common.response.LiveMigrationFilesVerificationResponse; +import org.apache.cassandra.sidecar.config.LiveMigrationConfiguration; +import org.apache.cassandra.sidecar.config.SidecarConfiguration; +import org.apache.cassandra.sidecar.config.yaml.SidecarConfigurationImpl; +import org.apache.cassandra.sidecar.livemigration.FakeFilesVerificationTask; +import org.apache.cassandra.sidecar.livemigration.LiveMigrationFilesVerificationTask.State; +import org.apache.cassandra.sidecar.livemigration.LiveMigrationFilesVerificationTaskFactory; +import org.apache.cassandra.sidecar.modules.SidecarModules; +import org.apache.cassandra.sidecar.server.Server; +import org.mockito.stubbing.Answer; + +import static org.apache.cassandra.sidecar.common.ApiEndpointsV1.LIVE_MIGRATION_FILES_VERIFICATION_TASKS_ROUTE; +import static org.apache.cassandra.sidecar.common.ApiEndpointsV1.LIVE_MIGRATION_STATUS_ROUTE; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +@ExtendWith(VertxExtension.class) +class LiveMigrationFilesVerificationTaskHandlerTest +{ + private static final Logger LOGGER = LoggerFactory.getLogger(LiveMigrationFilesVerificationTaskHandlerTest.class); + + // Destination host in the live migration map corresponding to the FIRST_SOURCE_HOST + private static final String FIRST_DESTINATION_HOST = "127.0.0.1"; + + // Source host in the live migration map corresponding to the FIRST_DESTINATION_HOST + private static final String FIRST_SOURCE_HOST = "127.0.0.2"; + + // Destination host in the live migration map corresponding to the SECOND_SOURCE_HOST + private static final String SECOND_DESTINATION_HOST = "127.0.0.4"; + + // Source host in the live migration map corresponding to the SECOND_DESTINATION_HOST + private static final String SECOND_SOURCE_HOST = "127.0.0.3"; + + private final Vertx vertx = Vertx.vertx(); + Server server; + private Injector injector; + + @BeforeEach + void setup(@TempDir Path tempDir) throws InterruptedException + { + FilesVerificationHandlerTestModule handlerTestModule = new FilesVerificationHandlerTestModule(); + InstanceFetcherTestModule instanceFetcherTestModule = new InstanceFetcherTestModule(tempDir); + + injector = Guice.createInjector(Modules.override(SidecarModules.all()) + .with(Modules.override(new TestModule()) + .with(handlerTestModule, instanceFetcherTestModule))); + server = injector.getInstance(Server.class); + VertxTestContext context = new VertxTestContext(); + + server.start() + .onSuccess(s -> context.completeNow()) + .onFailure(context::failNow); + context.awaitCompletion(15, TimeUnit.SECONDS); + } + + @AfterEach + void after() throws InterruptedException + { + CountDownLatch closeLatch = new CountDownLatch(1); + server.close().onSuccess(res -> closeLatch.countDown()); + if (closeLatch.await(60, TimeUnit.SECONDS)) + LOGGER.info("Close event received before timeout."); + else + LOGGER.error("Close event timed out."); + } + + @Test + void testTaskSubmission(VertxTestContext context) + { + WebClient client = WebClient.create(vertx); + LiveMigrationFilesVerificationTaskFactory taskFactory = injector.getInstance(LiveMigrationFilesVerificationTaskFactory.class); + + when(taskFactory.create(anyString(), anyString(), anyInt(), any(LiveMigrationFilesVerificationRequest.class), any(InstanceMetadata.class))) + .thenAnswer(getFakeVerificationTaskAnswer(State.COMPLETED)); + + final JsonObject filesVerificationTaskPayload = getFilesVerificationTaskPayload(5); + + // Files verification task request is submitted from a destination host + client.post(server.actualPort(), FIRST_DESTINATION_HOST, LIVE_MIGRATION_FILES_VERIFICATION_TASKS_ROUTE) + .as(BodyCodec.jsonObject()) + .sendJsonObject(filesVerificationTaskPayload) + .onSuccess(result -> context.verify(() -> { + assertThat(result.statusCode()).isEqualTo(HttpResponseStatus.ACCEPTED.code()); + JsonObject response = result.body(); + assertThat(response).isNotNull(); + assertThat(response.getString("taskId")).isNotNull(); + assertThat(response.getString("statusUrl")).isNotNull(); + })) + .onFailure(cause -> context.failNow("Files verification task submission request should not fail")) + .compose(result -> { + JsonObject response = result.body(); + return Future.succeededFuture(response.getString("statusUrl")); + }) + .compose(statusUrl -> client.get(server.actualPort(), FIRST_DESTINATION_HOST, statusUrl) + .as(BodyCodec.jsonObject()) + .send()) + .onFailure(cause -> context.failNow("Couldn't fetch files verification task status URL.")) + .onSuccess(taskStatusResult -> context.verify(() -> { + assertThat(taskStatusResult.statusCode()).isEqualTo(HttpResponseStatus.OK.code()); + JsonObject taskStatus = taskStatusResult.body(); + assertThat(taskStatus).isNotNull(); + assertThat(taskStatus.getString("id")).isNotNull(); + assertThat(taskStatus.getString("state")).isNotNull(); + })) + .compose(taskStatus -> client.get(server.actualPort(), + FIRST_DESTINATION_HOST, + LIVE_MIGRATION_FILES_VERIFICATION_TASKS_ROUTE) + .as(BodyCodec.jsonArray()) + .send()) + .onSuccess(allTasksResult -> context.verify(() -> { + assertThat(allTasksResult.statusCode()).isEqualTo(HttpResponseStatus.OK.code()); + JsonArray allTasksJsonArray = allTasksResult.body(); + assertThat(allTasksJsonArray).isNotNull(); + assertThat(allTasksJsonArray).hasSize(1); + JsonObject taskStatus = allTasksJsonArray.getJsonObject(0); + assertThat(taskStatus).isNotNull(); + assertThat(taskStatus.getString("id")).isNotNull(); + assertThat(taskStatus.getString("state")).isNotNull(); + })) + .onFailure(context::failNow) + .onSuccess(result -> context.completeNow()); + } + + @Test + void testTaskSubmissionWhenAnotherTaskIsInProgress(VertxTestContext context) + { + WebClient client = WebClient.create(vertx); + + LiveMigrationFilesVerificationTaskFactory taskFactory = injector.getInstance(LiveMigrationFilesVerificationTaskFactory.class); + when(taskFactory.create(anyString(), anyString(), anyInt(), any(LiveMigrationFilesVerificationRequest.class), any(InstanceMetadata.class))) + .thenAnswer(getFakeVerificationTaskAnswer(State.IN_PROGRESS)); + + final JsonObject filesVerificationTaskPayload = getFilesVerificationTaskPayload(5); + + // Files verification task request is submitted from a destination host + client.post(server.actualPort(), FIRST_DESTINATION_HOST, LIVE_MIGRATION_FILES_VERIFICATION_TASKS_ROUTE) + .as(BodyCodec.json(Map.class)) + .sendJsonObject(filesVerificationTaskPayload) + .onSuccess(result -> context.verify(() -> assertThat(result.statusCode()).isEqualTo(HttpResponseStatus.ACCEPTED.code()))) + .onFailure(cause -> context.failNow("First files verification task submission failed.")) + .compose(response -> client.post(server.actualPort(), FIRST_DESTINATION_HOST, LIVE_MIGRATION_FILES_VERIFICATION_TASKS_ROUTE) + .as(BodyCodec.jsonObject()) + .sendJsonObject(filesVerificationTaskPayload)) + .onSuccess(result -> context.verify(() -> { + assertThat(result.statusCode()).isEqualTo(HttpResponseStatus.CONFLICT.code()); + JsonObject task = result.body(); + assertThat(task).isNotNull(); + assertThat(task.getString("message")).isNotNull(); + })) + .onFailure(context::failNow) // The call should not result a failure + .onSuccess(result -> context.completeNow()); + } + + @Test + public void testTaskSubmissionWhenLiveMigrationAlreadyMarkedAsCompleted(VertxTestContext context) + { + WebClient client = WebClient.create(vertx); + + LiveMigrationFilesVerificationTaskFactory taskFactory = injector.getInstance(LiveMigrationFilesVerificationTaskFactory.class); + when(taskFactory.create(anyString(), anyString(), anyInt(), any(LiveMigrationFilesVerificationRequest.class), any(InstanceMetadata.class))) + .thenAnswer(getFakeVerificationTaskAnswer(State.NOT_STARTED)); + + final JsonObject filesVerificationTaskPayload = getFilesVerificationTaskPayload(5); + + // First make request to live migration status API to mimic live migration completion and then + // submit files verification task. + client.post(server.actualPort(), FIRST_DESTINATION_HOST, LIVE_MIGRATION_STATUS_ROUTE) + .send() + .compose(response -> { + assertThat(response.statusCode()).isEqualTo(HttpResponseStatus.OK.code()); + return Future.succeededFuture(response); + }) + .compose(response -> client.post(server.actualPort(), FIRST_DESTINATION_HOST, LIVE_MIGRATION_FILES_VERIFICATION_TASKS_ROUTE) + .as(BodyCodec.jsonObject()) + .sendJsonObject(filesVerificationTaskPayload)) + .compose(filesVerificationTaskResponse -> { + assertThat(filesVerificationTaskResponse.statusCode()).isEqualTo(HttpResponseStatus.BAD_REQUEST.code()); + return Future.succeededFuture(); + }) + .onSuccess(v -> context.completeNow()) + .onFailure(context::failNow) // The call should not result a failure + .onComplete(result -> client.close()); + } + + @Test + void testCreateTaskCancelAndCreateAnotherTask(VertxTestContext context) + { + WebClient client = WebClient.create(vertx); + + LiveMigrationFilesVerificationTaskFactory taskFactory = injector.getInstance(LiveMigrationFilesVerificationTaskFactory.class); + when(taskFactory.create(anyString(), anyString(), anyInt(), any(LiveMigrationFilesVerificationRequest.class), any(InstanceMetadata.class))) + .thenAnswer(getFakeVerificationTaskAnswer(State.IN_PROGRESS)); + + final JsonObject filesVerificationTaskPayload = getFilesVerificationTaskPayload(5); + + // Files verification task request is submitted from a destination host + client.post(server.actualPort(), FIRST_DESTINATION_HOST, LIVE_MIGRATION_FILES_VERIFICATION_TASKS_ROUTE) + .as(BodyCodec.jsonObject()) + .sendJsonObject(filesVerificationTaskPayload) + .onSuccess(response -> context.verify(() -> { + assertThat(response.statusCode()).isEqualTo(HttpResponseStatus.ACCEPTED.code()); + JsonObject taskInfo = response.body(); + assertThat(taskInfo).isNotNull(); + assertThat(taskInfo.getString("statusUrl")).isNotNull(); + })) + .onFailure(cause -> context.failNow("First files verification task submission failed.")) + .compose(response -> { + JsonObject taskInfo = response.body(); + return Future.succeededFuture(taskInfo.getString("statusUrl")); + }) + .compose(statusUrl -> client.patch(server.actualPort(), FIRST_DESTINATION_HOST, statusUrl) + .as(BodyCodec.jsonObject()) + .send()) + .onSuccess(response -> context.verify(() -> assertThat(response.statusCode()).isEqualTo(HttpResponseStatus.OK.code()))) + .compose(response -> client.post(server.actualPort(), FIRST_DESTINATION_HOST, LIVE_MIGRATION_FILES_VERIFICATION_TASKS_ROUTE) + .as(BodyCodec.jsonObject()) + .sendJsonObject(filesVerificationTaskPayload)) + .onSuccess(newTaskResult -> context.verify(() -> { + assertThat(newTaskResult.statusCode()).isEqualTo(HttpResponseStatus.ACCEPTED.code()); + JsonObject newTask = newTaskResult.body(); + assertThat(newTask).isNotNull(); + })) + .onFailure(context::failNow) + .onSuccess(result -> context.completeNow()); + } + + @Test + void testCreateTaskBadRequestSubmission(VertxTestContext context) + { + WebClient client = WebClient.create(vertx); + LiveMigrationFilesVerificationTaskFactory taskFactory = injector.getInstance(LiveMigrationFilesVerificationTaskFactory.class); + when(taskFactory.create(anyString(), anyString(), anyInt(), any(LiveMigrationFilesVerificationRequest.class), any(InstanceMetadata.class))) + .thenAnswer(getFakeVerificationTaskAnswer(State.NOT_STARTED)); + + // Invalid digest algorithm should throw validation errors + final JsonObject filesVerificationTaskPayload = getFilesVerificationTaskPayload(5); + JsonObject badRequest = filesVerificationTaskPayload.copy().put("digestAlgorithm", "INVALID_ALGO"); + + // Files verification task request is submitted from a destination host + client.post(server.actualPort(), FIRST_DESTINATION_HOST, LIVE_MIGRATION_FILES_VERIFICATION_TASKS_ROUTE) + .sendJsonObject(badRequest, statusResult -> context.verify(() -> { + assertThat(statusResult.succeeded()).isTrue(); + assertThat(statusResult.result().statusCode()).isEqualTo(HttpResponseStatus.BAD_REQUEST.code()); + context.completeNow(); + })); + } + + @Test + void testCreateFilesVerificationTaskInvalidMaxConcurrency(VertxTestContext context) + { + WebClient client = WebClient.create(vertx); + LiveMigrationFilesVerificationTaskFactory taskFactory = injector.getInstance(LiveMigrationFilesVerificationTaskFactory.class); + SidecarConfiguration sidecarConfiguration = injector.getInstance(SidecarConfiguration.class); + when(taskFactory.create(anyString(), anyString(), anyInt(), any(LiveMigrationFilesVerificationRequest.class), any(InstanceMetadata.class))) + .thenAnswer(getFakeVerificationTaskAnswer(State.NOT_STARTED)); + + // Request with max concurrency greater than the allowed limit should throw validation errors + int invalidMaxConcurrency = sidecarConfiguration.liveMigrationConfiguration().maxConcurrentFileRequests() + 1; + final JsonObject badRequest = getFilesVerificationTaskPayload(invalidMaxConcurrency); + + // Files verification task request is submitted from a destination host + client.post(server.actualPort(), FIRST_DESTINATION_HOST, LIVE_MIGRATION_FILES_VERIFICATION_TASKS_ROUTE) + .as(BodyCodec.jsonObject()) + .sendJsonObject(badRequest) + .onSuccess(result -> context.verify(() -> { + assertThat(result.statusCode()).isEqualTo(HttpResponseStatus.BAD_REQUEST.code()); + JsonObject response = result.body(); + assertThat(response).isNotNull(); + assertThat(response.getString("message")).isNotNull(); + })) + .onFailure(context::failNow) + .onSuccess(result -> context.completeNow()); + } + + @Test + void testCreateFilesVerificationTaskMalformedRequest(VertxTestContext context) + { + WebClient client = WebClient.create(vertx); + LiveMigrationFilesVerificationTaskFactory taskFactory = injector.getInstance(LiveMigrationFilesVerificationTaskFactory.class); + when(taskFactory.create(anyString(), anyString(), anyInt(), any(LiveMigrationFilesVerificationRequest.class), any(InstanceMetadata.class))) + .thenAnswer(getFakeVerificationTaskAnswer(State.NOT_STARTED)); + + // Malformed request - missing required field 'digestAlgorithm' + final JsonObject malformedRequest = new JsonObject() + .put("maxConcurrency", 5); + + client.post(server.actualPort(), FIRST_DESTINATION_HOST, LIVE_MIGRATION_FILES_VERIFICATION_TASKS_ROUTE) + .as(BodyCodec.jsonObject()) + .sendJsonObject(malformedRequest) + .onSuccess(result -> context.verify(() -> { + assertThat(result.statusCode()).isEqualTo(HttpResponseStatus.BAD_REQUEST.code()); + JsonObject response = result.body(); + assertThat(response).isNotNull(); + assertThat(response.getString("message")).isNotNull(); + })) + .onFailure(context::failNow) + .onSuccess(result -> context.completeNow()); + } + + @Test + void testGetTaskStatusWhichDoesNotExist(VertxTestContext context) + { + WebClient client = WebClient.create(vertx); + + client.get(server.actualPort(), FIRST_DESTINATION_HOST, getFilesVerificationTaskStatusUrl("taskdonotexist")) + .send(statusResult -> context.verify(() -> { + assertThat(statusResult.succeeded()).isTrue(); + assertThat(statusResult.result().statusCode()).isEqualTo(HttpResponseStatus.NOT_FOUND.code()); + context.completeNow(); + })); + } + + @Test + void testCancelTaskWhichDoesNotExist(VertxTestContext context) + { + WebClient client = WebClient.create(vertx); + + // No need to mock - the real manager will throw exception when task doesn't exist + client.patch(server.actualPort(), FIRST_DESTINATION_HOST, getFilesVerificationTaskStatusUrl("taskdonotexist")) + .send(statusResult -> context.verify(() -> { + assertThat(statusResult.succeeded()).isTrue(); + // When task is not found, the handler logs a warning but doesn't fail the response + assertThat(statusResult.result().statusCode()).isEqualTo(HttpResponseStatus.NOT_FOUND.code()); + assertThat(statusResult.result().bodyAsJsonObject()).isNotNull(); + context.completeNow(); + })); + } + + @Test + void testCancelSucceededTask(VertxTestContext context) + { + WebClient client = WebClient.create(vertx); + + LiveMigrationFilesVerificationTaskFactory taskFactory = injector.getInstance(LiveMigrationFilesVerificationTaskFactory.class); + when(taskFactory.create(anyString(), anyString(), anyInt(), any(LiveMigrationFilesVerificationRequest.class), any(InstanceMetadata.class))) + .thenAnswer(getFakeVerificationTaskAnswer(State.COMPLETED)); + + final JsonObject filesVerificationTaskPayload = getFilesVerificationTaskPayload(5); + + client.post(server.actualPort(), FIRST_DESTINATION_HOST, LIVE_MIGRATION_FILES_VERIFICATION_TASKS_ROUTE) + .as(BodyCodec.jsonObject()) + .sendJsonObject(filesVerificationTaskPayload) + .onSuccess(response -> context.verify(() -> { + assertThat(response.statusCode()).isEqualTo(HttpResponseStatus.ACCEPTED.code()); + JsonObject taskInfo = response.body(); + assertThat(taskInfo).isNotNull(); + assertThat(taskInfo.getString("statusUrl")).isNotNull(); + })) + .onFailure(cause -> context.failNow("Files verification task submission failed.")) + .compose(response -> { + JsonObject taskInfo = response.body(); + return Future.succeededFuture(taskInfo.getString("statusUrl")); + }) + .compose(statusUrl -> client.patch(server.actualPort(), FIRST_DESTINATION_HOST, statusUrl) + .as(BodyCodec.jsonObject()) + .send()) + .onSuccess(response -> context.verify(() -> assertThat(response.statusCode()).isEqualTo(HttpResponseStatus.OK.code()))) + .onSuccess(cancelTaskResult -> context.verify(() -> { + assertThat(cancelTaskResult.statusCode()).isEqualTo(HttpResponseStatus.OK.code()); + JsonObject cancelledTask = cancelTaskResult.body(); + assertThat(cancelledTask).isNotNull(); + assertThat(cancelledTask.getString("state")).isEqualTo("COMPLETED"); + })) + .onFailure(context::failNow) + .onSuccess(result -> context.completeNow()); + } + + @Test + void testCancelCancelledTask(VertxTestContext context) + { + WebClient client = WebClient.create(vertx); + + LiveMigrationFilesVerificationTaskFactory taskFactory = injector.getInstance(LiveMigrationFilesVerificationTaskFactory.class); + when(taskFactory.create(anyString(), anyString(), anyInt(), any(LiveMigrationFilesVerificationRequest.class), any(InstanceMetadata.class))) + .thenAnswer(getFakeVerificationTaskAnswer(State.CANCELLED)); + + final JsonObject filesVerificationTaskPayload = getFilesVerificationTaskPayload(5); + + client.post(server.actualPort(), FIRST_DESTINATION_HOST, LIVE_MIGRATION_FILES_VERIFICATION_TASKS_ROUTE) + .as(BodyCodec.jsonObject()) + .sendJsonObject(filesVerificationTaskPayload) + .onSuccess(response -> context.verify(() -> { + assertThat(response.statusCode()).isEqualTo(HttpResponseStatus.ACCEPTED.code()); + JsonObject taskInfo = response.body(); + assertThat(taskInfo).isNotNull(); + assertThat(taskInfo.getString("statusUrl")).isNotNull(); + })) + .onFailure(cause -> context.failNow("Files verification task submission failed.")) + .compose(response -> { + JsonObject taskInfo = response.body(); + return Future.succeededFuture(taskInfo.getString("statusUrl")); + }) + .compose(statusUrl -> client.patch(server.actualPort(), FIRST_DESTINATION_HOST, statusUrl) + .as(BodyCodec.jsonObject()) + .send()) + .onSuccess(response -> context.verify(() -> assertThat(response.statusCode()).isEqualTo(HttpResponseStatus.OK.code()))) + .onSuccess(cancelTaskResult -> context.verify(() -> { + assertThat(cancelTaskResult.statusCode()).isEqualTo(HttpResponseStatus.OK.code()); + JsonObject cancelledTask = cancelTaskResult.body(); + assertThat(cancelledTask).isNotNull(); + assertThat(cancelledTask.getString("state")).isEqualTo("CANCELLED"); + })) + .onFailure(context::failNow) + .onSuccess(result -> context.completeNow()); + } + + @Test + void testGetAllTasksWhenNoTasksExist(VertxTestContext context) + { + WebClient client = WebClient.create(vertx); + + client.get(server.actualPort(), FIRST_DESTINATION_HOST, LIVE_MIGRATION_FILES_VERIFICATION_TASKS_ROUTE) + .as(BodyCodec.jsonArray()) + .send() + .onSuccess(result -> context.verify(() -> { + assertThat(result.statusCode()).isEqualTo(HttpResponseStatus.OK.code()); + JsonArray allTasksJsonArray = result.body(); + assertThat(allTasksJsonArray).isNotNull(); + assertThat(allTasksJsonArray).isEmpty(); + context.completeNow(); + })) + .onFailure(context::failNow); + } + + @Test + void testGetAllTasksWithMultipleTasks(VertxTestContext context) + { + WebClient client = WebClient.create(vertx); + + LiveMigrationFilesVerificationTaskFactory taskFactory = injector.getInstance(LiveMigrationFilesVerificationTaskFactory.class); + when(taskFactory.create(anyString(), anyString(), anyInt(), any(LiveMigrationFilesVerificationRequest.class), any(InstanceMetadata.class))) + .thenAnswer(getFakeVerificationTaskAnswer(State.IN_PROGRESS)); + + final JsonObject filesVerificationTaskPayload = getFilesVerificationTaskPayload(5); + + // Submit first task + client.post(server.actualPort(), FIRST_DESTINATION_HOST, LIVE_MIGRATION_FILES_VERIFICATION_TASKS_ROUTE) + .as(BodyCodec.jsonObject()) + .sendJsonObject(filesVerificationTaskPayload) + .andThen(ar -> { + context.verify(() -> assertThat(ar.result().statusCode()) + .isEqualTo(HttpResponseStatus.ACCEPTED.code())); + }) + .onFailure(cause -> context.failNow("First task submission failed.")) + .compose(response -> { + JsonObject taskInfo = response.body(); + return Future.succeededFuture(taskInfo.getString("statusUrl")); + }) + // Cancel first task so we can submit a second one + .compose(statusUrl -> client.patch(server.actualPort(), FIRST_DESTINATION_HOST, statusUrl) + .as(BodyCodec.jsonObject()) + .send()) + .andThen(response -> { + context.verify(() -> assertThat(response.result().statusCode()) + .isEqualTo(HttpResponseStatus.OK.code())); + }) + // Submit second task + .compose(response -> client.post(server.actualPort(), FIRST_DESTINATION_HOST, LIVE_MIGRATION_FILES_VERIFICATION_TASKS_ROUTE) + .as(BodyCodec.jsonObject()) + .sendJsonObject(filesVerificationTaskPayload)) + .andThen(ar -> { + context.verify(() -> assertThat(ar.result().statusCode()) + .isEqualTo(HttpResponseStatus.ACCEPTED.code())); + }) + .onFailure(cause -> context.failNow("Second task submission failed.")) + // Get all tasks + .compose(response -> client.get(server.actualPort(), FIRST_DESTINATION_HOST, LIVE_MIGRATION_FILES_VERIFICATION_TASKS_ROUTE) + .as(BodyCodec.jsonArray()) + .send()) + .onSuccess(allTasksResult -> context.verify(() -> { + assertThat(allTasksResult.statusCode()).isEqualTo(HttpResponseStatus.OK.code()); + JsonArray allTasksJsonArray = allTasksResult.body(); + assertThat(allTasksJsonArray).isNotNull(); + assertThat(allTasksJsonArray).hasSize(1); + context.completeNow(); + })) + .onFailure(context::failNow); + } + + @Test + void testHostIsolationForTasks(VertxTestContext context) + { + WebClient client = WebClient.create(vertx); + + LiveMigrationFilesVerificationTaskFactory taskFactory = injector.getInstance(LiveMigrationFilesVerificationTaskFactory.class); + when(taskFactory.create(anyString(), anyString(), anyInt(), any(LiveMigrationFilesVerificationRequest.class), any(InstanceMetadata.class))) + .thenAnswer(getFakeVerificationTaskAnswer(State.IN_PROGRESS)); + + final JsonObject filesVerificationTaskPayload = getFilesVerificationTaskPayload(5); + + // Submit task on FIRST_DESTINATION_HOST + client.post(server.actualPort(), FIRST_DESTINATION_HOST, LIVE_MIGRATION_FILES_VERIFICATION_TASKS_ROUTE) + .as(BodyCodec.jsonObject()) + .sendJsonObject(filesVerificationTaskPayload) + .andThen(ar -> { + context.verify(() -> assertThat(ar.result().statusCode()).isEqualTo(HttpResponseStatus.ACCEPTED.code())); + }) + .onFailure(cause -> context.failNow("Task submission on first host failed.")) + // Submit task on SECOND_DESTINATION_HOST + .compose(response -> client.post(server.actualPort(), SECOND_DESTINATION_HOST, LIVE_MIGRATION_FILES_VERIFICATION_TASKS_ROUTE) + .as(BodyCodec.jsonObject()) + .sendJsonObject(filesVerificationTaskPayload)) + .andThen(ar -> { + context.verify(() -> assertThat(ar.result().statusCode()).isEqualTo(HttpResponseStatus.ACCEPTED.code())); + }) + .onFailure(cause -> context.failNow("Task submission on second host failed.")) + // Get tasks from FIRST_DESTINATION_HOST - should only see 1 task + .compose(response -> client.get(server.actualPort(), FIRST_DESTINATION_HOST, LIVE_MIGRATION_FILES_VERIFICATION_TASKS_ROUTE) + .as(BodyCodec.jsonArray()) + .send()) + .andThen(firstHostTasks -> context.verify(() -> { + assertThat(firstHostTasks.result().statusCode()).isEqualTo(HttpResponseStatus.OK.code()); + JsonArray tasksArray = firstHostTasks.result().body(); + assertThat(tasksArray).isNotNull(); + assertThat(tasksArray).hasSize(1); + })) + // Get tasks from SECOND_DESTINATION_HOST - should only see 1 task + .compose(response -> client.get(server.actualPort(), SECOND_DESTINATION_HOST, LIVE_MIGRATION_FILES_VERIFICATION_TASKS_ROUTE) + .as(BodyCodec.jsonArray()) + .send()) + .andThen(secondHostTasks -> context.verify(() -> { + assertThat(secondHostTasks.result().statusCode()).isEqualTo(HttpResponseStatus.OK.code()); + JsonArray tasksArray = secondHostTasks.result().body(); + assertThat(tasksArray).isNotNull(); + assertThat(tasksArray).hasSize(1); + context.completeNow(); + })) + .onFailure(context::failNow); + } + + @Test + void testValidDigestAlgorithmXXHash32(VertxTestContext context) + { + WebClient client = WebClient.create(vertx); + + LiveMigrationFilesVerificationTaskFactory taskFactory = injector.getInstance(LiveMigrationFilesVerificationTaskFactory.class); + when(taskFactory.create(anyString(), anyString(), anyInt(), any(LiveMigrationFilesVerificationRequest.class), any(InstanceMetadata.class))) + .thenAnswer(getFakeVerificationTaskAnswer(State.COMPLETED)); + + final JsonObject filesVerificationTaskPayload = new JsonObject() + .put("maxConcurrency", 5) + .put("digestAlgorithm", "XXHash32"); + + client.post(server.actualPort(), FIRST_DESTINATION_HOST, LIVE_MIGRATION_FILES_VERIFICATION_TASKS_ROUTE) + .as(BodyCodec.jsonObject()) + .sendJsonObject(filesVerificationTaskPayload) + .onSuccess(result -> context.verify(() -> { + assertThat(result.statusCode()).isEqualTo(HttpResponseStatus.ACCEPTED.code()); + JsonObject response = result.body(); + assertThat(response).isNotNull(); + assertThat(response.getString("taskId")).isNotNull(); + assertThat(response.getString("statusUrl")).isNotNull(); + context.completeNow(); + })) + .onFailure(context::failNow); + } + + @Test + void testMinValidMaxConcurrency(VertxTestContext context) + { + WebClient client = WebClient.create(vertx); + + LiveMigrationFilesVerificationTaskFactory taskFactory = injector.getInstance(LiveMigrationFilesVerificationTaskFactory.class); + when(taskFactory.create(anyString(), anyString(), anyInt(), any(LiveMigrationFilesVerificationRequest.class), any(InstanceMetadata.class))) + .thenAnswer(getFakeVerificationTaskAnswer(State.COMPLETED)); + + final JsonObject filesVerificationTaskPayload = getFilesVerificationTaskPayload(1); + + client.post(server.actualPort(), FIRST_DESTINATION_HOST, LIVE_MIGRATION_FILES_VERIFICATION_TASKS_ROUTE) + .as(BodyCodec.jsonObject()) + .sendJsonObject(filesVerificationTaskPayload) + .onSuccess(result -> context.verify(() -> { + assertThat(result.statusCode()).isEqualTo(HttpResponseStatus.ACCEPTED.code()); + JsonObject response = result.body(); + assertThat(response).isNotNull(); + assertThat(response.getString("taskId")).isNotNull(); + context.completeNow(); + })) + .onFailure(context::failNow); + } + + private JsonObject getFilesVerificationTaskPayload(int maxConcurrency) + { + return new JsonObject() + .put("maxConcurrency", maxConcurrency) + .put("digestAlgorithm", "MD5"); + } + + Answer getFakeVerificationTaskAnswer(State state) + { + return (Answer) invocation -> new FakeFilesVerificationTask( + new LiveMigrationFilesVerificationResponse(invocation.getArgument(0), + "MD5", + state.name(), + invocation.getArgument(1), + invocation.getArgument(2), + 0, + 0, + 50, + 0, + 0, + 0, + 50)); + } + + @SuppressWarnings("SameParameterValue") + private String getFilesVerificationTaskStatusUrl(String taskId) + { + return LIVE_MIGRATION_FILES_VERIFICATION_TASKS_ROUTE + "/" + taskId; + } + + static class FilesVerificationHandlerTestModule extends AbstractModule + { + + @Override + protected void configure() + { + final Map migrationMap = new HashMap<>() + {{ + put("localhost2", "localhost"); + put(FIRST_SOURCE_HOST, FIRST_DESTINATION_HOST); + put("localhost3", "localhost4"); + put(SECOND_SOURCE_HOST, SECOND_DESTINATION_HOST); + }}; + + LiveMigrationConfiguration mockLiveMigrationConfiguration = mock(LiveMigrationConfiguration.class); + when(mockLiveMigrationConfiguration.filesToExclude()) + .thenReturn(Collections.emptySet()); + when(mockLiveMigrationConfiguration.directoriesToExclude()) + .thenReturn(Collections.singleton("glob:${DATA_FILE_DIR}/*/*/snapshots")); + when(mockLiveMigrationConfiguration.migrationMap()) + .thenReturn(migrationMap); + when(mockLiveMigrationConfiguration.maxConcurrentFileRequests()).thenReturn(10); + + SidecarConfiguration sidecarConfiguration = SidecarConfigurationImpl.builder() + .liveMigrationConfiguration(mockLiveMigrationConfiguration) + .build(); + + bind(SidecarConfiguration.class).toInstance(sidecarConfiguration); + bind(LiveMigrationFilesVerificationTaskFactory.class) + .toInstance(mock(LiveMigrationFilesVerificationTaskFactory.class)); + } + } +} diff --git a/server/src/test/java/org/apache/cassandra/sidecar/handlers/livemigration/LiveMigrationListInstanceFilesHandlerTest.java b/server/src/test/java/org/apache/cassandra/sidecar/handlers/livemigration/LiveMigrationListInstanceFilesHandlerTest.java index b0337aa16..ac5a6ac56 100644 --- a/server/src/test/java/org/apache/cassandra/sidecar/handlers/livemigration/LiveMigrationListInstanceFilesHandlerTest.java +++ b/server/src/test/java/org/apache/cassandra/sidecar/handlers/livemigration/LiveMigrationListInstanceFilesHandlerTest.java @@ -68,11 +68,11 @@ import org.apache.cassandra.sidecar.server.Server; import static org.apache.cassandra.sidecar.common.ApiEndpointsV1.LIVE_MIGRATION_FILES_ROUTE; +import static org.apache.cassandra.sidecar.handlers.livemigration.InstanceMetadataTestUtil.LIVE_MIGRATION_COMMITLOG_DIR_PATH; +import static org.apache.cassandra.sidecar.handlers.livemigration.InstanceMetadataTestUtil.LIVE_MIGRATION_DATA_FILE_DIR_PATH; +import static org.apache.cassandra.sidecar.handlers.livemigration.InstanceMetadataTestUtil.LIVE_MIGRATION_HINTS_DIR_PATH; +import static org.apache.cassandra.sidecar.handlers.livemigration.InstanceMetadataTestUtil.LIVE_MIGRATION_SAVED_CACHES_DIR_PATH; import static org.apache.cassandra.sidecar.livemigration.InstanceFileInfoTestUtil.findInstanceFileInfo; -import static org.apache.cassandra.sidecar.livemigration.LiveMigrationInstanceMetadataUtil.LIVE_MIGRATION_COMMITLOG_DIR_PATH; -import static org.apache.cassandra.sidecar.livemigration.LiveMigrationInstanceMetadataUtil.LIVE_MIGRATION_DATA_FILE_DIR_PATH; -import static org.apache.cassandra.sidecar.livemigration.LiveMigrationInstanceMetadataUtil.LIVE_MIGRATION_HINTS_DIR_PATH; -import static org.apache.cassandra.sidecar.livemigration.LiveMigrationInstanceMetadataUtil.LIVE_MIGRATION_SAVED_CACHES_DIR_PATH; import static org.apache.cassandra.sidecar.utils.TestFileUtils.createFile; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.mock; diff --git a/server/src/test/java/org/apache/cassandra/sidecar/livemigration/CassandraInstanceFilesImplTest.java b/server/src/test/java/org/apache/cassandra/sidecar/livemigration/CassandraInstanceFilesImplTest.java index d3d58ee86..33371b771 100644 --- a/server/src/test/java/org/apache/cassandra/sidecar/livemigration/CassandraInstanceFilesImplTest.java +++ b/server/src/test/java/org/apache/cassandra/sidecar/livemigration/CassandraInstanceFilesImplTest.java @@ -34,13 +34,13 @@ import org.apache.cassandra.sidecar.config.LiveMigrationConfiguration; import static org.apache.cassandra.sidecar.common.response.InstanceFileInfo.FileType.DIRECTORY; +import static org.apache.cassandra.sidecar.handlers.livemigration.InstanceMetadataTestUtil.LIVE_MIGRATION_CDC_RAW_DIR_PATH; +import static org.apache.cassandra.sidecar.handlers.livemigration.InstanceMetadataTestUtil.LIVE_MIGRATION_COMMITLOG_DIR_PATH; +import static org.apache.cassandra.sidecar.handlers.livemigration.InstanceMetadataTestUtil.LIVE_MIGRATION_DATA_FILE_DIR_PATH; +import static org.apache.cassandra.sidecar.handlers.livemigration.InstanceMetadataTestUtil.LIVE_MIGRATION_HINTS_DIR_PATH; +import static org.apache.cassandra.sidecar.handlers.livemigration.InstanceMetadataTestUtil.LIVE_MIGRATION_LOCAL_SYSTEM_DATA_FILE_DIR_PATH; +import static org.apache.cassandra.sidecar.handlers.livemigration.InstanceMetadataTestUtil.LIVE_MIGRATION_SAVED_CACHES_DIR_PATH; import static org.apache.cassandra.sidecar.livemigration.InstanceFileInfoTestUtil.findInstanceFileInfo; -import static org.apache.cassandra.sidecar.livemigration.LiveMigrationInstanceMetadataUtil.LIVE_MIGRATION_CDC_RAW_DIR_PATH; -import static org.apache.cassandra.sidecar.livemigration.LiveMigrationInstanceMetadataUtil.LIVE_MIGRATION_COMMITLOG_DIR_PATH; -import static org.apache.cassandra.sidecar.livemigration.LiveMigrationInstanceMetadataUtil.LIVE_MIGRATION_DATA_FILE_DIR_PATH; -import static org.apache.cassandra.sidecar.livemigration.LiveMigrationInstanceMetadataUtil.LIVE_MIGRATION_HINTS_DIR_PATH; -import static org.apache.cassandra.sidecar.livemigration.LiveMigrationInstanceMetadataUtil.LIVE_MIGRATION_LOCAL_SYSTEM_DATA_FILE_DIR_PATH; -import static org.apache.cassandra.sidecar.livemigration.LiveMigrationInstanceMetadataUtil.LIVE_MIGRATION_SAVED_CACHES_DIR_PATH; import static org.apache.cassandra.sidecar.utils.TestFileUtils.createFile; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.mock; diff --git a/server/src/test/java/org/apache/cassandra/sidecar/livemigration/DataCopyTaskManagerTest.java b/server/src/test/java/org/apache/cassandra/sidecar/livemigration/DataCopyTaskManagerTest.java index 52bafd6e8..d18971976 100644 --- a/server/src/test/java/org/apache/cassandra/sidecar/livemigration/DataCopyTaskManagerTest.java +++ b/server/src/test/java/org/apache/cassandra/sidecar/livemigration/DataCopyTaskManagerTest.java @@ -28,34 +28,36 @@ import org.junit.jupiter.api.Test; -import com.google.inject.AbstractModule; import com.google.inject.Guice; import com.google.inject.Injector; import io.vertx.core.Future; import io.vertx.core.Vertx; -import org.apache.cassandra.sidecar.cluster.CassandraAdapterDelegate; import org.apache.cassandra.sidecar.cluster.InstancesMetadata; import org.apache.cassandra.sidecar.cluster.instance.InstanceMetadata; import org.apache.cassandra.sidecar.common.request.LiveMigrationDataCopyRequest; -import org.apache.cassandra.sidecar.common.response.LiveMigrationTaskResponse; -import org.apache.cassandra.sidecar.config.LiveMigrationConfiguration; -import org.apache.cassandra.sidecar.config.ServiceConfiguration; -import org.apache.cassandra.sidecar.config.SidecarConfiguration; +import org.apache.cassandra.sidecar.common.request.LiveMigrationFilesVerificationRequest; +import org.apache.cassandra.sidecar.common.response.LiveMigrationDataCopyResponse; +import org.apache.cassandra.sidecar.common.response.LiveMigrationFilesVerificationResponse; import org.apache.cassandra.sidecar.exceptions.CassandraUnavailableException; -import org.apache.cassandra.sidecar.exceptions.LiveMigrationExceptions.LiveMigrationDataCopyInProgressException; import org.apache.cassandra.sidecar.exceptions.LiveMigrationExceptions.LiveMigrationInvalidRequestException; +import org.apache.cassandra.sidecar.exceptions.LiveMigrationExceptions.LiveMigrationTaskInProgressException; import org.apache.cassandra.sidecar.exceptions.LiveMigrationExceptions.LiveMigrationTaskNotFoundException; import org.apache.cassandra.sidecar.handlers.livemigration.FakeLiveMigrationTask; -import org.apache.cassandra.sidecar.handlers.livemigration.LiveMigrationMap; import org.jetbrains.annotations.NotNull; import static org.apache.cassandra.sidecar.exceptions.CassandraUnavailableException.Service.CQL_AND_JMX; +import static org.apache.cassandra.sidecar.livemigration.LiveMigrationTaskManagerTestModule.DESTINATION_1; +import static org.apache.cassandra.sidecar.livemigration.LiveMigrationTaskManagerTestModule.DESTINATION_2; +import static org.apache.cassandra.sidecar.livemigration.LiveMigrationTaskManagerTestModule.DESTINATION_3; +import static org.apache.cassandra.sidecar.livemigration.LiveMigrationTaskManagerTestModule.DEST_1_ID; +import static org.apache.cassandra.sidecar.livemigration.LiveMigrationTaskManagerTestModule.DEST_2_ID; +import static org.apache.cassandra.sidecar.livemigration.LiveMigrationTaskManagerTestModule.SOURCE_1; +import static org.apache.cassandra.sidecar.livemigration.LiveMigrationTaskManagerTestModule.SOURCE_2; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; /** @@ -63,36 +65,30 @@ */ public class DataCopyTaskManagerTest { - private static final String source1Name = "source1"; - private static final String source2Name = "source2"; - private static final int dest1Id = 200001; - private static final int dest2Id = 200002; - private static final int dest3Id = 200003; - private static final String dest1Name = "destination1"; - private static final String dest2Name = "destination2"; - private static final String dest3Name = "destination3"; + private final Vertx vertx = Vertx.vertx(); private Injector getInjector() { - return Guice.createInjector(new DataCopyTaskManagerTestModule()); + return Guice.createInjector(new LiveMigrationTaskManagerTestModule(vertx)); } @Test public void getAllTasks() { Injector injector = getInjector(); - DataCopyTaskManager dataCopyTaskManager = getDataCopyTaskManager(injector); + LiveMigrationTaskManager liveMigrationTaskManager = injector.getInstance(LiveMigrationTaskManager.class); + DataCopyTaskManager dataCopyTaskManager = injector.getInstance(DataCopyTaskManager.class); - dataCopyTaskManager.currentTasks.put(dest1Id, getSucceededTask("task1", source1Name)); - dataCopyTaskManager.currentTasks.put(dest2Id, getSucceededTask("task2", source2Name)); + liveMigrationTaskManager.currentTasks.put(DEST_1_ID, getSucceededTask("task1", SOURCE_1)); + liveMigrationTaskManager.currentTasks.put(DEST_2_ID, getSucceededTask("task2", SOURCE_2)); - List tasks = dataCopyTaskManager.getAllTasks(dest1Name); + List> tasks = dataCopyTaskManager.getAllTasks(DESTINATION_1); assertThat(tasks.get(0).id()).isEqualTo("task1"); - tasks = dataCopyTaskManager.getAllTasks(dest2Name); + tasks = dataCopyTaskManager.getAllTasks(DESTINATION_2); assertThat(tasks.get(0).id()).isEqualTo("task2"); - tasks = dataCopyTaskManager.getAllTasks(dest3Name); + tasks = dataCopyTaskManager.getAllTasks(DESTINATION_3); assertThat(tasks).isEmpty(); } @@ -100,10 +96,10 @@ public void getAllTasks() public void testCreateTaskSuccess() throws InterruptedException { Injector injector = getInjector(); - DataCopyTaskManager dataCopyTaskManager = getDataCopyTaskManager(injector); + DataCopyTaskManager dataCopyTaskManager = injector.getInstance(DataCopyTaskManager.class); LiveMigrationDataCopyRequest request = new LiveMigrationDataCopyRequest(1, 1.0, 2); - Future future = dataCopyTaskManager.createTask(request, dest1Name); + Future> future = dataCopyTaskManager.createTask(request, DESTINATION_1); awaitForFuture(future); assertThat(future.succeeded()).isTrue(); @@ -111,14 +107,57 @@ public void testCreateTaskSuccess() throws InterruptedException assertThat(future.result().id()).isNotNull(); } + @Test + void testUseeDataCopyTaskWhenFilesVerificationTaskIsInProgress() throws InterruptedException + { + // Trying to create a data copy task while a files verification task in progress should not succeed + Injector injector = getInjector(); + DataCopyTaskManager dataCopyTaskManager = injector.getInstance(DataCopyTaskManager.class); + FilesVerificationTaskManager filesVerificationTaskManager = injector.getInstance(FilesVerificationTaskManager.class); + + LiveMigrationFilesVerificationTaskFactory verificationTaskFactory = injector.getInstance(LiveMigrationFilesVerificationTaskFactory.class); + when(verificationTaskFactory.create(anyString(), anyString(), anyInt(), + any(LiveMigrationFilesVerificationRequest.class), any(InstanceMetadata.class))) + .thenAnswer(invocation -> { + String id = invocation.getArgument(0); + String source = invocation.getArgument(1); + int port = invocation.getArgument(2); + + return new FakeFilesVerificationTask( + new LiveMigrationFilesVerificationResponse(id, "MD5", "IN_PROGRESS", source, port, 0, 0, 0, 0, 0, 0, 0)); + }); + + InstanceMetadata mockDest1InstanceMeta = injector.getInstance(InstancesMetadata.class).instanceFromHost(DESTINATION_1); + filesVerificationTaskManager.createTask(new LiveMigrationFilesVerificationRequest(1, "md5"), + SOURCE_1, mockDest1InstanceMeta); + + Future> future = dataCopyTaskManager.createTask( + new LiveMigrationDataCopyRequest(1, 0.4, 1), DESTINATION_1); + awaitForFuture(future); + + assertThat(future.isComplete()).isTrue(); + assertThat(future.result()).isNull(); + assertThat(future.cause()).isNotNull(); + assertThat(future.cause()).isInstanceOf(LiveMigrationTaskInProgressException.class); + + assertThat(dataCopyTaskManager.getAllTasks(DESTINATION_1)).isEmpty(); + + // Files verification task should not be usable with data copy task manager + String verificationTaskId = filesVerificationTaskManager.getAllTasks(DESTINATION_1).get(0).id(); + assertThatThrownBy(() -> dataCopyTaskManager.getTask(verificationTaskId, DESTINATION_1)) + .isInstanceOf(LiveMigrationTaskNotFoundException.class); + assertThatThrownBy(() -> dataCopyTaskManager.cancelTask(verificationTaskId, DESTINATION_1)) + .isInstanceOf(LiveMigrationTaskNotFoundException.class); + } + @Test public void testCreateTaskWithMaxConcurrencyExceeded() throws InterruptedException { Injector injector = getInjector(); - DataCopyTaskManager dataCopyTaskManager = getDataCopyTaskManager(injector); + DataCopyTaskManager dataCopyTaskManager = injector.getInstance(DataCopyTaskManager.class); LiveMigrationDataCopyRequest request = new LiveMigrationDataCopyRequest(1, 1.0, 10); // exceeds max concurrency of 5 - Future future = dataCopyTaskManager.createTask(request, dest1Name); + Future> future = dataCopyTaskManager.createTask(request, DESTINATION_1); awaitForFuture(future); assertThat(future.failed()).isTrue(); @@ -130,18 +169,19 @@ public void testCreateTaskWithMaxConcurrencyExceeded() throws InterruptedExcepti public void testCreateTaskWhenAnotherTaskInProgress() throws InterruptedException { Injector injector = getInjector(); - DataCopyTaskManager dataCopyTaskManager = getDataCopyTaskManager(injector); + LiveMigrationTaskManager liveMigrationTaskManager = injector.getInstance(LiveMigrationTaskManager.class); + DataCopyTaskManager dataCopyTaskManager = injector.getInstance(DataCopyTaskManager.class); // Add an in-progress task - LiveMigrationTask inProgressTask = getInProgressTask("existing-task"); - dataCopyTaskManager.currentTasks.put(dest1Id, inProgressTask); + LiveMigrationTask inProgressTask = getInProgressTask("existing-task"); + liveMigrationTaskManager.currentTasks.put(DEST_1_ID, inProgressTask); LiveMigrationDataCopyRequest request = new LiveMigrationDataCopyRequest(1, 1.0, 2); - Future future = dataCopyTaskManager.createTask(request, dest1Name); + Future> future = dataCopyTaskManager.createTask(request, DESTINATION_1); awaitForFuture(future); assertThat(future.failed()).isTrue(); - assertThat(future.cause()).isInstanceOf(LiveMigrationDataCopyInProgressException.class); + assertThat(future.cause()).isInstanceOf(LiveMigrationTaskInProgressException.class); assertThat(future.cause().getMessage()).contains("Another task is already under progress"); } @@ -149,14 +189,15 @@ public void testCreateTaskWhenAnotherTaskInProgress() throws InterruptedExceptio public void testCreateTaskWhenPreviousTaskCompleted() throws InterruptedException { Injector injector = getInjector(); - DataCopyTaskManager dataCopyTaskManager = getDataCopyTaskManager(injector); + LiveMigrationTaskManager liveMigrationTaskManager = injector.getInstance(LiveMigrationTaskManager.class); + DataCopyTaskManager dataCopyTaskManager = injector.getInstance(DataCopyTaskManager.class); // Add a completed task - LiveMigrationTask completedTask = getSucceededTask("completed-task", source1Name); - dataCopyTaskManager.currentTasks.put(dest1Id, completedTask); + LiveMigrationTask completedTask = getSucceededTask("completed-task", SOURCE_1); + liveMigrationTaskManager.currentTasks.put(DEST_1_ID, completedTask); LiveMigrationDataCopyRequest request = new LiveMigrationDataCopyRequest(1, 1.0, 2); - Future future = dataCopyTaskManager.createTask(request, dest1Name); + Future> future = dataCopyTaskManager.createTask(request, DESTINATION_1); awaitForFuture(future); assertThat(future.succeeded()).isTrue(); @@ -168,15 +209,15 @@ public void testCreateTaskWhenPreviousTaskCompleted() throws InterruptedExceptio public void testCreateTaskShouldFailWhenCassandraInstanceJMXIsUp() throws InterruptedException { Injector injector = getInjector(); - DataCopyTaskManager dataCopyTaskManager = getDataCopyTaskManager(injector); + DataCopyTaskManager dataCopyTaskManager = injector.getInstance(DataCopyTaskManager.class); InstancesMetadata instancesMetadata = injector.getInstance(InstancesMetadata.class); - InstanceMetadata destinationMetadata = instancesMetadata.instanceFromHost(dest1Name); + InstanceMetadata destinationMetadata = instancesMetadata.instanceFromHost(DESTINATION_1); // Mocking JMX as up when(destinationMetadata.delegate().isJmxUp()).thenReturn(true); LiveMigrationDataCopyRequest request = new LiveMigrationDataCopyRequest(1, 1.0, 2); - Future future = dataCopyTaskManager.createTask(request, dest1Name); + Future> future = dataCopyTaskManager.createTask(request, DESTINATION_1); awaitForFuture(future); assertThat(future.succeeded()).isFalse(); @@ -190,16 +231,16 @@ public void testCreateTaskShouldFailWhenCassandraInstanceJMXIsUp() throws Interr public void testCreateTaskShouldFailWhenCassandraInstanceNativeIsUp() throws InterruptedException { Injector injector = getInjector(); - DataCopyTaskManager dataCopyTaskManager = getDataCopyTaskManager(injector); + DataCopyTaskManager dataCopyTaskManager = injector.getInstance(DataCopyTaskManager.class); InstancesMetadata instancesMetadata = injector.getInstance(InstancesMetadata.class); - InstanceMetadata destinationMetadata = instancesMetadata.instanceFromHost(dest1Name); + InstanceMetadata destinationMetadata = instancesMetadata.instanceFromHost(DESTINATION_1); // Mocking native (CQL) as up but JMX as down when(destinationMetadata.delegate().isJmxUp()).thenReturn(false); when(destinationMetadata.delegate().isNativeUp()).thenReturn(true); LiveMigrationDataCopyRequest request = new LiveMigrationDataCopyRequest(1, 1.0, 2); - Future future = dataCopyTaskManager.createTask(request, dest1Name); + Future> future = dataCopyTaskManager.createTask(request, DESTINATION_1); awaitForFuture(future); assertThat(future.succeeded()).isFalse(); @@ -213,14 +254,14 @@ public void testCreateTaskShouldFailWhenCassandraInstanceNativeIsUp() throws Int public void testCreateTaskShouldSucceedWhenCassandraAdapterIsNotAvailable() throws InterruptedException { Injector injector = getInjector(); - DataCopyTaskManager dataCopyTaskManager = getDataCopyTaskManager(injector); + DataCopyTaskManager dataCopyTaskManager = injector.getInstance(DataCopyTaskManager.class); InstancesMetadata instancesMetadata = injector.getInstance(InstancesMetadata.class); - InstanceMetadata destinationMetadata = instancesMetadata.instanceFromHost(dest1Name); + InstanceMetadata destinationMetadata = instancesMetadata.instanceFromHost(DESTINATION_1); when(destinationMetadata.delegate()) .thenThrow(new CassandraUnavailableException(CQL_AND_JMX, "CassandraAdapterDelegate is not available")); LiveMigrationDataCopyRequest request = new LiveMigrationDataCopyRequest(1, 1.0, 2); - Future future = dataCopyTaskManager.createTask(request, dest1Name); + Future> future = dataCopyTaskManager.createTask(request, DESTINATION_1); awaitForFuture(future); assertThat(future.succeeded()).isTrue(); @@ -234,12 +275,13 @@ public void testCreateTaskShouldSucceedWhenCassandraAdapterIsNotAvailable() thro public void testGetTaskSuccess() { Injector injector = getInjector(); - DataCopyTaskManager dataCopyTaskManager = getDataCopyTaskManager(injector); + LiveMigrationTaskManager liveMigrationTaskManager = injector.getInstance(LiveMigrationTaskManager.class); + DataCopyTaskManager dataCopyTaskManager = injector.getInstance(DataCopyTaskManager.class); - LiveMigrationTask task = getSucceededTask("test-task", source1Name); - dataCopyTaskManager.currentTasks.put(dest1Id, task); + LiveMigrationTask task = getSucceededTask("test-task", SOURCE_1); + liveMigrationTaskManager.currentTasks.put(DEST_1_ID, task); - LiveMigrationTask retrievedTask = dataCopyTaskManager.getTask("test-task", dest1Name); + LiveMigrationTask retrievedTask = dataCopyTaskManager.getTask("test-task", DESTINATION_1); assertThat(retrievedTask).isNotNull(); assertThat(retrievedTask.id()).isEqualTo("test-task"); @@ -249,9 +291,9 @@ public void testGetTaskSuccess() public void testGetTaskNotFound() { Injector injector = getInjector(); - DataCopyTaskManager dataCopyTaskManager = getDataCopyTaskManager(injector); + DataCopyTaskManager dataCopyTaskManager = injector.getInstance(DataCopyTaskManager.class); - assertThatThrownBy(() -> dataCopyTaskManager.getTask("non-existent-task", dest1Name)) + assertThatThrownBy(() -> dataCopyTaskManager.getTask("non-existent-task", DESTINATION_1)) .isInstanceOf(LiveMigrationTaskNotFoundException.class); } @@ -259,12 +301,13 @@ public void testGetTaskNotFound() public void testGetTaskWrongTaskId() { Injector injector = getInjector(); - DataCopyTaskManager dataCopyTaskManager = getDataCopyTaskManager(injector); + LiveMigrationTaskManager liveMigrationTaskManager = injector.getInstance(LiveMigrationTaskManager.class); + DataCopyTaskManager dataCopyTaskManager = injector.getInstance(DataCopyTaskManager.class); - LiveMigrationTask task = getSucceededTask("actual-task", source1Name); - dataCopyTaskManager.currentTasks.put(dest1Id, task); + LiveMigrationTask task = getSucceededTask("actual-task", SOURCE_1); + liveMigrationTaskManager.currentTasks.put(DEST_1_ID, task); - assertThatThrownBy(() -> dataCopyTaskManager.getTask("wrong-task-id", dest1Name)) + assertThatThrownBy(() -> dataCopyTaskManager.getTask("wrong-task-id", DESTINATION_1)) .isInstanceOf(LiveMigrationTaskNotFoundException.class); } @@ -272,12 +315,13 @@ public void testGetTaskWrongTaskId() public void testCancelTaskSuccess() { Injector injector = getInjector(); - DataCopyTaskManager dataCopyTaskManager = getDataCopyTaskManager(injector); + LiveMigrationTaskManager liveMigrationTaskManager = injector.getInstance(LiveMigrationTaskManager.class); + DataCopyTaskManager dataCopyTaskManager = injector.getInstance(DataCopyTaskManager.class); - LiveMigrationTask task = getInProgressTask("cancelable-task"); - dataCopyTaskManager.currentTasks.put(dest1Id, task); + LiveMigrationTask task = getInProgressTask("cancelable-task"); + liveMigrationTaskManager.currentTasks.put(DEST_1_ID, task); - LiveMigrationTask cancelledTask = dataCopyTaskManager.cancelTask("cancelable-task", dest1Name); + LiveMigrationTask cancelledTask = dataCopyTaskManager.cancelTask("cancelable-task", DESTINATION_1); assertThat(cancelledTask).isNotNull(); assertThat(cancelledTask.id()).isEqualTo("cancelable-task"); @@ -288,9 +332,9 @@ public void testCancelTaskSuccess() public void testCancelTaskNotFound() { Injector injector = getInjector(); - DataCopyTaskManager dataCopyTaskManager = getDataCopyTaskManager(injector); + DataCopyTaskManager dataCopyTaskManager = injector.getInstance(DataCopyTaskManager.class); - assertThatThrownBy(() -> dataCopyTaskManager.cancelTask("non-existent-task", dest1Name)) + assertThatThrownBy(() -> dataCopyTaskManager.cancelTask("non-existent-task", DESTINATION_1)) .isInstanceOf(LiveMigrationTaskNotFoundException.class); } @@ -302,7 +346,8 @@ public void testCancelTaskNotFound() public void testConcurrentTaskCreation() throws InterruptedException { Injector injector = getInjector(); - DataCopyTaskManager dataCopyTaskManager = getDataCopyTaskManager(injector); + LiveMigrationTaskManager liveMigrationTaskManager = injector.getInstance(LiveMigrationTaskManager.class); + DataCopyTaskManager dataCopyTaskManager = injector.getInstance(DataCopyTaskManager.class); LiveMigrationTaskFactory mockTaskFactory = injector.getInstance(LiveMigrationTaskFactory.class); when(mockTaskFactory.create(anyString(), any(LiveMigrationDataCopyRequest.class), anyString(), anyInt(), any(InstanceMetadata.class))) @@ -318,7 +363,7 @@ public void testConcurrentTaskCreation() throws InterruptedException final AtomicInteger successCount = new AtomicInteger(0); final AtomicInteger failureCount = new AtomicInteger(0); - final ConcurrentLinkedQueue> results = new ConcurrentLinkedQueue<>(); + final ConcurrentLinkedQueue>> results = new ConcurrentLinkedQueue<>(); try { @@ -333,7 +378,7 @@ public void testConcurrentTaskCreation() throws InterruptedException LiveMigrationDataCopyRequest request = new LiveMigrationDataCopyRequest(1, 1.0, 2); - Future future = dataCopyTaskManager.createTask(request, dest1Name); + Future> future = dataCopyTaskManager.createTask(request, DESTINATION_1); results.add(future); // Use onComplete instead of awaitForFuture @@ -366,17 +411,17 @@ public void testConcurrentTaskCreation() throws InterruptedException // Verify concurrency behavior assertThat(successCount.get()).isEqualTo(1); // Only one task should succeed assertThat(failureCount.get()).isEqualTo(numberOfThreads - 1); // All others should fail - assertThat(dataCopyTaskManager.currentTasks).hasSize(1); // Only one task in the map + assertThat(liveMigrationTaskManager.currentTasks).hasSize(1); // Only one task in the map // Verify the successful task - LiveMigrationTask successfulTask = dataCopyTaskManager.currentTasks.values().iterator().next(); + LiveMigrationTask successfulTask = liveMigrationTaskManager.currentTasks.values().iterator().next(); assertThat(successfulTask).isNotNull(); assertThat(successfulTask.id()).isNotNull(); // Verify all failed tasks have the correct exception long inProgressExceptions = results.stream() .filter(Future::failed) - .mapToLong(f -> f.cause() instanceof LiveMigrationDataCopyInProgressException ? 1 : 0) + .mapToLong(f -> f.cause() instanceof LiveMigrationTaskInProgressException ? 1 : 0) .sum(); assertThat(inProgressExceptions).isEqualTo(numberOfThreads - 1); } @@ -396,104 +441,21 @@ private void awaitForFuture(Future future) throws InterruptedException latch.await(2, TimeUnit.SECONDS); } - private DataCopyTaskManager getDataCopyTaskManager(Injector injector) - { - InstancesMetadata instancesMetadata = injector.getInstance(InstancesMetadata.class); - SidecarConfiguration sidecarConfiguration = injector.getInstance(SidecarConfiguration.class); - LiveMigrationMap liveMigrationMap = injector.getInstance(LiveMigrationMap.class); - LiveMigrationTaskFactory liveMigrationTaskFactory = injector.getInstance(LiveMigrationTaskFactory.class); - - return new DataCopyTaskManager(instancesMetadata, sidecarConfiguration, liveMigrationMap, - liveMigrationTaskFactory); - } - - private LiveMigrationTask getInProgressTask(@NotNull String taskId) + private LiveMigrationTask getInProgressTask(@NotNull String taskId) { - List statusList = - List.of(new LiveMigrationTaskResponse.Status(0, "PREPARING", 500L, 1, 1, 1, 0, 0, 500L)); + List statusList = + List.of(new LiveMigrationDataCopyResponse.Status(0, "PREPARING", 500L, 1, 1, 1, 0, 0, 500L)); LiveMigrationDataCopyRequest request = new LiveMigrationDataCopyRequest(1, 1.0, 1); - LiveMigrationTaskResponse response = new LiveMigrationTaskResponse(taskId, source1Name, 9043, request, statusList); + LiveMigrationDataCopyResponse response = new LiveMigrationDataCopyResponse(taskId, SOURCE_1, 9043, request, statusList); return new FakeLiveMigrationTask(response); } - private LiveMigrationTask getSucceededTask(@NotNull String taskId, @NotNull String sourceHost) + private LiveMigrationTask getSucceededTask(@NotNull String taskId, @NotNull String sourceHost) { - List statusList = - List.of(new LiveMigrationTaskResponse.Status(0, "SUCCESS", 1000L, 1, 1, 1, 1, 0, 1000L)); + List statusList = + List.of(new LiveMigrationDataCopyResponse.Status(0, "SUCCESS", 1000L, 1, 1, 1, 1, 0, 1000L)); LiveMigrationDataCopyRequest request = new LiveMigrationDataCopyRequest(1, 1.0, 1); - LiveMigrationTaskResponse response = new LiveMigrationTaskResponse(taskId, sourceHost, 9043, request, statusList); + LiveMigrationDataCopyResponse response = new LiveMigrationDataCopyResponse(taskId, sourceHost, 9043, request, statusList); return new FakeLiveMigrationTask(response); } - - private static class DataCopyTaskManagerTestModule extends AbstractModule - { - private final LiveMigrationTaskFactory mockLiveMigrationTaskFactory = mock(LiveMigrationTaskFactory.class); - private final SidecarConfiguration mockSidecarConfiguration = mock(SidecarConfiguration.class); - private final ServiceConfiguration mockServiceConfiguration = mock(ServiceConfiguration.class); - private final LiveMigrationConfiguration mockLiveMigrationConfiguration = mock(LiveMigrationConfiguration.class); - private final LiveMigrationMap mockLiveMigrationmap = mock(LiveMigrationMap.class); - private final InstanceMetadata mockDest1InstanceMeta = mock(InstanceMetadata.class); - private final InstanceMetadata mockDest2InstanceMeta = mock(InstanceMetadata.class); - private final InstanceMetadata mockDest3InstanceMeta = mock(InstanceMetadata.class); - private final InstanceMetadata mockSourceInstanceMeta = mock(InstanceMetadata.class); - private final InstancesMetadata mockInstancesMetadata = mock(InstancesMetadata.class); - - @Override - protected void configure() - { - bind(Vertx.class).toInstance(Vertx.vertx()); - bind(LiveMigrationTaskFactory.class).toInstance(mockLiveMigrationTaskFactory); - bind(SidecarConfiguration.class).toInstance(mockSidecarConfiguration); - bind(LiveMigrationMap.class).toInstance(mockLiveMigrationmap); - bind(InstancesMetadata.class).toInstance(mockInstancesMetadata); - - // Configure SidecarConfiguration mocks - when(mockSidecarConfiguration.serviceConfiguration()).thenReturn(mockServiceConfiguration); - when(mockServiceConfiguration.port()).thenReturn(9043); - when(mockSidecarConfiguration.liveMigrationConfiguration()).thenReturn(mockLiveMigrationConfiguration); - when(mockLiveMigrationConfiguration.maxConcurrentDownloads()).thenReturn(5); - - // Configure InstanceMetadata mocks - when(mockInstancesMetadata.instanceFromHost(dest1Name)).thenReturn(mockDest1InstanceMeta); - when(mockDest1InstanceMeta.id()).thenReturn(dest1Id); - when(mockDest1InstanceMeta.dataDirs()).thenReturn(List.of("/data1", "/data2")); - when(mockDest1InstanceMeta.delegate()).thenReturn(mock(CassandraAdapterDelegate.class)); - - when(mockInstancesMetadata.instanceFromHost(dest2Name)).thenReturn(mockDest2InstanceMeta); - when(mockDest2InstanceMeta.id()).thenReturn(dest2Id); - when(mockDest2InstanceMeta.dataDirs()).thenReturn(List.of("/data1", "/data2")); - when(mockDest2InstanceMeta.delegate()).thenReturn(mock(CassandraAdapterDelegate.class)); - - when(mockInstancesMetadata.instanceFromHost(dest3Name)).thenReturn(mockDest3InstanceMeta); - when(mockDest3InstanceMeta.id()).thenReturn(dest3Id); - when(mockDest3InstanceMeta.dataDirs()).thenReturn(List.of("/data1", "/data2")); - when(mockDest3InstanceMeta.delegate()).thenReturn(mock(CassandraAdapterDelegate.class)); - - when(mockInstancesMetadata.instanceFromHost(source1Name)).thenReturn(mockSourceInstanceMeta); - when(mockSourceInstanceMeta.dataDirs()).thenReturn(List.of("/data1")); - when(mockSourceInstanceMeta.delegate()).thenReturn(mock(CassandraAdapterDelegate.class)); - - // Configure LiveMigrationTaskFactory to return fake tasks - when(mockLiveMigrationTaskFactory.create(anyString(), any(LiveMigrationDataCopyRequest.class), anyString(), anyInt(), any(InstanceMetadata.class))).thenAnswer(invocation -> { - String id = invocation.getArgument(0); - LiveMigrationDataCopyRequest request = invocation.getArgument(1); - String source = invocation.getArgument(2); - int port = invocation.getArgument(3); - - final List statusList = - List.of(new LiveMigrationTaskResponse.Status(0, "SUCCESS", 1000L, 1, 1, 1, 1, 0, 1000L)); - final LiveMigrationTaskResponse taskResponse = new LiveMigrationTaskResponse(id, source, port, request, statusList); - return new FakeLiveMigrationTask(taskResponse); - }); - - try - { - when(mockLiveMigrationmap.getSource(anyString())).thenReturn(Future.succeededFuture(source1Name)); - } - catch (Exception e) - { - throw new RuntimeException(e); - } - } - } } diff --git a/server/src/test/java/org/apache/cassandra/sidecar/livemigration/FakeFilesVerificationTask.java b/server/src/test/java/org/apache/cassandra/sidecar/livemigration/FakeFilesVerificationTask.java new file mode 100644 index 000000000..a450e9555 --- /dev/null +++ b/server/src/test/java/org/apache/cassandra/sidecar/livemigration/FakeFilesVerificationTask.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.sidecar.livemigration; + +import org.apache.cassandra.sidecar.common.response.LiveMigrationFilesVerificationResponse; +import org.apache.cassandra.sidecar.livemigration.LiveMigrationFilesVerificationTask.State; + +import static org.apache.cassandra.sidecar.livemigration.LiveMigrationFilesVerificationTask.FILES_VERIFICATION_TASK_TYPE; + +/** + * Fake implementation of LiveMigrationFilesVerificationTask for testing purposes. + */ +public class FakeFilesVerificationTask implements LiveMigrationTask +{ + private final LiveMigrationFilesVerificationResponse response; + private boolean cancelled = false; + + public FakeFilesVerificationTask(LiveMigrationFilesVerificationResponse response) + { + this.response = response; + } + + @Override + public String id() + { + return response.id(); + } + + @Override + public String type() + { + return FILES_VERIFICATION_TASK_TYPE; + } + + @Override + public void start() + { + // No-op for fake implementation - response is pre-configured + } + + @Override + public LiveMigrationFilesVerificationResponse getResponse() + { + return response; + } + + @Override + public void cancel() + { + cancelled = true; + } + + @Override + public boolean isCompleted() + { + if (cancelled) + return true; + + State state = State.valueOf(response.state()); + return state == State.COMPLETED || state == State.CANCELLED || state == State.FAILED; + } +} diff --git a/server/src/test/java/org/apache/cassandra/sidecar/livemigration/FilesVerificationTaskManagerTest.java b/server/src/test/java/org/apache/cassandra/sidecar/livemigration/FilesVerificationTaskManagerTest.java new file mode 100644 index 000000000..30c9bad30 --- /dev/null +++ b/server/src/test/java/org/apache/cassandra/sidecar/livemigration/FilesVerificationTaskManagerTest.java @@ -0,0 +1,458 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.sidecar.livemigration; + +import java.util.List; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import org.junit.jupiter.api.Test; + +import com.google.inject.Guice; +import com.google.inject.Injector; +import io.vertx.core.Future; +import io.vertx.core.Vertx; +import org.apache.cassandra.sidecar.cluster.InstancesMetadata; +import org.apache.cassandra.sidecar.cluster.instance.InstanceMetadata; +import org.apache.cassandra.sidecar.common.request.LiveMigrationDataCopyRequest; +import org.apache.cassandra.sidecar.common.request.LiveMigrationFilesVerificationRequest; +import org.apache.cassandra.sidecar.common.response.LiveMigrationDataCopyResponse; +import org.apache.cassandra.sidecar.common.response.LiveMigrationFilesVerificationResponse; +import org.apache.cassandra.sidecar.exceptions.LiveMigrationExceptions.LiveMigrationTaskInProgressException; +import org.apache.cassandra.sidecar.exceptions.LiveMigrationExceptions.LiveMigrationTaskNotFoundException; +import org.apache.cassandra.sidecar.handlers.livemigration.FakeLiveMigrationTask; +import org.jetbrains.annotations.NotNull; + +import static org.apache.cassandra.sidecar.livemigration.LiveMigrationTaskManagerTestModule.DESTINATION_1; +import static org.apache.cassandra.sidecar.livemigration.LiveMigrationTaskManagerTestModule.DESTINATION_2; +import static org.apache.cassandra.sidecar.livemigration.LiveMigrationTaskManagerTestModule.DESTINATION_3; +import static org.apache.cassandra.sidecar.livemigration.LiveMigrationTaskManagerTestModule.DEST_1_ID; +import static org.apache.cassandra.sidecar.livemigration.LiveMigrationTaskManagerTestModule.DEST_2_ID; +import static org.apache.cassandra.sidecar.livemigration.LiveMigrationTaskManagerTestModule.SOURCE_1; +import static org.apache.cassandra.sidecar.livemigration.LiveMigrationTaskManagerTestModule.SOURCE_2; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.when; + +/** + * Test class for {@link FilesVerificationTaskManager}. + */ +class FilesVerificationTaskManagerTest +{ + + private static final int PORT = 9043; + private final Vertx vertx = Vertx.vertx(); + + private Injector getInjector() + { + return Guice.createInjector(new LiveMigrationTaskManagerTestModule(vertx)); + } + + + @Test + public void testGetAllTasks() + { + Injector injector = getInjector(); + + LiveMigrationTaskManager liveMigrationTaskManager = injector.getInstance(LiveMigrationTaskManager.class); + liveMigrationTaskManager.currentTasks.put(DEST_1_ID, getSucceededTask("task1", SOURCE_1)); + liveMigrationTaskManager.currentTasks.put(DEST_2_ID, getSucceededTask("task2", SOURCE_2)); + + FilesVerificationTaskManager verificationTaskManager = injector.getInstance(FilesVerificationTaskManager.class); + List> tasks = + verificationTaskManager.getAllTasks(DESTINATION_1); + assertThat(tasks).hasSize(1); + assertThat(tasks.get(0).id()).isEqualTo("task1"); + + tasks = verificationTaskManager.getAllTasks(DESTINATION_2); + assertThat(tasks).hasSize(1); + assertThat(tasks.get(0).id()).isEqualTo("task2"); + + tasks = verificationTaskManager.getAllTasks(DESTINATION_3); + assertThat(tasks).isEmpty(); + } + + @Test + public void testCreateTaskSuccess() throws InterruptedException + { + Injector injector = getInjector(); + FilesVerificationTaskManager verificationTaskManager = injector.getInstance(FilesVerificationTaskManager.class); + InstanceMetadata mockDest1InstanceMeta = injector.getInstance(InstancesMetadata.class).instanceFromHost(DESTINATION_1); + LiveMigrationFilesVerificationRequest request = new LiveMigrationFilesVerificationRequest(2, "MD5"); + + Future> future = + verificationTaskManager.createTask(request, SOURCE_1, mockDest1InstanceMeta); + awaitForFuture(future); + + assertThat(future.succeeded()).isTrue(); + assertThat(future.result()).isNotNull(); + assertThat(future.result().id()).isNotNull(); + + LiveMigrationTaskManager liveMigrationTaskManager = injector.getInstance(LiveMigrationTaskManager.class); + assertThat(liveMigrationTaskManager.getAllTasks(mockDest1InstanceMeta.host())).hasSize(1); + } + + @Test + public void testUseFilesVerificationTaskWhenDataCopyTaskIsInProgress() throws InterruptedException + { + // Files verification task creation should not succeed when a data copy task is in progress. + + Injector injector = getInjector(); + DataCopyTaskManager dataCopyTaskManager = injector.getInstance(DataCopyTaskManager.class); + InstanceMetadata mockDest1InstanceMeta = injector.getInstance(InstancesMetadata.class).instanceFromHost(DESTINATION_1); + + LiveMigrationTaskFactory liveMigrationTaskFactory = injector.getInstance(LiveMigrationTaskFactory.class); + when(liveMigrationTaskFactory.create(anyString(), any(LiveMigrationDataCopyRequest.class), + any(), anyInt(), any(InstanceMetadata.class))) + .thenAnswer(invocationOnMock -> getInProgressDataCopyTask(invocationOnMock.getArgument(0))); + + // Create data copy task first + awaitForFuture(dataCopyTaskManager.createTask(new LiveMigrationDataCopyRequest(1, 1, 1), DESTINATION_1)); + + // Try to create verification task now + LiveMigrationFilesVerificationRequest request = new LiveMigrationFilesVerificationRequest(2, "MD5"); + FilesVerificationTaskManager verificationTaskManager = injector.getInstance(FilesVerificationTaskManager.class); + Future> future = + verificationTaskManager.createTask(request, SOURCE_1, mockDest1InstanceMeta); + awaitForFuture(future); + + assertThat(future.isComplete()).isTrue(); + assertThat(future.succeeded()).isFalse(); + assertThat(future.result()).isNull(); + assertThat(future.cause()).isNotNull().isInstanceOf(LiveMigrationTaskInProgressException.class); + + assertThat(verificationTaskManager.getAllTasks(mockDest1InstanceMeta.host())).isEmpty(); + assertThat(dataCopyTaskManager.getAllTasks(mockDest1InstanceMeta.host())).hasSize(1); + + // Data copy task ID should not be usable with files verification task manager. + String dataCopyTaskId = dataCopyTaskManager.getAllTasks(DESTINATION_1).get(0).id(); + assertThatThrownBy(() -> verificationTaskManager.getTask(dataCopyTaskId, DESTINATION_1)) + .isInstanceOf(LiveMigrationTaskNotFoundException.class); + assertThatThrownBy(() -> verificationTaskManager.cancelTask(dataCopyTaskId, DESTINATION_1)) + .isInstanceOf(LiveMigrationTaskNotFoundException.class); + } + + @Test + public void testCreateTaskWhenAnotherTaskInProgress() throws InterruptedException + { + Injector injector = getInjector(); + LiveMigrationTaskManager liveMigrationTaskManager = injector.getInstance(LiveMigrationTaskManager.class); + FilesVerificationTaskManager verificationTaskManager = injector.getInstance(FilesVerificationTaskManager.class); + InstanceMetadata mockDest1InstanceMeta = injector.getInstance(InstancesMetadata.class).instanceFromHost(DESTINATION_1); + + // Add an in-progress task + LiveMigrationTask inProgressTask = getInProgressTask("existing-task"); + liveMigrationTaskManager.currentTasks.put(DEST_1_ID, inProgressTask); + + LiveMigrationFilesVerificationRequest request = new LiveMigrationFilesVerificationRequest(2, "MD5"); + Future> future = verificationTaskManager.createTask(request, SOURCE_1, mockDest1InstanceMeta); + awaitForFuture(future); + + assertThat(future.failed()).isTrue(); + assertThat(future.cause()).isInstanceOf(LiveMigrationTaskInProgressException.class); + assertThat(future.cause().getMessage()).contains("Another files digest verification is in progress"); + } + + @Test + public void testCreateTaskWhenPreviousTaskCompleted() throws InterruptedException + { + Injector injector = getInjector(); + LiveMigrationTaskManager liveMigrationTaskManager = injector.getInstance(LiveMigrationTaskManager.class); + FilesVerificationTaskManager verificationTaskManager = injector.getInstance(FilesVerificationTaskManager.class); + InstanceMetadata mockDest1InstanceMeta = injector.getInstance(InstancesMetadata.class).instanceFromHost(DESTINATION_1); + + // Add a completed task + LiveMigrationTask completedTask = + getSucceededTask("completed-task", SOURCE_1); + liveMigrationTaskManager.currentTasks.put(DEST_1_ID, completedTask); + + LiveMigrationFilesVerificationRequest request = new LiveMigrationFilesVerificationRequest(2, "MD5"); + Future> future = + verificationTaskManager.createTask(request, SOURCE_1, mockDest1InstanceMeta); + awaitForFuture(future); + + assertThat(future.succeeded()).isTrue(); + assertThat(future.result()).isNotNull(); + assertThat(future.result().id()).isNotEqualTo("completed-task"); + } + + /** + * Tests that multiple tasks can be created sequentially after each previous task completes. + * This validates that the system properly allows new task creation once a task has finished, + * and that each sequential task gets a unique ID and properly replaces the previous one. + */ + @Test + public void testMultipleSequentialTaskCreations() throws InterruptedException + { + Injector injector = getInjector(); + FilesVerificationTaskManager verificationTaskManager = injector.getInstance(FilesVerificationTaskManager.class); + LiveMigrationTaskManager liveMigrationTaskManager = injector.getInstance(LiveMigrationTaskManager.class); + InstanceMetadata mockDest1InstanceMeta = injector.getInstance(InstancesMetadata.class).instanceFromHost(DESTINATION_1); + + LiveMigrationFilesVerificationTaskFactory mockTaskFactory = injector.getInstance(LiveMigrationFilesVerificationTaskFactory.class); + when(mockTaskFactory.create(anyString(), anyString(), anyInt(), any(LiveMigrationFilesVerificationRequest.class), any(InstanceMetadata.class))) + .thenAnswer(invocation -> { + String taskId = invocation.getArgument(0); + String sourceHost = invocation.getArgument(1); + return getSucceededTask(taskId, sourceHost); + }); + + final int numberOfSequentialTasks = 5; + ConcurrentLinkedQueue taskIds = new ConcurrentLinkedQueue<>(); + + for (int i = 0; i < numberOfSequentialTasks; i++) + { + LiveMigrationFilesVerificationRequest request = new LiveMigrationFilesVerificationRequest(2, "MD5"); + Future> future = + verificationTaskManager.createTask(request, SOURCE_1, mockDest1InstanceMeta); + awaitForFuture(future); + + assertThat(future.succeeded()).isTrue(); + assertThat(future.result()).isNotNull(); + assertThat(future.result().id()).isNotNull(); + + String currentTaskId = future.result().id(); + taskIds.add(currentTaskId); + + // Verify the task is in the manager + assertThat(liveMigrationTaskManager.getAllTasks(mockDest1InstanceMeta.host())).hasSize(1); + assertThat(liveMigrationTaskManager.getAllTasks(mockDest1InstanceMeta.host()).get(0).id()) + .isEqualTo(currentTaskId); + + // Verify task is marked as completed + LiveMigrationTask currentTask = + verificationTaskManager.getTask(currentTaskId, DESTINATION_1); + assertThat(currentTask.isCompleted()).isTrue(); + } + + // Verify all task IDs are unique + assertThat(taskIds).doesNotHaveDuplicates(); + assertThat(taskIds).hasSize(numberOfSequentialTasks); + + // Verify the last task is still in the manager + assertThat(liveMigrationTaskManager.getAllTasks(mockDest1InstanceMeta.host())).hasSize(1); + } + + @Test + public void testGetTaskSuccess() + { + Injector injector = getInjector(); + LiveMigrationTaskManager liveMigrationTaskManager = injector.getInstance(LiveMigrationTaskManager.class); + FilesVerificationTaskManager verificationTaskManager = injector.getInstance(FilesVerificationTaskManager.class); + + LiveMigrationTask task = getSucceededTask("test-task", SOURCE_1); + liveMigrationTaskManager.currentTasks.put(DEST_1_ID, task); + + LiveMigrationTask retrievedTask = + verificationTaskManager.getTask("test-task", DESTINATION_1); + + assertThat(retrievedTask).isNotNull(); + assertThat(retrievedTask.id()).isEqualTo("test-task"); + + assertThat(retrievedTask.id()).isEqualTo(verificationTaskManager.getAllTasks(DESTINATION_1).get(0).id()); + } + + @Test + public void testGetTaskNotFound() + { + Injector injector = getInjector(); + FilesVerificationTaskManager verificationTaskManager = injector.getInstance(FilesVerificationTaskManager.class); + + assertThatThrownBy(() -> verificationTaskManager.getTask("non-existent-task", DESTINATION_1)) + .isInstanceOf(LiveMigrationTaskNotFoundException.class); + } + + @Test + public void testGetTaskWrongTaskId() + { + Injector injector = getInjector(); + LiveMigrationTaskManager liveMigrationTaskManager = injector.getInstance(LiveMigrationTaskManager.class); + FilesVerificationTaskManager verificationTaskManager = injector.getInstance(FilesVerificationTaskManager.class); + + LiveMigrationTask task = getSucceededTask("actual-task", SOURCE_1); + liveMigrationTaskManager.currentTasks.put(DEST_1_ID, task); + + assertThatThrownBy(() -> verificationTaskManager.getTask("wrong-task-id", DESTINATION_1)) + .isInstanceOf(LiveMigrationTaskNotFoundException.class); + } + + @Test + public void testCancelTaskSuccess() + { + Injector injector = getInjector(); + LiveMigrationTaskManager liveMigrationTaskManager = injector.getInstance(LiveMigrationTaskManager.class); + FilesVerificationTaskManager verificationTaskManager = injector.getInstance(FilesVerificationTaskManager.class); + + LiveMigrationTask task = getInProgressTask("cancelable-task"); + liveMigrationTaskManager.currentTasks.put(DEST_1_ID, task); + + LiveMigrationTask cancelledTask = + verificationTaskManager.cancelTask("cancelable-task", DESTINATION_1); + + assertThat(cancelledTask).isNotNull(); + assertThat(cancelledTask.id()).isEqualTo("cancelable-task"); + assertThat(cancelledTask.isCompleted()).isTrue(); // FakeLiveMigrationFilesVerificationTask returns true when cancelled + } + + @Test + public void testCancelTaskNotFound() + { + Injector injector = getInjector(); + FilesVerificationTaskManager verificationTaskManager = injector.getInstance(FilesVerificationTaskManager.class); + + assertThatThrownBy(() -> verificationTaskManager.cancelTask("non-existent-task", DESTINATION_1)) + .isInstanceOf(LiveMigrationTaskNotFoundException.class); + } + + /** + * Tests concurrent task creation to ensure only one task can be active at a time per instance. + * This test specifically validates the atomic compute() operation in createTask() method. + */ + @Test + public void testConcurrentTaskCreation() throws InterruptedException + { + Injector injector = getInjector(); + LiveMigrationTaskManager liveMigrationTaskManager = injector.getInstance(LiveMigrationTaskManager.class); + FilesVerificationTaskManager verificationTaskManager = injector.getInstance(FilesVerificationTaskManager.class); + InstanceMetadata mockDest1InstanceMeta = injector.getInstance(InstancesMetadata.class).instanceFromHost(DESTINATION_1); + + LiveMigrationFilesVerificationTaskFactory mockTaskFactory = injector.getInstance(LiveMigrationFilesVerificationTaskFactory.class); + when(mockTaskFactory.create(anyString(), anyString(), anyInt(), any(LiveMigrationFilesVerificationRequest.class), any(InstanceMetadata.class))) + .thenAnswer(invocation -> { + String taskId = invocation.getArgument(0); + return getInProgressTask(taskId); + }); + + final int numberOfThreads = 20; + final ExecutorService executor = Executors.newFixedThreadPool(numberOfThreads); + final CountDownLatch startLatch = new CountDownLatch(1); + final CountDownLatch completionLatch = new CountDownLatch(numberOfThreads); + + final AtomicInteger successCount = new AtomicInteger(0); + final AtomicInteger failureCount = new AtomicInteger(0); + final ConcurrentLinkedQueue>> results = new ConcurrentLinkedQueue<>(); + + try + { + // Launch multiple threads that all try to create tasks simultaneously + for (int i = 0; i < numberOfThreads; i++) + { + executor.submit(() -> { + try + { + // Wait for all threads to be ready before starting + startLatch.await(); + + LiveMigrationFilesVerificationRequest request + = new LiveMigrationFilesVerificationRequest(2, "MD5"); + Future> future = verificationTaskManager.createTask(request, SOURCE_1, mockDest1InstanceMeta); + results.add(future); + + // Use onComplete instead of awaitForFuture + future.onComplete(result -> { + if (result.succeeded()) + { + successCount.incrementAndGet(); + } + else + { + failureCount.incrementAndGet(); + } + completionLatch.countDown(); + }); + } + catch (Exception e) + { + failureCount.incrementAndGet(); + completionLatch.countDown(); + } + }); + } + + // Release all threads at once to maximize concurrency + startLatch.countDown(); + + // Wait for all tasks to complete + assertThat(completionLatch.await(5, TimeUnit.SECONDS)).isTrue(); + + // Verify concurrency behavior + assertThat(successCount.get()).isEqualTo(1); // Only one task should succeed + assertThat(failureCount.get()).isEqualTo(numberOfThreads - 1); // All others should fail + assertThat(liveMigrationTaskManager.currentTasks).hasSize(1); // Only one task in the map + + // Verify the successful task + LiveMigrationTask successfulTask = liveMigrationTaskManager.currentTasks.values().iterator().next(); + assertThat(successfulTask).isNotNull(); + assertThat(successfulTask.id()).isNotNull(); + + // Verify all failed tasks have the correct exception + long inProgressExceptions = results.stream() + .filter(Future::failed) + .mapToLong(f -> f.cause() instanceof LiveMigrationTaskInProgressException ? 1 : 0) + .sum(); + assertThat(inProgressExceptions).isEqualTo(numberOfThreads - 1); + } + finally + { + executor.shutdown(); + assertThat(executor.awaitTermination(1, TimeUnit.SECONDS)).isTrue(); + } + } + + @SuppressWarnings("ResultOfMethodCallIgnored") + private void awaitForFuture(Future future) throws InterruptedException + { + CountDownLatch latch = new CountDownLatch(1); + future.onComplete(res -> latch.countDown()); + + latch.await(5, TimeUnit.SECONDS); + } + + private LiveMigrationTask getInProgressTask(@NotNull String taskId) + { + LiveMigrationFilesVerificationResponse response = new LiveMigrationFilesVerificationResponse( + taskId, "MD5", "IN_PROGRESS", SOURCE_1, PORT, 0, 0, 0, 0, 0, 0, 0 + ); + return new FakeFilesVerificationTask(response); + } + + private LiveMigrationTask getSucceededTask(@NotNull String taskId, @NotNull String sourceHost) + { + LiveMigrationFilesVerificationResponse response = new LiveMigrationFilesVerificationResponse( + taskId, "MD5", "COMPLETED", sourceHost, PORT, 0, 0, 10, 0, 0, 0, 10 + ); + return new FakeFilesVerificationTask(response); + } + + private LiveMigrationTask getInProgressDataCopyTask(@NotNull String taskId) + { + List statusList = + List.of(new LiveMigrationDataCopyResponse.Status(0, "PREPARING", 500L, 1, 1, 1, 0, 0, 500L)); + LiveMigrationDataCopyRequest request = new LiveMigrationDataCopyRequest(1, 1.0, 1); + LiveMigrationDataCopyResponse response = new LiveMigrationDataCopyResponse(taskId, SOURCE_1, PORT, request, statusList); + return new FakeLiveMigrationTask(response); + } +} diff --git a/server/src/test/java/org/apache/cassandra/sidecar/livemigration/LiveMigrationTaskImplTest.java b/server/src/test/java/org/apache/cassandra/sidecar/livemigration/LiveMigrationDataCopyTaskTest.java similarity index 86% rename from server/src/test/java/org/apache/cassandra/sidecar/livemigration/LiveMigrationTaskImplTest.java rename to server/src/test/java/org/apache/cassandra/sidecar/livemigration/LiveMigrationDataCopyTaskTest.java index 1ebe83ed2..3266ad5eb 100644 --- a/server/src/test/java/org/apache/cassandra/sidecar/livemigration/LiveMigrationTaskImplTest.java +++ b/server/src/test/java/org/apache/cassandra/sidecar/livemigration/LiveMigrationDataCopyTaskTest.java @@ -28,7 +28,7 @@ import org.apache.cassandra.sidecar.client.SidecarClient; import org.apache.cassandra.sidecar.cluster.instance.InstanceMetadata; import org.apache.cassandra.sidecar.common.request.LiveMigrationDataCopyRequest; -import org.apache.cassandra.sidecar.common.response.LiveMigrationTaskResponse; +import org.apache.cassandra.sidecar.common.response.LiveMigrationDataCopyResponse; import org.apache.cassandra.sidecar.concurrent.ExecutorPools; import org.apache.cassandra.sidecar.config.LiveMigrationConfiguration; import org.apache.cassandra.sidecar.utils.SidecarClientProvider; @@ -39,17 +39,17 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -class LiveMigrationTaskImplTest +class LiveMigrationDataCopyTaskTest { private static final String SOURCE = "127.0.0.1"; private static final int PORT = 9043; - private LiveMigrationTaskImpl createTask() + private LiveMigrationDataCopyTask createTask() { return createTask("test-task-id", new LiveMigrationDataCopyRequest(5, 0.8, 10)); } - private LiveMigrationTaskImpl createTask(String id, LiveMigrationDataCopyRequest request) + private LiveMigrationDataCopyTask createTask(String id, LiveMigrationDataCopyRequest request) { Vertx vertx = mock(Vertx.class); SidecarClientProvider sidecarClientProvider = mock(SidecarClientProvider.class); @@ -61,14 +61,14 @@ private LiveMigrationTaskImpl createTask(String id, LiveMigrationDataCopyRequest ExecutorPools executorPools = ExecutorPoolsHelper.createdSharedTestPool(vertx); - return new LiveMigrationTaskImpl(vertx, executorPools, sidecarClientProvider, liveMigrationConfiguration, - id, request, SOURCE, PORT, instanceMetadata, LiveMigrationFileDownloadPreCheck.DEFAULT); + return new LiveMigrationDataCopyTask(vertx, executorPools, sidecarClientProvider, liveMigrationConfiguration, + id, request, SOURCE, PORT, instanceMetadata, LiveMigrationFileDownloadPreCheck.DEFAULT); } @Test void testCancelWithoutPromiseOrDownloader() { - LiveMigrationTaskImpl task = createTask(); + LiveMigrationDataCopyTask task = createTask(); task.cancel(); @@ -78,7 +78,7 @@ void testCancelWithoutPromiseOrDownloader() @Test void testCancelWithPromise() { - LiveMigrationTaskImpl task = createTask(); + LiveMigrationDataCopyTask task = createTask(); task.promise = Promise.promise(); task.cancel(); @@ -91,7 +91,7 @@ void testCancelWithPromise() @Test void testCancelWithDownloader() { - LiveMigrationTaskImpl task = createTask(); + LiveMigrationDataCopyTask task = createTask(); LiveMigrationFileDownloader mockDownloader = mock(LiveMigrationFileDownloader.class); task.downloader = mockDownloader; @@ -104,7 +104,7 @@ void testCancelWithDownloader() @Test void testCancelAlreadyCancelled() { - LiveMigrationTaskImpl task = createTask(); + LiveMigrationDataCopyTask task = createTask(); LiveMigrationFileDownloader mockDownloader = mock(LiveMigrationFileDownloader.class); task.downloader = mockDownloader; @@ -117,9 +117,9 @@ void testCancelAlreadyCancelled() @Test void testGetResponseEmptyStatusMap() { - LiveMigrationTaskImpl task = createTask(); + LiveMigrationDataCopyTask task = createTask(); - LiveMigrationTaskResponse response = task.getResponse(); + LiveMigrationDataCopyResponse response = task.getResponse(); assertThat(response).isNotNull(); assertThat(response.taskId()).isEqualTo("test-task-id"); @@ -131,7 +131,7 @@ void testGetResponseEmptyStatusMap() @Test void testGetResponseWithOperationStatusUpdates() { - LiveMigrationTaskImpl task = createTask(); + LiveMigrationDataCopyTask task = createTask(); // Create actual OperationStatus objects using state transitions OperationStatus status1 = OperationStatus.startingState() @@ -155,17 +155,17 @@ void testGetResponseWithOperationStatusUpdates() task.statusUpdater(0).accept(status1); task.statusUpdater(1).accept(status2); - LiveMigrationTaskResponse response = task.getResponse(); + LiveMigrationDataCopyResponse response = task.getResponse(); assertThat(response).isNotNull(); assertThat(response.taskId()).isEqualTo("test-task-id"); assertThat(response.source()).isEqualTo(SOURCE); assertThat(response.port()).isEqualTo(PORT); - List statusList = response.status(); + List statusList = response.status(); assertThat(statusList).hasSize(2); - LiveMigrationTaskResponse.Status responseStatus1 = statusList.get(0); + LiveMigrationDataCopyResponse.Status responseStatus1 = statusList.get(0); assertThat(responseStatus1.iteration()).isEqualTo(0); assertThat(responseStatus1.state()).isEqualTo("DOWNLOADING"); assertThat(responseStatus1.totalSize()).isEqualTo(1000L); @@ -176,7 +176,7 @@ void testGetResponseWithOperationStatusUpdates() assertThat(responseStatus1.downloadFailures()).isEqualTo(2); assertThat(responseStatus1.bytesDownloaded()).isEqualTo(300L); - LiveMigrationTaskResponse.Status responseStatus2 = statusList.get(1); + LiveMigrationDataCopyResponse.Status responseStatus2 = statusList.get(1); assertThat(responseStatus2.iteration()).isEqualTo(1); assertThat(responseStatus2.state()).isEqualTo("DOWNLOAD_COMPLETE"); assertThat(responseStatus2.totalSize()).isEqualTo(2000L); @@ -191,7 +191,7 @@ void testGetResponseWithOperationStatusUpdates() @Test void testId() { - LiveMigrationTaskImpl task = createTask(); + LiveMigrationDataCopyTask task = createTask(); assertThat(task.id()).isEqualTo("test-task-id"); } @@ -199,7 +199,7 @@ void testId() @Test void testIsCompletedNotStarted() { - LiveMigrationTaskImpl task = createTask(); + LiveMigrationDataCopyTask task = createTask(); assertThat(task.isCompleted()).isFalse(); } @@ -207,7 +207,7 @@ void testIsCompletedNotStarted() @Test void testIsCompletedCancelled() { - LiveMigrationTaskImpl task = createTask(); + LiveMigrationDataCopyTask task = createTask(); task.cancel(); assertThat(task.isCompleted()).isTrue(); @@ -216,7 +216,7 @@ void testIsCompletedCancelled() @Test void testGetResponseWithMultipleStateTransitions() { - LiveMigrationTaskImpl task = createTask(); + LiveMigrationDataCopyTask task = createTask(); // Test with different state combinations OperationStatus failedStatus = OperationStatus.startingState() @@ -238,8 +238,8 @@ void testGetResponseWithMultipleStateTransitions() task.statusUpdater(1).accept(cancelledStatus); task.statusUpdater(2).accept(downloadCompleteStatus); - LiveMigrationTaskResponse response = task.getResponse(); - List statusList = response.status(); + LiveMigrationDataCopyResponse response = task.getResponse(); + List statusList = response.status(); assertThat(statusList).hasSize(3); assertThat(statusList.get(0).state()).isEqualTo("FAILED"); @@ -250,7 +250,7 @@ void testGetResponseWithMultipleStateTransitions() @Test void testGetResponseDownloadsInProgress() { - LiveMigrationTaskImpl task = createTask(); + LiveMigrationDataCopyTask task = createTask(); // Test with different state combinations OperationStatus downloadingState = OperationStatus.startingState() @@ -261,7 +261,7 @@ void testGetResponseDownloadsInProgress() task.statusUpdater(0).accept(downloadingState); - List statusList = task.getResponse().status(); + List statusList = task.getResponse().status(); assertThat(statusList).hasSize(1); assertThat(statusList.get(0).state()).isEqualTo("DOWNLOADING"); diff --git a/server/src/test/java/org/apache/cassandra/sidecar/livemigration/LiveMigrationFileDownloaderTest.java b/server/src/test/java/org/apache/cassandra/sidecar/livemigration/LiveMigrationFileDownloaderTest.java index 7606d2088..370ae2649 100644 --- a/server/src/test/java/org/apache/cassandra/sidecar/livemigration/LiveMigrationFileDownloaderTest.java +++ b/server/src/test/java/org/apache/cassandra/sidecar/livemigration/LiveMigrationFileDownloaderTest.java @@ -20,7 +20,6 @@ import java.io.File; import java.io.IOException; -import java.io.RandomAccessFile; import java.nio.file.Files; import java.nio.file.NoSuchFileException; import java.nio.file.Path; @@ -66,7 +65,6 @@ import org.apache.cassandra.sidecar.common.response.LiveMigrationStatus; import org.apache.cassandra.sidecar.config.LiveMigrationConfiguration; import org.apache.cassandra.sidecar.config.SidecarConfiguration; -import org.apache.cassandra.sidecar.handlers.livemigration.LiveMigrationDirType; import org.apache.cassandra.sidecar.utils.SidecarClientProvider; import org.jetbrains.annotations.NotNull; @@ -76,6 +74,7 @@ import static org.apache.cassandra.sidecar.handlers.livemigration.LiveMigrationDirType.DATA_FILE_DIR; import static org.apache.cassandra.sidecar.handlers.livemigration.LiveMigrationDirType.HINTS_DIR; import static org.apache.cassandra.sidecar.livemigration.LiveMigrationInstanceMetadataUtil.localPath; +import static org.apache.cassandra.sidecar.utils.TestFileUtils.createFile; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.ArgumentMatchers.any; @@ -1274,6 +1273,82 @@ void testCanDeleteWhenFileTimestampsDoNotMatch(@TempDir Path tempDir) throws IOE assertThat(downloaderSpy.canDelete(fileAttrsToCheck, Path.of(rightTimestampFileLocalPath), dataDirPath)).isFalse(); } + @Test + void testExcludedFileNotDeletedWhenMissingFromRemote(@TempDir Path tempDir) throws IOException + { + String storageDir = tempDir.resolve("testExcludedFileNotDeletedWhenMissingFromRemote") + .toAbsolutePath().toString(); + List dataDirs = getDataDirList(storageDir); + int fileSize = 32; + long timestamp = System.currentTimeMillis(); + + // Create files that exist locally + TestFile excludedFile = new TestFile(DATA_FILE_DIR, 0, "ks1/t1/excluded.db", fileSize, timestamp); + TestFile normalFile = new TestFile(DATA_FILE_DIR, 0, "ks1/t1/normal.db", fileSize, timestamp); + prepareDataHomeDir(storageDir, List.of(excludedFile, normalFile)); + + // Create remote files list that doesn't include the excluded file or normal file + TestFile remoteFile = new TestFile(DATA_FILE_DIR, 0, "ks1/t2/remote.db", fileSize, timestamp); + TestFile remoteKeyspaceDir = new TestFile(DATA_FILE_DIR, 0, "ks1", -1, timestamp); + TestFile remoteTableDir = new TestFile(DATA_FILE_DIR, 0, "ks1/t1", -1, timestamp); + List remoteFiles = List.of(remoteFile, remoteKeyspaceDir, remoteTableDir); + + Consumer mockStatusUpdater = mock(Consumer.class); + + // Create injector with excluded files configuration + Injector injector = getInjectorWithExcludedFiles(Set.of("glob:${DATA_FILE_DIR}/**/excluded.db"), Set.of()); + + LiveMigrationFileDownloader downloaderSpy = + getDownloaderSpy(injector, dummyRequest100pThreshold, 1, mockStatusUpdater, storageDir, dataDirs); + + InstanceFilesListResponse instanceFilesListResponse = getInstanceFilesListResponse(remoteFiles); + + InstanceFilesListResponse responseFuture = downloaderSpy.deleteUnnecessaryFilesAndDirectories(instanceFilesListResponse); + + assertThat(responseFuture).isEqualTo(instanceFilesListResponse); + // Verify the excluded file still exists even though it's not in the remote list + assertFileExists(excludedFile.getFilePath(storageDir), excludedFile.size); + // Verify the normal file was deleted as expected since it's not excluded and not in remote + assertFileDoesNotExists(normalFile.getFilePath(storageDir)); + } + + @Test + void testExcludedDirectoryNotDeletedWhenMissingFromRemote(@TempDir Path tempDir) throws IOException + { + String storageDir = tempDir.resolve("testExcludedDirectoryNotDeletedWhenMissingFromRemote") + .toAbsolutePath().toString(); + List dataDirs = getDataDirList(storageDir); + long timestamp = System.currentTimeMillis(); + + // Create directories that exist locally + TestFile excludedDir = new TestFile(DATA_FILE_DIR, 0, "ks1/excluded_table", -1, timestamp); + TestFile normalDir = new TestFile(DATA_FILE_DIR, 0, "ks1/normal_table", -1, timestamp); + prepareDataHomeDir(storageDir, List.of(excludedDir, normalDir)); + + // Create remote files list that doesn't include the excluded directory or normal directory + TestFile remoteKeyspaceDir = new TestFile(DATA_FILE_DIR, 0, "ks1", -1, timestamp); + TestFile remoteTableDir = new TestFile(DATA_FILE_DIR, 0, "ks1/t1", -1, timestamp); + List remoteFiles = List.of(remoteKeyspaceDir, remoteTableDir); + + Consumer mockStatusUpdater = mock(Consumer.class); + + // Create injector with excluded directories configuration + Injector injector = getInjectorWithExcludedFiles(Set.of(), Set.of("glob:${DATA_FILE_DIR}/**/excluded_table")); + + LiveMigrationFileDownloader downloaderSpy = + getDownloaderSpy(injector, dummyRequest100pThreshold, 1, mockStatusUpdater, storageDir, dataDirs); + + InstanceFilesListResponse instanceFilesListResponse = getInstanceFilesListResponse(remoteFiles); + + InstanceFilesListResponse responseFuture = downloaderSpy.deleteUnnecessaryFilesAndDirectories(instanceFilesListResponse); + + assertThat(responseFuture).isEqualTo(instanceFilesListResponse); + // Verify the excluded directory still exists even though it's not in the remote list + assertDirExists(excludedDir.getFilePath(storageDir)); + // Verify the normal directory was deleted as expected since it's not excluded and not in remote + assertFileDoesNotExists(normalDir.getFilePath(storageDir)); + } + void assertFileExists(String dataDir, String relativeFilePath, int expectedFileSize) { File file = new File(dataDir + "/" + relativeFilePath); @@ -1317,23 +1392,6 @@ void prepareDataHomeDir(String storageDir, List testFiles) throws IOEx } } - void createFile(String file, int size, long lastModifiedTime) throws IOException - { - createFile(new File(file), size, lastModifiedTime); - } - - @SuppressWarnings("ResultOfMethodCallIgnored") - void createFile(File f, int size, long lastModifiedTime) throws IOException - { - f.getParentFile().mkdirs(); - RandomAccessFile randomAccessFile = new RandomAccessFile(f, "rw"); - randomAccessFile.seek(size - 1); - randomAccessFile.write(0); - randomAccessFile.close(); - f.setLastModified(lastModifiedTime); - } - - @Test void testShortlistDownloadFiles(@TempDir Path tmpDir) throws IOException { @@ -1396,6 +1454,11 @@ Injector getInjector() return Guice.createInjector(new LiveMigrationFileDownloaderTestModule()); } + Injector getInjectorWithExcludedFiles(Set filesToExclude, Set directoriesToExclude) + { + return Guice.createInjector(new LiveMigrationFileDownloaderTestModule(filesToExclude, directoriesToExclude)); + } + LiveMigrationFileDownloader getDownloader(Injector injector, LiveMigrationDataCopyRequest request, int currentIteration, @@ -1502,51 +1565,25 @@ List getInstanceFileInfo(List testFiles) { return testFiles.stream().map(TestFile::getInstanceFileInfo).collect(Collectors.toList()); } - - private static class TestFile + private static class LiveMigrationFileDownloaderTestModule extends AbstractModule { - final LiveMigrationDirType dirType; - final int dirIndex; - final String relativePath; - final int size; - final long lastModifiedTime; - - public TestFile(LiveMigrationDirType dirType, int dirIndex, String relativePath, int size, long lastModifiedTime) - { - this.dirType = dirType; - this.dirIndex = dirIndex; - this.relativePath = relativePath; - this.size = size; - this.lastModifiedTime = lastModifiedTime; - } - - InstanceFileInfo getInstanceFileInfo() - { - return new InstanceFileInfo(getFileUrl(), size, getFileType(), lastModifiedTime); - } - - String getFileUrl() - { - return LIVE_MIGRATION_FILES_ROUTE + "/" + dirType.dirType + "/" + dirIndex + "/" + relativePath; - } + SidecarClient sidecarClient = mock(SidecarClient.class); + SidecarClientProvider sidecarClientProvider = mock(SidecarClientProvider.class); + SidecarConfiguration mockSidecarConfiguration = mock(SidecarConfiguration.class); + LiveMigrationConfiguration mockLiveMigrationConfig = mock(LiveMigrationConfiguration.class); + private final Set filesToExclude; + private final Set directoriesToExclude; - FileType getFileType() + LiveMigrationFileDownloaderTestModule() { - return -1 == size ? FileType.DIRECTORY : FileType.FILE; + this(Set.of(), Set.of()); } - String getFilePath(String storageDir) + LiveMigrationFileDownloaderTestModule(Set filesToExclude, Set directoriesToExclude) { - return storageDir + "/" + dirType.dirType + "/" + relativePath; + this.filesToExclude = filesToExclude; + this.directoriesToExclude = directoriesToExclude; } - } - - private static class LiveMigrationFileDownloaderTestModule extends AbstractModule - { - SidecarClient sidecarClient = mock(SidecarClient.class); - SidecarClientProvider sidecarClientProvider = mock(SidecarClientProvider.class); - SidecarConfiguration mockSidecarConfiguration = mock(SidecarConfiguration.class); - LiveMigrationConfiguration mockLiveMigrationConfig = mock(LiveMigrationConfiguration.class); @Override protected void configure() @@ -1559,8 +1596,8 @@ protected void configure() .thenReturn(CompletableFuture.completedFuture(new LiveMigrationStatus(NOT_COMPLETED, 1L))); when(sidecarClientProvider.get()).thenReturn(sidecarClient); when(mockSidecarConfiguration.liveMigrationConfiguration()).thenReturn(mockLiveMigrationConfig); - when(mockLiveMigrationConfig.filesToExclude()).thenReturn(Set.of()); - when(mockLiveMigrationConfig.directoriesToExclude()).thenReturn(Set.of()); + when(mockLiveMigrationConfig.filesToExclude()).thenReturn(filesToExclude); + when(mockLiveMigrationConfig.directoriesToExclude()).thenReturn(directoriesToExclude); } } } diff --git a/server/src/test/java/org/apache/cassandra/sidecar/livemigration/LiveMigrationFilesVerificationTaskTest.java b/server/src/test/java/org/apache/cassandra/sidecar/livemigration/LiveMigrationFilesVerificationTaskTest.java new file mode 100644 index 000000000..f9427e1c1 --- /dev/null +++ b/server/src/test/java/org/apache/cassandra/sidecar/livemigration/LiveMigrationFilesVerificationTaskTest.java @@ -0,0 +1,1033 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.sidecar.livemigration; + + +import java.io.IOException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.io.TempDir; + +import com.google.inject.AbstractModule; +import com.google.inject.Guice; +import com.google.inject.Injector; +import io.vertx.core.Future; +import io.vertx.core.Promise; +import io.vertx.core.Vertx; +import io.vertx.junit5.VertxExtension; +import org.apache.cassandra.sidecar.HelperTestModules.DigestAlgorithmProviderTestModule; +import org.apache.cassandra.sidecar.client.SidecarClient; +import org.apache.cassandra.sidecar.client.SidecarInstance; +import org.apache.cassandra.sidecar.client.SidecarInstanceImpl; +import org.apache.cassandra.sidecar.cluster.instance.InstanceMetadata; +import org.apache.cassandra.sidecar.common.request.LiveMigrationFilesVerificationRequest; +import org.apache.cassandra.sidecar.common.response.DigestResponse; +import org.apache.cassandra.sidecar.common.response.InstanceFileInfo.FileType; +import org.apache.cassandra.sidecar.common.response.InstanceFilesListResponse; +import org.apache.cassandra.sidecar.common.response.LiveMigrationFilesVerificationResponse; +import org.apache.cassandra.sidecar.concurrent.ExecutorPools; +import org.apache.cassandra.sidecar.config.LiveMigrationConfiguration; +import org.apache.cassandra.sidecar.config.yaml.LiveMigrationConfigurationImpl; +import org.apache.cassandra.sidecar.config.yaml.ServiceConfigurationImpl; +import org.apache.cassandra.sidecar.handlers.livemigration.InstanceMetadataTestUtil; +import org.apache.cassandra.sidecar.livemigration.LiveMigrationFilesVerificationTask.State; +import org.apache.cassandra.sidecar.utils.DigestAlgorithm; +import org.apache.cassandra.sidecar.utils.DigestAlgorithmFactory; +import org.apache.cassandra.sidecar.utils.DigestVerifierFactory; +import org.jetbrains.annotations.NotNull; + +import static org.apache.cassandra.sidecar.handlers.livemigration.LiveMigrationDirType.CDC_RAW_DIR; +import static org.apache.cassandra.sidecar.handlers.livemigration.LiveMigrationDirType.COMMIT_LOG_DIR; +import static org.apache.cassandra.sidecar.handlers.livemigration.LiveMigrationDirType.DATA_FILE_DIR; +import static org.apache.cassandra.sidecar.handlers.livemigration.LiveMigrationDirType.HINTS_DIR; +import static org.apache.cassandra.sidecar.handlers.livemigration.LiveMigrationDirType.LOCAL_SYSTEM_DATA_FILE_DIR; +import static org.apache.cassandra.sidecar.handlers.livemigration.LiveMigrationDirType.SAVED_CACHES_DIR; +import static org.apache.cassandra.sidecar.livemigration.TestFile.getInstanceFilesListResponse; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@ExtendWith(VertxExtension.class) +class LiveMigrationFilesVerificationTaskTest +{ + + private static final String SOURCE = "127.0.0.1"; + private static final String DESTINATION = "127.0.0.2"; + + private static @NotNull List getTestFiles(long lastModifiedTime) + { + int fileSize = 32; + return List.of( + new TestFile(DATA_FILE_DIR, 0, "ks1", -1, lastModifiedTime), + new TestFile(DATA_FILE_DIR, 0, "ks1/t1", -1, lastModifiedTime), + new TestFile(DATA_FILE_DIR, 0, "ks1/t1/data.db", fileSize, lastModifiedTime), + new TestFile(DATA_FILE_DIR, 1, "ks1", -1, lastModifiedTime), + new TestFile(DATA_FILE_DIR, 1, "ks1/t2", -1, lastModifiedTime), + new TestFile(DATA_FILE_DIR, 1, "ks1/t2/data.db", fileSize * 2, lastModifiedTime), + new TestFile(HINTS_DIR, 0, "empty-file", 0, lastModifiedTime), + new TestFile(COMMIT_LOG_DIR, 0, "commitlog-7-1.db", fileSize - 1, lastModifiedTime), + new TestFile(SAVED_CACHES_DIR, 0, "cache.bin", fileSize + 1, lastModifiedTime), + new TestFile(CDC_RAW_DIR, 0, "commitlog-7-1.db", fileSize / 2, lastModifiedTime), + new TestFile(LOCAL_SYSTEM_DATA_FILE_DIR, 0, "data.db", 1, lastModifiedTime)); + } + + @Test + void testNewDigestVerificationTask(@TempDir Path tempDir) + { + Injector injector = getInjector(); + InstanceMetadata instanceMetadata = InstanceMetadataTestUtil.getInstanceMetadata(DESTINATION, 2, tempDir); + LiveMigrationFilesVerificationTask verificationTask = createVerificationTask(injector, instanceMetadata); + + assertThat(verificationTask.hasStarted()).isFalse(); + assertThat(verificationTask.isCancelled()).isFalse(); + assertThat(verificationTask.isCompleted()).isFalse(); + assertThat(verificationTask.id()).isNotNull(); + assertThat(verificationTask.type()).isEqualTo(LiveMigrationFilesVerificationTask.FILES_VERIFICATION_TASK_TYPE); + + LiveMigrationFilesVerificationResponse response = verificationTask.getResponse(); + assertThat(response).isNotNull(); + assertThat(State.valueOf(response.state())).isEqualTo(State.NOT_STARTED); + } + + @Test + void testCancelBeforeStartingTheTask(@TempDir Path tempDir) + { + // Task is cancelled before starting it. + // The task state should be 'CANCELLED' even after starting it. + Injector injector = getInjector(); + + SidecarClient sidecarClient = injector.getInstance(SidecarClient.class); + when(sidecarClient.liveMigrationListInstanceFilesAsync(any())) + .thenReturn(Future.succeededFuture(new InstanceFilesListResponse(Collections.emptyList())) + .toCompletionStage().toCompletableFuture()); + + InstanceMetadata instanceMetadata = InstanceMetadataTestUtil.getInstanceMetadata(DESTINATION, 2, tempDir); + LiveMigrationFilesVerificationTask digestVerificationTask = + createVerificationTask(injector, instanceMetadata); + + // Cancel task before starting it + digestVerificationTask.cancel(); + + LiveMigrationFilesVerificationResponse response = digestVerificationTask.getResponse(); + assertThat(response).isNotNull(); + assertThat(State.valueOf(response.state())).isEqualTo(State.CANCELLED); + assertThat(digestVerificationTask.isCancelled()).isTrue(); + assertThat(digestVerificationTask.isCompleted()).isTrue(); + } + + @Test + void testCancelAndStartTask(@TempDir Path tempDir) + { + // Task is cancelled before starting it. + // The task state should be 'CANCELLED' even after starting it. + Injector injector = getInjector(); + + SidecarClient sidecarClient = injector.getInstance(SidecarClient.class); + when(sidecarClient.liveMigrationListInstanceFilesAsync(any())) + .thenReturn(Future.succeededFuture(new InstanceFilesListResponse(Collections.emptyList())) + .toCompletionStage().toCompletableFuture()); + + InstanceMetadata instanceMetadata = InstanceMetadataTestUtil.getInstanceMetadata(DESTINATION, 2, tempDir); + LiveMigrationFilesVerificationTask verificationTask = + createVerificationTask(injector, instanceMetadata); + + verificationTask.cancel(); + verificationTask.start(); + + LiveMigrationFilesVerificationResponse response = verificationTask.getResponse(); + assertThat(response).isNotNull(); + assertThat(State.valueOf(response.state())).isEqualTo(State.CANCELLED); + assertThat(verificationTask.isCancelled()).isTrue(); + assertThat(verificationTask.isCompleted()).isTrue(); + + // Cancelling task again should not cause any changes + verificationTask.cancel(); + assertThat(verificationTask.getResponse().state()).isEqualTo(State.CANCELLED.name()); + } + + @Test + public void testVerifyFilesUsingMD5(@TempDir Path tempDir) throws IOException, InterruptedException + { + LiveMigrationFilesVerificationRequest request = new LiveMigrationFilesVerificationRequest(20, "md5"); + verifyCompletesSuccessfully(tempDir, request); + } + + @Test + public void testVerifyFilesUsingXXHash(@TempDir Path tempDir) throws IOException, InterruptedException + { + LiveMigrationFilesVerificationRequest request = new LiveMigrationFilesVerificationRequest(20, "XXHash32"); + verifyCompletesSuccessfully(tempDir, request); + } + + @Test + public void testCancelCompletedTask(@TempDir Path tempDir) throws IOException, InterruptedException + { + LiveMigrationFilesVerificationRequest request = new LiveMigrationFilesVerificationRequest(20, "XXHash32"); + LiveMigrationFilesVerificationTask task = verifyCompletesSuccessfully(tempDir, request); + + assertThat(task.isCompleted()).isTrue(); + assertThat(State.valueOf(task.getResponse().state())).isEqualTo(State.COMPLETED); + + task.cancel(); + // State should not change the task got completed already + assertThat(State.valueOf(task.getResponse().state())).isEqualTo(State.COMPLETED); + assertThat(task.isCompleted()).isTrue(); + } + + private LiveMigrationFilesVerificationTask verifyCompletesSuccessfully(Path tempDir, + LiveMigrationFilesVerificationRequest request) throws IOException, InterruptedException + { + Injector injector = getInjector(); + InstanceMetadata instanceMetadata = InstanceMetadataTestUtil.getInstanceMetadata(DESTINATION, 2, tempDir); + + long lastModifiedTime = System.currentTimeMillis(); + List filesToDownload = getTestFiles(lastModifiedTime); + + mockListInstanceFilesResponse(injector, filesToDownload); + mockSourceFileDigestResponse(injector, request, filesToDownload, instanceMetadata); + + LiveMigrationFilesVerificationTask digestVerificationTask = + createVerificationTask(injector, request, instanceMetadata); + digestVerificationTask.start(); + + awaitForFuture(digestVerificationTask.future()); + assertThat(digestVerificationTask.isCompleted()).isTrue(); + LiveMigrationFilesVerificationResponse response = digestVerificationTask.getResponse(); + assertThat(response).isNotNull(); + assertThat(State.valueOf(response.state())).isEqualTo(State.COMPLETED); + assertThat(response.isVerificationSuccessful()).isTrue(); + assertThat(response.metadataMismatches()).isEqualTo(0); + assertThat(response.metadataMatched()).isEqualTo(filesToDownload.size()); + // Digest is compared only for files, so files matched should be equal to + // number of actual files present. + assertThat(response.filesMatched()).isEqualTo(filesCount(filesToDownload)); + assertThat(response.digestMismatches()).isEqualTo(0); + assertThat(response.digestVerificationFailures()).isEqualTo(0); + + return digestVerificationTask; + } + + @Test + public void testFilesListingAtSourceFailed(@TempDir Path tempDir) throws IOException, InterruptedException + { + LiveMigrationFilesVerificationRequest request = new LiveMigrationFilesVerificationRequest(20, "XXHash32"); + Injector injector = getInjector(); + InstanceMetadata instanceMetadata = InstanceMetadataTestUtil.getInstanceMetadata(DESTINATION, 2, tempDir); + + long lastModifiedTime = System.currentTimeMillis(); + List filesToDownload = getTestFiles(lastModifiedTime); + + mockListInstanceFilesResponse(injector, filesToDownload); + mockSourceFileDigestResponse(injector, request, filesToDownload, instanceMetadata); + + SidecarClient sidecarClient = injector.getInstance(SidecarClient.class); + when(sidecarClient.liveMigrationListInstanceFilesAsync(any(SidecarInstance.class))) + .thenReturn(CompletableFuture.failedFuture(new RuntimeException("Internal Server Error"))); + + LiveMigrationFilesVerificationTask digestVerificationTask = + createVerificationTask(injector, request, instanceMetadata); + digestVerificationTask.start(); + + Future completionFuture = digestVerificationTask.future(); + awaitForFuture(completionFuture); + assertThat(completionFuture.isComplete()).isTrue(); + LiveMigrationFilesVerificationResponse response = digestVerificationTask.getResponse(); + assertThat(response).isNotNull(); + assertThat(State.valueOf(response.state())).isEqualTo(State.FAILED); + assertThat(response.isVerificationSuccessful()).isFalse(); + assertThat(response.metadataMismatches()).isEqualTo(0); + assertThat(response.metadataMatched()).isEqualTo(0); + + assertThat(response.filesMatched()).isEqualTo(0); + assertThat(response.digestMismatches()).isEqualTo(0); + assertThat(response.digestVerificationFailures()).isEqualTo(0); + } + + @Test + public void testFileDigestCallToSourceFailed(@TempDir Path tempDir) throws IOException, InterruptedException + { + LiveMigrationFilesVerificationRequest request = new LiveMigrationFilesVerificationRequest(20, "XXHash32"); + Injector injector = getInjector(); + InstanceMetadata instanceMetadata = InstanceMetadataTestUtil.getInstanceMetadata(DESTINATION, 2, tempDir); + + long lastModifiedTime = System.currentTimeMillis(); + List filesToDownload = getTestFiles(lastModifiedTime); + + mockListInstanceFilesResponse(injector, filesToDownload); + mockSourceFileDigestResponse(injector, request, filesToDownload, instanceMetadata); + + // Mock file digest call failure + SidecarClient sidecarClient = injector.getInstance(SidecarClient.class); + TestFile randomFile = getRandomFile(filesToDownload); + when(sidecarClient.liveMigrationFileDigestAsync(any(SidecarInstance.class), + eq(randomFile.getFileUrl()), + anyString())) + .thenReturn(CompletableFuture.failedFuture(new IOException("File digest call failed"))); + + LiveMigrationFilesVerificationTask digestVerificationTask = + createVerificationTask(injector, request, instanceMetadata); + digestVerificationTask.start(); + + awaitForFuture(digestVerificationTask.future()); + LiveMigrationFilesVerificationResponse response = digestVerificationTask.getResponse(); + assertThat(response).isNotNull(); + assertThat(State.valueOf(response.state())).isEqualTo(State.FAILED); + assertThat(response.isVerificationSuccessful()).isFalse(); + assertThat(response.metadataMismatches()).isEqualTo(0); + assertThat(response.metadataMatched()).isEqualTo(filesToDownload.size()); + + assertThat(response.filesMatched()).isEqualTo(filesCount(filesToDownload) - 1); // -1 for the digest call failed. + assertThat(response.digestMismatches()).isEqualTo(0); + assertThat(response.digestVerificationFailures()).isEqualTo(1); + } + + @Test + public void testFailedToCalculateDigestForFilesInLocal(@TempDir Path tempDir) throws IOException, InterruptedException + { + String digestAlgorithm = "XXHash32"; + LiveMigrationFilesVerificationRequest request = new LiveMigrationFilesVerificationRequest(20, digestAlgorithm); + Injector injector = getInjector(); + InstanceMetadata instanceMetadata = InstanceMetadataTestUtil.getInstanceMetadata(DESTINATION, 2, tempDir); + + long lastModifiedTime = System.currentTimeMillis(); + List filesToDownload = getTestFiles(lastModifiedTime); + + mockListInstanceFilesResponse(injector, filesToDownload); + mockSourceFileDigestResponse(injector, request, filesToDownload, instanceMetadata); + + AtomicBoolean shouldFail = new AtomicBoolean(true); + + LiveMigrationFilesVerificationTask digestVerificationTask = + spy(createVerificationTask(injector, request, instanceMetadata)); + doAnswer(invocation -> { + if (shouldFail.get()) + { + shouldFail.set(false); + return Future.failedFuture("Failed to calculate digest for file."); + } + return invocation.callRealMethod(); + }).when(digestVerificationTask.digestVerifierFactory).verifier(any()); + + digestVerificationTask.start(); + + awaitForFuture(digestVerificationTask.future()); + LiveMigrationFilesVerificationResponse response = digestVerificationTask.getResponse(); + assertThat(response).isNotNull(); + assertThat(State.valueOf(response.state())).isEqualTo(State.FAILED); + assertThat(response.isVerificationSuccessful()).isFalse(); + assertThat(response.metadataMismatches()).isEqualTo(0); + assertThat(response.metadataMatched()).isEqualTo(filesToDownload.size()); + + assertThat(response.filesMatched()).isEqualTo(filesCount(filesToDownload) - 1); // -1 for the digest call failed. + assertThat(response.digestMismatches()).isEqualTo(0); + assertThat(response.digestVerificationFailures()).isEqualTo(1); + } + + @Test + public void testFilesTimestampsAreNotMatching(@TempDir Path tempDir) throws IOException, InterruptedException + { + // Verification task should fail when file's last modified timestamps are not matching + Injector injector = getInjector(); + LiveMigrationFilesVerificationRequest request = new LiveMigrationFilesVerificationRequest(20, "md5"); + InstanceMetadata instanceMetadata = InstanceMetadataTestUtil.getInstanceMetadata(DESTINATION, 2, tempDir); + + long lastModifiedTime = System.currentTimeMillis(); + List localTestFiles = getTestFiles(lastModifiedTime); + + List filesToDownload = new ArrayList<>(localTestFiles.size()); + for (int i = 0; i < localTestFiles.size(); i++) + { + // Since 'i' starts with 0, first file last modified remains the same + TestFile file = localTestFiles.get(i); + filesToDownload.add(updateLastModifiedTime(file, file.lastModifiedTime + i + 1)); + } + + mockListInstanceFilesResponse(injector, localTestFiles); + mockSourceFileDigestResponse(injector, request, filesToDownload, instanceMetadata); + + LiveMigrationFilesVerificationTask digestVerificationTask = + createVerificationTask(injector, request, instanceMetadata); + + // Start verification task + digestVerificationTask.start(); + + awaitForFuture(digestVerificationTask.future()); + LiveMigrationFilesVerificationResponse response = digestVerificationTask.getResponse(); + assertThat(response).isNotNull(); + assertThat(State.valueOf(response.state())).isEqualTo(State.FAILED); + + assertThat(response.isVerificationSuccessful()).isFalse(); + // directories time stamp is not compared, so they should match + assertThat(response.metadataMatched()).isEqualTo(dirsCount(localTestFiles)); + assertThat(response.metadataMismatches()).isEqualTo(filesCount(localTestFiles)); + assertThat(response.filesNotFoundAtSource()).isEqualTo(0); + assertThat(response.filesNotFoundAtDestination()).isEqualTo(0); + + // since the last modified times do not match, verification will not go to digest comparison stage, + // so the files matched and digest mismatched counts remain zero. + assertThat(response.filesMatched()).isEqualTo(0); + assertThat(response.digestMismatches()).isEqualTo(0); + assertThat(response.digestVerificationFailures()).isEqualTo(0); + } + + @Test + public void testFilesSizesAreNotMatching(@TempDir Path tempDir) throws IOException, InterruptedException + { + // Verification task should fail when files sizes are not matching + Injector injector = getInjector(); + LiveMigrationFilesVerificationRequest request = new LiveMigrationFilesVerificationRequest(20, "md5"); + InstanceMetadata instanceMetadata = InstanceMetadataTestUtil.getInstanceMetadata(DESTINATION, 2, tempDir); + + long lastModifiedTime = System.currentTimeMillis(); + List localTestFiles = getTestFiles(lastModifiedTime); + + List filesToDownload = new ArrayList<>(localTestFiles.size()); + for (int i = 0; i < localTestFiles.size(); i++) + { + TestFile file = localTestFiles.get(i); + filesToDownload.add(updateSize(file, file.size > -1 ? file.size + i + 1 : file.size)); + } + + mockListInstanceFilesResponse(injector, localTestFiles); + mockSourceFileDigestResponse(injector, request, filesToDownload, instanceMetadata); + + LiveMigrationFilesVerificationTask digestVerificationTask = + createVerificationTask(injector, request, instanceMetadata); + + // Start verification task + digestVerificationTask.start(); + + awaitForFuture(digestVerificationTask.future()); + LiveMigrationFilesVerificationResponse response = digestVerificationTask.getResponse(); + assertThat(response).isNotNull(); + assertThat(State.valueOf(response.state())).isEqualTo(State.FAILED); + + assertThat(response.isVerificationSuccessful()).isFalse(); + assertThat(response.metadataMatched()).isEqualTo(dirsCount(localTestFiles)); + assertThat(response.metadataMismatches()).isEqualTo(filesCount(localTestFiles)); + assertThat(response.filesNotFoundAtSource()).isEqualTo(0); + assertThat(response.filesNotFoundAtDestination()).isEqualTo(0); + + // Since the file sizes do not match, verification will not reach digest comparison stage, + // so the files matched and digest mismatched counts remain zero. + assertThat(response.filesMatched()).isEqualTo(0); + assertThat(response.digestMismatches()).isEqualTo(0); + assertThat(response.digestVerificationFailures()).isEqualTo(0); + } + + @Test + public void testFilesDigestsAreNotMatching(@TempDir Path tempDir) throws IOException, InterruptedException + { + // Verification task should fail when file digests do not match + Injector injector = getInjector(); + LiveMigrationFilesVerificationRequest request = new LiveMigrationFilesVerificationRequest(20, "md5"); + InstanceMetadata instanceMetadata = InstanceMetadataTestUtil.getInstanceMetadata(DESTINATION, 2, tempDir); + + long lastModifiedTime = System.currentTimeMillis(); + List localTestFiles = getTestFiles(lastModifiedTime); + + mockListInstanceFilesResponse(injector, localTestFiles); + mockSourceFileDigestResponse(injector, request, localTestFiles, instanceMetadata); + + // Now update file contents to re-create that file digests are not matching + for (TestFile testFile : localTestFiles) + { + recreateFile(testFile, instanceMetadata); + } + + LiveMigrationFilesVerificationTask digestVerificationTask = + createVerificationTask(injector, request, instanceMetadata); + + // Start verification task + digestVerificationTask.start(); + + awaitForFuture(digestVerificationTask.future()); + LiveMigrationFilesVerificationResponse response = digestVerificationTask.getResponse(); + assertThat(response).isNotNull(); + assertThat(State.valueOf(response.state())).isEqualTo(State.FAILED); + + assertThat(response.isVerificationSuccessful()).isFalse(); + assertThat(response.metadataMatched()).isEqualTo(localTestFiles.size()); + assertThat(response.metadataMismatches()).isEqualTo(0); + assertThat(response.filesNotFoundAtSource()).isEqualTo(0); + assertThat(response.filesNotFoundAtDestination()).isEqualTo(0); + + // In this scenario, metadata matches and verification task compares digests for normal files (non-directory) + // digest verification should fail for the as the file contents got changed. + int emptyFilesCount = emptyFilesCount(localTestFiles); + assertThat(response.filesMatched()).isEqualTo(emptyFilesCount); + assertThat(response.digestMismatches()).isEqualTo(filesCount(localTestFiles) - emptyFilesCount); + assertThat(response.digestVerificationFailures()).isEqualTo(0); + } + + @Test + public void testFewFilesAreMissing(@TempDir Path tempDir) throws IOException, InterruptedException + { + // Test the scenario where few files are missing at both source and destination + Injector injector = getInjector(); + LiveMigrationFilesVerificationRequest request = new LiveMigrationFilesVerificationRequest(20, "md5"); + InstanceMetadata instanceMetadata = InstanceMetadataTestUtil.getInstanceMetadata(DESTINATION, 2, tempDir); + + long lastModifiedTime = System.currentTimeMillis(); + List testFiles = getTestFiles(lastModifiedTime); + List localFiles = testFiles.subList(1, testFiles.size()); // Excluding first file + List filesToDownload = testFiles.subList(0, testFiles.size() - 1); // Excluding last file + + mockListInstanceFilesResponse(injector, localFiles); + mockSourceFileDigestResponse(injector, request, filesToDownload, instanceMetadata); + + LiveMigrationFilesVerificationTask digestVerificationTask = + createVerificationTask(injector, request, instanceMetadata); + + // Start verification task + digestVerificationTask.start(); + + awaitForFuture(digestVerificationTask.future()); + LiveMigrationFilesVerificationResponse response = digestVerificationTask.getResponse(); + assertThat(response).isNotNull(); + assertThat(State.valueOf(response.state())).isEqualTo(State.FAILED); + + assertThat(response.isVerificationSuccessful()).isFalse(); + assertThat(response.metadataMatched()).isEqualTo(testFiles.size() - 2); + assertThat(response.metadataMismatches()).isEqualTo(0); + assertThat(response.filesNotFoundAtSource()).isEqualTo(1); + assertThat(response.filesNotFoundAtDestination()).isEqualTo(1); + assertThat(response.filesMatched()).isEqualTo(0); + assertThat(response.digestMismatches()).isEqualTo(0); + assertThat(response.digestVerificationFailures()).isEqualTo(0); + } + + @Test + public void testFewDirectoriesAreMissing(@TempDir Path tempDir) throws IOException, InterruptedException + { + // Test the scenario where few directories are missing at source and destination + Injector injector = getInjector(); + LiveMigrationFilesVerificationRequest request = new LiveMigrationFilesVerificationRequest(20, "md5"); + InstanceMetadata instanceMetadata = InstanceMetadataTestUtil.getInstanceMetadata(DESTINATION, 2, tempDir); + + long lastModifiedTime = System.currentTimeMillis(); + List testFiles = getTestFiles(lastModifiedTime); + List localFiles = new ArrayList<>(testFiles); + // Size of test file is set to -1 to indicate that it is directory + localFiles.add(new TestFile(DATA_FILE_DIR, 0, "newkeyspace", -1, lastModifiedTime)); + + List filesToDownload = new ArrayList<>(testFiles); + filesToDownload.add(new TestFile(DATA_FILE_DIR, 1, "newsrckeyspace", -1, lastModifiedTime)); + + mockListInstanceFilesResponse(injector, localFiles); + mockSourceFileDigestResponse(injector, request, filesToDownload, instanceMetadata); + + LiveMigrationFilesVerificationTask digestVerificationTask = + createVerificationTask(injector, request, instanceMetadata); + + // Start verification task + digestVerificationTask.start(); + + awaitForFuture(digestVerificationTask.future()); + LiveMigrationFilesVerificationResponse response = digestVerificationTask.getResponse(); + assertThat(response).isNotNull(); + assertThat(State.valueOf(response.state())).isEqualTo(State.FAILED); + + assertThat(response.isVerificationSuccessful()).isFalse(); + assertThat(response.metadataMatched()).isEqualTo(testFiles.size()); + assertThat(response.metadataMismatches()).isEqualTo(0); + assertThat(response.filesNotFoundAtSource()).isEqualTo(1); + assertThat(response.filesNotFoundAtDestination()).isEqualTo(1); + assertThat(response.filesMatched()).isEqualTo(0); + assertThat(response.digestMismatches()).isEqualTo(0); + assertThat(response.digestVerificationFailures()).isEqualTo(0); + } + + @Test + public void testFileTypesAreNotMatching(@TempDir Path tempDir) throws IOException, InterruptedException + { + // Test the scenario where fileType is not matching between source and destination + // i.e. an entry is expected to be a file, but it is a directory at source and vice versa. + Injector injector = getInjector(); + LiveMigrationFilesVerificationRequest request = new LiveMigrationFilesVerificationRequest(20, "md5"); + InstanceMetadata instanceMetadata = InstanceMetadataTestUtil.getInstanceMetadata(DESTINATION, 2, tempDir); + + long lastModifiedTime = System.currentTimeMillis(); + List testFiles = new ArrayList<>(getTestFiles(lastModifiedTime)); + TestFile randomFile1 = getRandomFile(testFiles); + testFiles.remove(randomFile1); + TestFile randomFile2 = getRandomFile(testFiles); + testFiles.remove(randomFile2); + + List localFiles = new ArrayList<>(testFiles); + localFiles.add(convertToDirectory(randomFile1)); + localFiles.add(randomFile2); + + List filesAtSource = new ArrayList<>(testFiles); + filesAtSource.add(randomFile1); + filesAtSource.add(convertToDirectory(randomFile2)); + + mockListInstanceFilesResponse(injector, localFiles); + mockSourceFileDigestResponse(injector, request, filesAtSource, instanceMetadata); + + LiveMigrationFilesVerificationTask digestVerificationTask = + createVerificationTask(injector, request, instanceMetadata); + + // Start verification task + digestVerificationTask.start(); + + awaitForFuture(digestVerificationTask.future()); + LiveMigrationFilesVerificationResponse response = digestVerificationTask.getResponse(); + assertThat(response).isNotNull(); + assertThat(State.valueOf(response.state())).isEqualTo(State.FAILED); + + assertThat(response.isVerificationSuccessful()).isFalse(); + + // files types of randomFile1 and randomFile2 are not matching between source and destination, + // hence the metadata mismatches should be equal to 2 + assertThat(response.metadataMismatches()).isEqualTo(2); + assertThat(response.metadataMatched()).isEqualTo(localFiles.size() - 2); + assertThat(response.filesNotFoundAtSource()).isEqualTo(0); + assertThat(response.filesNotFoundAtDestination()).isEqualTo(0); + + assertThat(response.filesMatched()).isEqualTo(0); + assertThat(response.digestMismatches()).isEqualTo(0); + assertThat(response.digestVerificationFailures()).isEqualTo(0); + } + + @Test + public void testAbortIfCancelledWhenTaskNotCancelled(@TempDir Path tempDir) + { + // Test that abortIfCancelled returns succeeded future when task is not cancelled + Injector injector = getInjector(); + InstanceMetadata instanceMetadata = InstanceMetadataTestUtil.getInstanceMetadata(DESTINATION, 2, tempDir); + LiveMigrationFilesVerificationTask verificationTask = createVerificationTask(injector, instanceMetadata); + + String testValue = "test-value"; + Future result = verificationTask.abortIfCancelled(testValue); + + assertThat(result.succeeded()).isTrue(); + assertThat(result.result()).isEqualTo(testValue); + assertThat(verificationTask.isCancelled()).isFalse(); + } + + @Test + public void testAbortIfCancelledWhenTaskCancelledInProgress(@TempDir Path tempDir) + { + // Test that abortIfCancelled returns failed future and updates state to CANCELLED + // when task is cancelled and state is IN_PROGRESS + Injector injector = getInjector(); + InstanceMetadata instanceMetadata = InstanceMetadataTestUtil.getInstanceMetadata(DESTINATION, 2, tempDir); + LiveMigrationFilesVerificationTask verificationTask = createVerificationTask(injector, instanceMetadata); + + // Start the task to set state to IN_PROGRESS + SidecarClient sidecarClient = injector.getInstance(SidecarClient.class); + when(sidecarClient.liveMigrationListInstanceFilesAsync(any())) + .thenReturn(Future.succeededFuture(new InstanceFilesListResponse(Collections.emptyList())) + .toCompletionStage().toCompletableFuture()); + + verificationTask.start(); + assertThat(State.valueOf(verificationTask.getResponse().state())).isEqualTo(State.IN_PROGRESS); + + // Cancel the task + verificationTask.cancel(); + assertThat(verificationTask.isCancelled()).isTrue(); + + // Call abortIfCancelled + String testValue = "test-value"; + Future result = verificationTask.abortIfCancelled(testValue); + + assertThat(result.failed()).isTrue(); + assertThat(result.cause().getMessage()).isEqualTo("Task got cancelled"); + assertThat(State.valueOf(verificationTask.getResponse().state())).isEqualTo(State.CANCELLED); + } + + @Test + public void testAbortIfCancelledWhenTaskCancelledButNotInProgress(@TempDir Path tempDir) + { + // Test that abortIfCancelled returns failed future but does not update state + // when task is cancelled but state is not IN_PROGRESS + Injector injector = getInjector(); + InstanceMetadata instanceMetadata = InstanceMetadataTestUtil.getInstanceMetadata(DESTINATION, 2, tempDir); + LiveMigrationFilesVerificationTask verificationTask = createVerificationTask(injector, instanceMetadata); + + // Task is in NOT_STARTED state + assertThat(State.valueOf(verificationTask.getResponse().state())).isEqualTo(State.NOT_STARTED); + + // Cancel the task + verificationTask.cancel(); + assertThat(verificationTask.isCancelled()).isTrue(); + + // Call abortIfCancelled + String testValue = "test-value"; + Future result = verificationTask.abortIfCancelled(testValue); + + assertThat(result.failed()).isTrue(); + assertThat(result.cause().getMessage()).isEqualTo("Task got cancelled"); + // State should remain NOT_STARTED since compareAndSet will fail + assertThat(State.valueOf(verificationTask.getResponse().state())).isEqualTo(State.CANCELLED); + } + + @Test + public void testAbortIfCancelledWithDifferentTypes(@TempDir Path tempDir) + { + // Test that abortIfCancelled works with different generic types + Injector injector = getInjector(); + InstanceMetadata instanceMetadata = InstanceMetadataTestUtil.getInstanceMetadata(DESTINATION, 2, tempDir); + LiveMigrationFilesVerificationTask verificationTask = createVerificationTask(injector, instanceMetadata); + + // Test with Integer + Integer intValue = 42; + Future intResult = verificationTask.abortIfCancelled(intValue); + assertThat(intResult.succeeded()).isTrue(); + assertThat(intResult.result()).isEqualTo(intValue); + + // Test with List + List listValue = List.of("a", "b", "c"); + Future> listResult = verificationTask.abortIfCancelled(listValue); + assertThat(listResult.succeeded()).isTrue(); + assertThat(listResult.result()).isEqualTo(listValue); + + // Test with null + Future nullResult = verificationTask.abortIfCancelled(null); + assertThat(nullResult.succeeded()).isTrue(); + assertThat(nullResult.result()).isNull(); + } + + @Test + public void testCancelDuringValidation(@TempDir Path tempDir) throws IOException, InterruptedException + { + // Test that cancel() racing with validation completion doesn't cause IllegalStateException + // This tests the thread-safety of using tryComplete/tryFail + Injector injector = getInjector(); + LiveMigrationFilesVerificationRequest request = new LiveMigrationFilesVerificationRequest(20, "md5"); + InstanceMetadata instanceMetadata = InstanceMetadataTestUtil.getInstanceMetadata(DESTINATION, 2, tempDir); + + long lastModifiedTime = System.currentTimeMillis(); + List filesToDownload = getTestFiles(lastModifiedTime); + + // Create a Promise that we control to simulate async operation + Promise listFilesPromise = Promise.promise(); + + SidecarClient sidecarClient = injector.getInstance(SidecarClient.class); + when(sidecarClient.liveMigrationListInstanceFilesAsync(any(SidecarInstance.class))) + .thenReturn(listFilesPromise.future().toCompletionStage().toCompletableFuture()); + + mockSourceFileDigestResponse(injector, request, filesToDownload, instanceMetadata); + + LiveMigrationFilesVerificationTask verificationTask = + createVerificationTask(injector, request, instanceMetadata); + + // Start the task - it will now be blocked waiting for listFilesPromise + verificationTask.start(); + assertThat(verificationTask.hasStarted()).isTrue(); + assertThat(State.valueOf(verificationTask.getResponse().state())).isEqualTo(State.IN_PROGRESS); + + // Cancel the task while it's waiting on the promise + verificationTask.cancel(); + assertThat(verificationTask.isCancelled()).isTrue(); + + // Now complete the promise - this creates a race between: + // - cancel() calling completionPromise.tryFail() + // - validation completing and calling completionPromise.tryComplete() + listFilesPromise.complete(getInstanceFilesListResponse(filesToDownload)); + + // Wait for the task to complete + awaitForFuture(verificationTask.future()); + + // Verify the task completed without throwing IllegalStateException + assertThat(verificationTask.isCompleted()).isTrue(); + assertThat(verificationTask.isCancelled()).isTrue(); + + // The final state should be CANCELLED since cancel was called + assertThat(State.valueOf(verificationTask.getResponse().state())).isEqualTo(State.CANCELLED); + + verify(sidecarClient, times(0)) + .liveMigrationFileDigestAsync(any(SidecarInstance.class), anyString(), anyString()); + } + + @Test + public void testCancelWhileValidatingDigests(@TempDir Path tempDir) throws IOException, InterruptedException + { + // Cancels the verification task when the task is comparing files digests. + Injector injector = getInjector(); + LiveMigrationFilesVerificationRequest request = new LiveMigrationFilesVerificationRequest(20, "md5"); + InstanceMetadata instanceMetadata = InstanceMetadataTestUtil.getInstanceMetadata(DESTINATION, 2, tempDir); + + long lastModifiedTime = System.currentTimeMillis(); + List filesToDownload = getTestFiles(lastModifiedTime); + mockListInstanceFilesResponse(injector, filesToDownload); + + for (TestFile testFile: filesToDownload) + { + testFile.createFile(instanceMetadata); + } + + // Create a Promise that we control + Promise filesDigestsPromise = Promise.promise(); + SidecarClient sidecarClient = injector.getInstance(SidecarClient.class); + when(sidecarClient.liveMigrationFileDigestAsync(any(SidecarInstanceImpl.class), anyString(), anyString())) + .thenReturn(filesDigestsPromise.future().toCompletionStage().toCompletableFuture()); + + LiveMigrationFilesVerificationTask verificationTask = + createVerificationTask(injector, request, instanceMetadata); + + // Start the task + verificationTask.start(); + assertThat(verificationTask.hasStarted()).isTrue(); + assertThat(State.valueOf(verificationTask.getResponse().state())).isEqualTo(State.IN_PROGRESS); + + verificationTask.cancel(); + + // Wait for validation to complete + awaitForFuture(verificationTask.future()); + assertThat(verificationTask.isCompleted()).isTrue(); + assertThat(State.valueOf(verificationTask.getResponse().state())).isEqualTo(State.CANCELLED); + + filesDigestsPromise.fail("Digest calls failed"); + + awaitForFuture(verificationTask.future()); + + // CANCELLED is a final state, task status should not change + assertThat(State.valueOf(verificationTask.getResponse().state())).isEqualTo(State.CANCELLED); + } + + @Test + public void testUnknownDigestAlgorithm(@TempDir Path tempDir) throws IOException, InterruptedException + { + // Verification task should fail when an unknown digest algorithm is provided + String unknownAlgorithm = "UnknownDigestAlgo"; + LiveMigrationFilesVerificationRequest request = new LiveMigrationFilesVerificationRequest(20, unknownAlgorithm); + Injector injector = getInjector(); + InstanceMetadata instanceMetadata = InstanceMetadataTestUtil.getInstanceMetadata(DESTINATION, 2, tempDir); + + long lastModifiedTime = System.currentTimeMillis(); + List filesToDownload = getTestFiles(lastModifiedTime); + + mockListInstanceFilesResponse(injector, filesToDownload); + + // Create files locally + for (TestFile testFile : filesToDownload) + { + testFile.createFile(instanceMetadata); + } + + // Mock digest response from source with unknown algorithm + SidecarClient sidecarClient = injector.getInstance(SidecarClient.class); + when(sidecarClient.liveMigrationFileDigestAsync(eq(new SidecarInstanceImpl(SOURCE, 9043)), + anyString(), + eq(unknownAlgorithm))) + .thenAnswer(invocationOnMock -> { + // Return a digest response with the unknown algorithm + return Future.succeededFuture(new DigestResponse("dummy-digest", unknownAlgorithm)) + .toCompletionStage().toCompletableFuture(); + }); + + LiveMigrationFilesVerificationTask verificationTask = + createVerificationTask(injector, request, instanceMetadata); + + // Start verification task + verificationTask.start(); + + awaitForFuture(verificationTask.future()); + LiveMigrationFilesVerificationResponse response = verificationTask.getResponse(); + assertThat(response).isNotNull(); + assertThat(State.valueOf(response.state())).isEqualTo(State.FAILED); + + assertThat(response.isVerificationSuccessful()).isFalse(); + assertThat(response.metadataMatched()).isEqualTo(filesToDownload.size()); + assertThat(response.metadataMismatches()).isEqualTo(0); + assertThat(response.filesNotFoundAtSource()).isEqualTo(0); + assertThat(response.filesNotFoundAtDestination()).isEqualTo(0); + + // Since the digest algorithm is unknown, verification will fail for all files + assertThat(response.filesMatched()).isEqualTo(0); + assertThat(response.digestMismatches()).isEqualTo(0); + assertThat(response.digestVerificationFailures()).isEqualTo(filesCount(filesToDownload)); + } + + Injector getInjector() + { + return Guice.createInjector(new TestModule()); + } + + private void awaitForFuture(Future future) throws InterruptedException + { + CountDownLatch latch = new CountDownLatch(1); + future.onComplete(res -> latch.countDown()); + + //noinspection ResultOfMethodCallIgnored + latch.await(5, TimeUnit.SECONDS); // Change to latch.await() for debugging + } + + private LiveMigrationFilesVerificationTask createVerificationTask(Injector injector, + InstanceMetadata instanceMetadata) + { + LiveMigrationFilesVerificationRequest request = new LiveMigrationFilesVerificationRequest(20, "md5"); + return createVerificationTask(injector, request, instanceMetadata); + } + + private LiveMigrationFilesVerificationTask createVerificationTask(Injector injector, + LiveMigrationFilesVerificationRequest request, + InstanceMetadata instanceMetadata) + { + ServiceConfigurationImpl serviceConfiguration = new ServiceConfigurationImpl(); + Vertx vertx = injector.getInstance(Vertx.class); + DigestVerifierFactory digestVerifierFactory = spy(injector.getInstance(DigestVerifierFactory.class)); + SidecarClient sidecarClient = injector.getInstance(SidecarClient.class); + + return LiveMigrationFilesVerificationTask.builder() + .id("id") + .source(LiveMigrationFilesVerificationTaskTest.SOURCE) + .port(9043) + .executorPools(new ExecutorPools(vertx, serviceConfiguration)) + .liveMigrationConfiguration(getLiveMigrationConfig()) + .request(request) + .instanceMetadata(instanceMetadata) + .sidecarClient(sidecarClient) + .vertx(vertx) + .digestVerifierFactory(digestVerifierFactory) + .build(); + } + + private LiveMigrationConfiguration getLiveMigrationConfig() + { + return new LiveMigrationConfigurationImpl(Set.of(), Set.of(), Map.of(SOURCE, DESTINATION), 20); + } + + private void mockListInstanceFilesResponse(Injector injector, List files) + { + SidecarClient sidecarClient = injector.getInstance(SidecarClient.class); + when(sidecarClient.liveMigrationListInstanceFilesAsync(any(SidecarInstance.class))) + .thenReturn(CompletableFuture.completedFuture(getInstanceFilesListResponse(files))); + } + + private void mockSourceFileDigestResponse(Injector injector, + LiveMigrationFilesVerificationRequest request, + List sourceFiles, + InstanceMetadata instanceMetadata) throws IOException + { + Map digestsByFileUrl = new HashMap<>(sourceFiles.size()); + for (TestFile testFile : sourceFiles) + { + testFile.createFile(instanceMetadata); + digestsByFileUrl.put(testFile.getFileUrl(), + testFile.digest(instanceMetadata, () -> getDigestAlgorithm(injector, request))); + } + + SidecarClient sidecarClient = injector.getInstance(SidecarClient.class); + when(sidecarClient.liveMigrationFileDigestAsync(eq(new SidecarInstanceImpl(SOURCE, 9043)), + anyString(), + eq(request.digestAlgorithm()))) + .thenAnswer(invocationOnMock -> { + String fileUrl = invocationOnMock.getArgument(1); + String digest = digestsByFileUrl.get(fileUrl); + + return Future.succeededFuture(new DigestResponse(digest, request.digestAlgorithm())) + .toCompletionStage().toCompletableFuture(); + }); + } + + DigestAlgorithm getDigestAlgorithm(Injector injector, + LiveMigrationFilesVerificationRequest request) + { + DigestAlgorithmFactory digestAlgorithmFactory = injector.getInstance(DigestAlgorithmFactory.class); + return digestAlgorithmFactory.getDigestAlgorithm(request.digestAlgorithm(), 0); + } + + TestFile updateLastModifiedTime(TestFile file, long lastModifiedTime) + { + return new TestFile(file.dirType, file.dirIndex, file.relativePath, file.size, lastModifiedTime); + } + + TestFile updateSize(TestFile testFile, int size) + { + return new TestFile(testFile.dirType, testFile.dirIndex, testFile.relativePath, size, testFile.lastModifiedTime); + } + + TestFile convertToDirectory(TestFile testFile) + { + return new TestFile(testFile.dirType, testFile.dirIndex, testFile.relativePath, -1, testFile.lastModifiedTime); + } + + TestFile getRandomFile(List testFiles) + { + //noinspection OptionalGetWithoutIsPresent + return testFiles.stream().filter(testFile -> testFile.getFileType() == FileType.FILE).findFirst().get(); + } + + /** + * When a test file is created, it is filled with random data. + * Deleting and re-creating file to update the contents of the file. + * + * @param testFile test file to re-create + */ + void recreateFile(TestFile testFile, InstanceMetadata instanceMetadata) throws IOException + { + if (testFile.getFileType() == FileType.DIRECTORY) + { + // do nothing for directory + return; + } + testFile.deleteFile(instanceMetadata); + testFile.createFile(instanceMetadata); + } + + int filesCount(List testFiles) + { + return (int) testFiles.stream().filter(testFile -> testFile.getFileType() == FileType.FILE) + .count(); + } + + int dirsCount(List testFiles) + { + return testFiles.size() - filesCount(testFiles); + } + + int emptyFilesCount(List testFiles) + { + return (int) testFiles.stream().filter(testFile -> testFile.size == 0).count(); + } + + private static class TestModule extends AbstractModule + { + @Override + protected void configure() + { + bind(Vertx.class).toInstance(Vertx.vertx()); + bind(DigestVerifierFactory.class); + bind(SidecarClient.class).toInstance(mock(SidecarClient.class)); + install(new DigestAlgorithmProviderTestModule()); + } + } +} diff --git a/server/src/test/java/org/apache/cassandra/sidecar/livemigration/LiveMigrationInstanceMetadataUtilTest.java b/server/src/test/java/org/apache/cassandra/sidecar/livemigration/LiveMigrationInstanceMetadataUtilTest.java index f6cfae467..1c4b3854d 100644 --- a/server/src/test/java/org/apache/cassandra/sidecar/livemigration/LiveMigrationInstanceMetadataUtilTest.java +++ b/server/src/test/java/org/apache/cassandra/sidecar/livemigration/LiveMigrationInstanceMetadataUtilTest.java @@ -33,12 +33,12 @@ import org.apache.cassandra.sidecar.cluster.instance.InstanceMetadata; import org.mockito.Mockito; -import static org.apache.cassandra.sidecar.livemigration.LiveMigrationInstanceMetadataUtil.LIVE_MIGRATION_CDC_RAW_DIR_PATH; -import static org.apache.cassandra.sidecar.livemigration.LiveMigrationInstanceMetadataUtil.LIVE_MIGRATION_COMMITLOG_DIR_PATH; -import static org.apache.cassandra.sidecar.livemigration.LiveMigrationInstanceMetadataUtil.LIVE_MIGRATION_DATA_FILE_DIR_PATH; -import static org.apache.cassandra.sidecar.livemigration.LiveMigrationInstanceMetadataUtil.LIVE_MIGRATION_HINTS_DIR_PATH; -import static org.apache.cassandra.sidecar.livemigration.LiveMigrationInstanceMetadataUtil.LIVE_MIGRATION_LOCAL_SYSTEM_DATA_FILE_DIR_PATH; -import static org.apache.cassandra.sidecar.livemigration.LiveMigrationInstanceMetadataUtil.LIVE_MIGRATION_SAVED_CACHES_DIR_PATH; +import static org.apache.cassandra.sidecar.handlers.livemigration.InstanceMetadataTestUtil.LIVE_MIGRATION_CDC_RAW_DIR_PATH; +import static org.apache.cassandra.sidecar.handlers.livemigration.InstanceMetadataTestUtil.LIVE_MIGRATION_COMMITLOG_DIR_PATH; +import static org.apache.cassandra.sidecar.handlers.livemigration.InstanceMetadataTestUtil.LIVE_MIGRATION_DATA_FILE_DIR_PATH; +import static org.apache.cassandra.sidecar.handlers.livemigration.InstanceMetadataTestUtil.LIVE_MIGRATION_HINTS_DIR_PATH; +import static org.apache.cassandra.sidecar.handlers.livemigration.InstanceMetadataTestUtil.LIVE_MIGRATION_LOCAL_SYSTEM_DATA_FILE_DIR_PATH; +import static org.apache.cassandra.sidecar.handlers.livemigration.InstanceMetadataTestUtil.LIVE_MIGRATION_SAVED_CACHES_DIR_PATH; import static org.apache.cassandra.sidecar.livemigration.LiveMigrationInstanceMetadataUtil.localPath; import static org.apache.cassandra.sidecar.livemigration.LiveMigrationPlaceholderUtil.CDC_RAW_DIR_PLACEHOLDER; import static org.apache.cassandra.sidecar.livemigration.LiveMigrationPlaceholderUtil.COMMITLOG_DIR_PLACEHOLDER; diff --git a/server/src/test/java/org/apache/cassandra/sidecar/livemigration/LiveMigrationTaskManagerTestModule.java b/server/src/test/java/org/apache/cassandra/sidecar/livemigration/LiveMigrationTaskManagerTestModule.java new file mode 100644 index 000000000..b2e61da7d --- /dev/null +++ b/server/src/test/java/org/apache/cassandra/sidecar/livemigration/LiveMigrationTaskManagerTestModule.java @@ -0,0 +1,155 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.sidecar.livemigration; + +import java.util.List; + +import com.google.inject.AbstractModule; +import io.vertx.core.Future; +import io.vertx.core.Vertx; +import org.apache.cassandra.sidecar.cluster.CassandraAdapterDelegate; +import org.apache.cassandra.sidecar.cluster.InstancesMetadata; +import org.apache.cassandra.sidecar.cluster.instance.InstanceMetadata; +import org.apache.cassandra.sidecar.common.request.LiveMigrationDataCopyRequest; +import org.apache.cassandra.sidecar.common.request.LiveMigrationFilesVerificationRequest; +import org.apache.cassandra.sidecar.common.response.LiveMigrationDataCopyResponse; +import org.apache.cassandra.sidecar.common.response.LiveMigrationFilesVerificationResponse; +import org.apache.cassandra.sidecar.config.LiveMigrationConfiguration; +import org.apache.cassandra.sidecar.config.ServiceConfiguration; +import org.apache.cassandra.sidecar.config.SidecarConfiguration; +import org.apache.cassandra.sidecar.handlers.livemigration.FakeLiveMigrationTask; +import org.apache.cassandra.sidecar.handlers.livemigration.LiveMigrationMap; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Common Guice test module for live migration task manager tests. + * Provides shared mock configurations for InstancesMetadata, SidecarConfiguration, + * and other common dependencies used across different task manager tests. + */ +class LiveMigrationTaskManagerTestModule extends AbstractModule +{ + static final String SOURCE_1 = "source1"; + static final String SOURCE_2 = "source2"; + static final int DEST_1_ID = 200001; + static final int DEST_2_ID = 200002; + static final int DEST_3_ID = 200003; + static final String DESTINATION_1 = "destination1"; + static final String DESTINATION_2 = "destination2"; + static final String DESTINATION_3 = "destination3"; + static final int CONCURRENT_FILE_REQUESTS = 5; + + protected final LiveMigrationTaskFactory mockLiveMigrationTaskFactory = mock(LiveMigrationTaskFactory.class); + protected final LiveMigrationFilesVerificationTaskFactory mockFilesVerificationTaskFactory = mock(LiveMigrationFilesVerificationTaskFactory.class); + protected final SidecarConfiguration mockSidecarConfiguration = mock(SidecarConfiguration.class); + protected final ServiceConfiguration mockServiceConfiguration = mock(ServiceConfiguration.class); + protected final LiveMigrationConfiguration mockLiveMigrationConfiguration = mock(LiveMigrationConfiguration.class); + protected final LiveMigrationMap mockLiveMigrationMap = mock(LiveMigrationMap.class); + protected final InstanceMetadata mockDest1InstanceMeta = mock(InstanceMetadata.class); + protected final InstanceMetadata mockDest2InstanceMeta = mock(InstanceMetadata.class); + protected final InstanceMetadata mockDest3InstanceMeta = mock(InstanceMetadata.class); + protected final InstanceMetadata mockSourceInstanceMeta = mock(InstanceMetadata.class); + protected final InstancesMetadata mockInstancesMetadata = mock(InstancesMetadata.class); + protected final Vertx vertx; // = Vertx.vertx(); + + LiveMigrationTaskManagerTestModule(Vertx vertx) + { + this.vertx = vertx; + } + + @Override + protected void configure() + { + // Bind common dependencies + bind(Vertx.class).toInstance(vertx); + bind(SidecarConfiguration.class).toInstance(mockSidecarConfiguration); + bind(InstancesMetadata.class).toInstance(mockInstancesMetadata); + bind(LiveMigrationMap.class).toInstance(mockLiveMigrationMap); + bind(LiveMigrationTaskFactory.class).toInstance(mockLiveMigrationTaskFactory); + bind(LiveMigrationFilesVerificationTaskFactory.class).toInstance(mockFilesVerificationTaskFactory); + + // Configure SidecarConfiguration mocks + when(mockSidecarConfiguration.serviceConfiguration()).thenReturn(mockServiceConfiguration); + when(mockServiceConfiguration.port()).thenReturn(9043); + when(mockSidecarConfiguration.liveMigrationConfiguration()).thenReturn(mockLiveMigrationConfiguration); + when(mockLiveMigrationConfiguration.maxConcurrentFileRequests()).thenReturn(CONCURRENT_FILE_REQUESTS); + + // Configure InstanceMetadata mocks + List twoDataDirs = List.of("/data1", "/data2"); + configureInstanceMetadata(mockDest1InstanceMeta, DESTINATION_1, DEST_1_ID, twoDataDirs); + configureInstanceMetadata(mockDest2InstanceMeta, DESTINATION_2, DEST_2_ID, twoDataDirs); + configureInstanceMetadata(mockDest3InstanceMeta, DESTINATION_3, DEST_3_ID, twoDataDirs); + configureInstanceMetadata(mockSourceInstanceMeta, SOURCE_1, 0, List.of("/data1")); + + when(mockInstancesMetadata.instanceFromHost(DESTINATION_1)).thenReturn(mockDest1InstanceMeta); + when(mockInstancesMetadata.instanceFromHost(DESTINATION_2)).thenReturn(mockDest2InstanceMeta); + when(mockInstancesMetadata.instanceFromHost(DESTINATION_3)).thenReturn(mockDest3InstanceMeta); + when(mockInstancesMetadata.instanceFromHost(SOURCE_1)).thenReturn(mockSourceInstanceMeta); + + // Configure LiveMigrationMap + when(mockLiveMigrationMap.getSource(anyString())).thenReturn(Future.succeededFuture(SOURCE_1)); + + // Configure default factory behaviors + configureDataCopyTaskFactory(); + configureFilesVerificationTaskFactory(); + } + + private void configureInstanceMetadata(InstanceMetadata instanceMeta, String hostName, int id, List dataDirs) + { + when(instanceMeta.host()).thenReturn(hostName); + when(instanceMeta.id()).thenReturn(id); + when(instanceMeta.dataDirs()).thenReturn(dataDirs); + when(instanceMeta.delegate()).thenReturn(mock(CassandraAdapterDelegate.class)); + } + + private void configureDataCopyTaskFactory() + { + when(mockLiveMigrationTaskFactory.create(anyString(), any(LiveMigrationDataCopyRequest.class), anyString(), anyInt(), any(InstanceMetadata.class))) + .thenAnswer(invocation -> { + String id = invocation.getArgument(0); + LiveMigrationDataCopyRequest request = invocation.getArgument(1); + String source = invocation.getArgument(2); + int port = invocation.getArgument(3); + + List statusList = + List.of(new LiveMigrationDataCopyResponse.Status(0, "SUCCESS", 1000L, 1, 1, 1, 1, 0, 1000L)); + LiveMigrationDataCopyResponse taskResponse = new LiveMigrationDataCopyResponse(id, source, port, request, statusList); + return new FakeLiveMigrationTask(taskResponse); + }); + } + + private void configureFilesVerificationTaskFactory() + { + when(mockFilesVerificationTaskFactory.create(anyString(), anyString(), anyInt(), any(LiveMigrationFilesVerificationRequest.class), any(InstanceMetadata.class))) + .thenAnswer(invocation -> { + String id = invocation.getArgument(0); + String source = invocation.getArgument(1); + int port = invocation.getArgument(2); + + LiveMigrationFilesVerificationResponse response = new LiveMigrationFilesVerificationResponse( + id, "MD5", "COMPLETED", source, port, 0, 0, CONCURRENT_FILE_REQUESTS, 0, 0, CONCURRENT_FILE_REQUESTS, 0 + ); + return new FakeFilesVerificationTask(response); + }); + } +} diff --git a/server/src/test/java/org/apache/cassandra/sidecar/livemigration/TestFile.java b/server/src/test/java/org/apache/cassandra/sidecar/livemigration/TestFile.java new file mode 100644 index 000000000..2f9df0db7 --- /dev/null +++ b/server/src/test/java/org/apache/cassandra/sidecar/livemigration/TestFile.java @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.sidecar.livemigration; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +import org.apache.cassandra.sidecar.cluster.instance.InstanceMetadata; +import org.apache.cassandra.sidecar.common.response.InstanceFileInfo; +import org.apache.cassandra.sidecar.common.response.InstanceFileInfo.FileType; +import org.apache.cassandra.sidecar.common.response.InstanceFilesListResponse; +import org.apache.cassandra.sidecar.handlers.livemigration.LiveMigrationDirType; +import org.apache.cassandra.sidecar.utils.DigestAlgorithm; +import org.apache.cassandra.sidecar.utils.TestFileUtils; + +import static org.apache.cassandra.sidecar.common.ApiEndpointsV1.LIVE_MIGRATION_FILES_ROUTE; + +class TestFile +{ + public static final int DIR_FILE_SIZE = -1; + final LiveMigrationDirType dirType; + final int dirIndex; + final String relativePath; + final int size; + final long lastModifiedTime; + + public TestFile(LiveMigrationDirType dirType, int dirIndex, String relativePath, int size, long lastModifiedTime) + { + this.dirType = dirType; + this.dirIndex = dirIndex; + this.relativePath = relativePath; + this.size = size; + this.lastModifiedTime = lastModifiedTime; + } + + /** + * Converts a list of TestFile objects to a list of InstanceFileInfo objects. + */ + static List getInstanceFileInfo(List testFiles) + { + return testFiles.stream().map(TestFile::getInstanceFileInfo).collect(Collectors.toList()); + } + + /** + * Converts a list of TestFile objects to an InstanceFilesListResponse. + */ + static InstanceFilesListResponse getInstanceFilesListResponse(List testFiles) + { + return new InstanceFilesListResponse(getInstanceFileInfo(testFiles)); + } + + /** + * Returns the InstanceFileInfo representation of this test file. + */ + InstanceFileInfo getInstanceFileInfo() + { + return new InstanceFileInfo(getFileUrl(), size, getFileType(), lastModifiedTime); + } + + /** + * Returns the URL path for this file in the live migration API. + */ + String getFileUrl() + { + return LIVE_MIGRATION_FILES_ROUTE + "/" + dirType.dirType + "/" + dirIndex + "/" + relativePath; + } + + /** + * Returns the file type (DIRECTORY or FILE) based on the size field. + */ + FileType getFileType() + { + return size == DIR_FILE_SIZE ? FileType.DIRECTORY : FileType.FILE; + } + + /** + * Returns the absolute file path given a storage directory. + */ + String getFilePath(String storageDir) + { + return storageDir + "/" + dirType.dirType + "/" + relativePath; + } + + /** + * Creates the file or directory represented by this TestFile on the filesystem. + */ + void createFile(InstanceMetadata instanceMetadata) throws IOException + { + Path path = LiveMigrationInstanceMetadataUtil.localPath(getFileUrl(), instanceMetadata); + if (getFileType() == FileType.DIRECTORY) + { + Files.createDirectories(path); + return; + } + TestFileUtils.createFile(path.toAbsolutePath().toString(), size, lastModifiedTime); + } + + /** + * Deletes the file or directory represented by this TestFile from the filesystem. + */ + void deleteFile(InstanceMetadata instanceMetadata) throws IOException + { + Path path = LiveMigrationInstanceMetadataUtil.localPath(getFileUrl(), instanceMetadata); + Files.deleteIfExists(path); + } + + /** + * Calculates and returns the digest of this file using the provided algorithm supplier. + * Returns null for directories as they are not included in digest comparisons. + */ + String digest(InstanceMetadata instanceMetadata, Supplier digestAlgorithmSupplier) throws IOException + { + if (getFileType() == FileType.DIRECTORY) + { + // directories are not considered for digest comparison, hence returning null + return null; + } + Path path = LiveMigrationInstanceMetadataUtil.localPath(getFileUrl(), instanceMetadata); + byte[] bytes = Files.readAllBytes(path); + + DigestAlgorithm digestAlgorithm = digestAlgorithmSupplier.get(); + digestAlgorithm.update(bytes, 0, bytes.length); + return digestAlgorithm.digest(); + } +} diff --git a/server/src/test/java/org/apache/cassandra/sidecar/utils/AsyncFileDigestCalculatorTest.java b/server/src/test/java/org/apache/cassandra/sidecar/utils/AsyncFileDigestCalculatorTest.java new file mode 100644 index 000000000..3601c37ea --- /dev/null +++ b/server/src/test/java/org/apache/cassandra/sidecar/utils/AsyncFileDigestCalculatorTest.java @@ -0,0 +1,237 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.sidecar.utils; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.Base64; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.io.TempDir; + +import io.vertx.core.Vertx; +import io.vertx.core.file.OpenOptions; +import io.vertx.junit5.VertxExtension; +import io.vertx.junit5.VertxTestContext; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** + * Unit tests for {@link AsyncFileDigestCalculator} + */ +@ExtendWith(VertxExtension.class) +class AsyncFileDigestCalculatorTest +{ + + @Test + void testCalculateDigestForSmallFile(Vertx vertx, VertxTestContext context, @TempDir Path tempDir) throws IOException, NoSuchAlgorithmException + { + Path testFile = TestFileUtils.prepareTestFile(tempDir, "small-file.txt", 1024); + String expectedDigest = md5Sum(testFile); + DigestAlgorithm digestAlgorithm = new JdkMd5DigestProvider().get(); + + AsyncFileDigestCalculator.calculateDigest(vertx, testFile.toString(), digestAlgorithm) + .onComplete(context.succeeding(digest -> context.verify(() -> { + assertThat(digest).isNotNull(); + assertThat(digest).isNotEmpty(); + assertThat(digest).isEqualTo(expectedDigest); + context.completeNow(); + }))); + } + + @Test + void testCalculateDigestForLargeFile(Vertx vertx, VertxTestContext context, @TempDir Path tempDir) throws IOException, NoSuchAlgorithmException + { + // File larger than DEFAULT_READ_BUFFER_SIZE (512KB) + Path testFile = TestFileUtils.prepareTestFile(tempDir, "large-file.txt", 1024 * 1024); + String expectedDigest = md5Sum(testFile); + DigestAlgorithm digestAlgorithm = new JdkMd5DigestProvider().get(); + + AsyncFileDigestCalculator.calculateDigest(vertx, testFile.toString(), digestAlgorithm) + .onComplete(context.succeeding(digest -> context.verify(() -> { + assertThat(digest).isNotNull(); + assertThat(digest).isNotEmpty(); + assertThat(digest).isEqualTo(expectedDigest); + context.completeNow(); + }))); + } + + @Test + void testCalculateDigestForEmptyFile(Vertx vertx, VertxTestContext context, @TempDir Path tempDir) throws IOException, NoSuchAlgorithmException + { + Path testFile = Files.createFile(tempDir.resolve("empty-file.txt")); + String expectedDigest = md5Sum(testFile); + DigestAlgorithm digestAlgorithm = new JdkMd5DigestProvider().get(); + + AsyncFileDigestCalculator.calculateDigest(vertx, testFile.toString(), digestAlgorithm) + .onComplete(context.succeeding(digest -> context.verify(() -> { + assertThat(digest).isNotNull(); + assertThat(digest).isNotEmpty(); + assertThat(digest).isEqualTo(expectedDigest); + context.completeNow(); + }))); + } + + @Test + void testCalculateDigestWithXXHash32(Vertx vertx, VertxTestContext context, @TempDir Path tempDir) throws IOException + { + Path testFile = TestFileUtils.prepareTestFile(tempDir, "test-xxhash.txt", 2048); + String expectedDigest = xxHash32Sum(testFile); + DigestAlgorithm digestAlgorithm = new XXHash32Provider().get(); + + AsyncFileDigestCalculator.calculateDigest(vertx, testFile.toString(), digestAlgorithm) + .onComplete(context.succeeding(digest -> context.verify(() -> { + assertThat(digest).isNotNull(); + assertThat(digest).isNotEmpty(); + assertThat(digest).isEqualTo(expectedDigest); + context.completeNow(); + }))); + } + + @Test + void testCalculateDigestWithXXHash32WithSeed(Vertx vertx, + VertxTestContext context, + @TempDir Path tempDir) throws IOException + { + Path testFile = TestFileUtils.prepareTestFile(tempDir, "test-xxhash-seed.txt", 2048); + String expectedDigest = xxHash32Sum(testFile, 12345); + DigestAlgorithm digestAlgorithm = new XXHash32Provider().get(12345); + + AsyncFileDigestCalculator.calculateDigest(vertx, testFile.toString(), digestAlgorithm) + .onComplete(context.succeeding(digest -> context.verify(() -> { + assertThat(digest).isNotNull(); + assertThat(digest).isNotEmpty(); + assertThat(digest).isEqualTo(expectedDigest); + context.completeNow(); + }))); + } + + @Test + void testCalculateDigestForNonExistentFile(Vertx vertx, VertxTestContext context, @TempDir Path tempDir) + { + Path nonExistentFile = tempDir.resolve("does-not-exist.txt"); + DigestAlgorithm digestAlgorithm = new JdkMd5DigestProvider().get(); + + AsyncFileDigestCalculator.calculateDigest(vertx, nonExistentFile.toString(), digestAlgorithm) + .onComplete(context.failing(cause -> context.verify(() -> { + assertThat(cause).isNotNull(); + context.completeNow(); + }))); + } + + @Test + void testCalculateDigestConsistency(Vertx vertx, VertxTestContext context, @TempDir Path tempDir) throws IOException, NoSuchAlgorithmException + { + Path testFile = TestFileUtils.prepareTestFile(tempDir, "consistent-file.txt", 4096); + String expectedDigest = md5Sum(testFile); + DigestAlgorithm digestAlgorithm1 = new JdkMd5DigestProvider().get(); + DigestAlgorithm digestAlgorithm2 = new JdkMd5DigestProvider().get(); + + AsyncFileDigestCalculator.calculateDigest(vertx, testFile.toString(), digestAlgorithm1) + .compose(digest1 -> + AsyncFileDigestCalculator.calculateDigest(vertx, testFile.toString(), digestAlgorithm2) + .map(digest2 -> { + assertThat(digest1).isEqualTo(digest2); + assertThat(digest1).isEqualTo(expectedDigest); + assertThat(digest2).isEqualTo(expectedDigest); + return digest2; + }) + ) + .onComplete(context.succeeding(digest -> context.verify(() -> { + assertThat(digest).isNotNull(); + context.completeNow(); + }))); + } + + @Test + void testCalculateDigestWithDifferentContent(Vertx vertx, + VertxTestContext context, + @TempDir Path tempDir) throws IOException, NoSuchAlgorithmException + { + Path testFile1 = TestFileUtils.prepareTestFile(tempDir, "file1.txt", 1024); + Path testFile2 = TestFileUtils.prepareTestFile(tempDir, "file2.txt", 1024); + String expectedDigest1 = md5Sum(testFile1); + String expectedDigest2 = md5Sum(testFile2); + DigestAlgorithm digestAlgorithm1 = new JdkMd5DigestProvider().get(); + DigestAlgorithm digestAlgorithm2 = new JdkMd5DigestProvider().get(); + + AsyncFileDigestCalculator.calculateDigest(vertx, testFile1.toString(), digestAlgorithm1) + .compose(digest1 -> + AsyncFileDigestCalculator.calculateDigest(vertx, testFile2.toString(), digestAlgorithm2) + .map(digest2 -> { + assertThat(digest1).isNotEqualTo(digest2); + assertThat(digest1).isEqualTo(expectedDigest1); + assertThat(digest2).isEqualTo(expectedDigest2); + return digest2; + }) + ) + .onComplete(context.succeeding(digest -> context.verify(() -> { + assertThat(digest).isNotNull(); + context.completeNow(); + }))); + } + + @Test + void testAsyncFileIsClosedOnSuccess(Vertx vertx, VertxTestContext context, @TempDir Path tempDir) throws IOException, NoSuchAlgorithmException + { + Path testFile = TestFileUtils.prepareTestFile(tempDir, "file-closed-success.txt", 1024); + String expectedDigest = md5Sum(testFile); + DigestAlgorithm digestAlgorithm = new JdkMd5DigestProvider().get(); + + vertx.fileSystem() + .open(testFile.toString(), new OpenOptions().setRead(true)) + .onSuccess(asyncFile -> { + AsyncFileDigestCalculator.calculateDigest(asyncFile, digestAlgorithm) + .onComplete(context.succeeding(digest -> context.verify(() -> { + assertThat(digest).isNotNull(); + assertThat(digest).isEqualTo(expectedDigest); + // Closing a file which is already closed throws IllegalStateException + // Closing it again to verify that it is closed + assertThatThrownBy(asyncFile::end).isInstanceOf(IllegalStateException.class); + context.completeNow(); + }))); + }); + } + + String md5Sum(Path path) throws NoSuchAlgorithmException, IOException + { + MessageDigest md5 = MessageDigest.getInstance("MD5"); + byte[] bytes = Files.readAllBytes(path); + return Base64.getEncoder() + .encodeToString(md5.digest(bytes)); + } + + String xxHash32Sum(Path path) throws IOException + { + return xxHash32Sum(path, 0); + } + + String xxHash32Sum(Path path, int seed) throws IOException + { + DigestAlgorithm digestAlgorithm = new XXHash32Provider().get(seed); + byte[] bytes = Files.readAllBytes(path); + digestAlgorithm.update(bytes, 0, bytes.length); + return digestAlgorithm.digest(); + } +} diff --git a/server/src/test/java/org/apache/cassandra/sidecar/utils/DigestAlgorithmFactoryTest.java b/server/src/test/java/org/apache/cassandra/sidecar/utils/DigestAlgorithmFactoryTest.java new file mode 100644 index 000000000..bc3f91676 --- /dev/null +++ b/server/src/test/java/org/apache/cassandra/sidecar/utils/DigestAlgorithmFactoryTest.java @@ -0,0 +1,159 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.sidecar.utils; + +import org.junit.jupiter.api.Test; + +import com.google.inject.Guice; +import com.google.inject.Injector; +import com.google.inject.Key; +import com.google.inject.name.Names; +import org.apache.cassandra.sidecar.HelperTestModules.DigestAlgorithmProviderTestModule; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +/** + * Test {@link DigestAlgorithmFactory} behavior. + */ +class DigestAlgorithmFactoryTest +{ + @Test + void testGetMD5Algorithm() + { + Injector injector = getInjector(); + DigestAlgorithmFactory factory = injector.getInstance(DigestAlgorithmFactory.class); + DigestAlgorithm result = factory.getDigestAlgorithm("MD5", null); + + assertThat(result).isInstanceOf(JdkMd5DigestProvider.JdkMD5DigestAlgorithm.class); + } + + @Test + void testGetMD5AlgorithmCaseInsensitive() + { + Injector injector = getInjector(); + DigestAlgorithmFactory factory = injector.getInstance(DigestAlgorithmFactory.class); + DigestAlgorithm result = factory.getDigestAlgorithm("mD5", null); + + assertThat(result).isInstanceOf(JdkMd5DigestProvider.JdkMD5DigestAlgorithm.class); + } + + @Test + void testGetXXHash32AlgorithmWithoutSeed() + { + Injector injector = getInjector(); + DigestAlgorithmFactory factory = injector.getInstance(DigestAlgorithmFactory.class); + DigestAlgorithm result = factory.getDigestAlgorithm("XXHash32", null); + + assertThat(result).isInstanceOf(XXHash32Provider.Lz4XXHash32DigestAlgorithm.class); + } + + @Test + void testGetXXHash32AlgorithmCaseInsensitive() + { + Injector injector = getInjector(); + DigestAlgorithmFactory factory = injector.getInstance(DigestAlgorithmFactory.class); + DigestAlgorithm result = factory.getDigestAlgorithm("xxhASh32", null); + + assertThat(result).isInstanceOf(XXHash32Provider.Lz4XXHash32DigestAlgorithm.class); + } + + @Test + void testGetXXHash32AlgorithmWithSeed() + { + Injector injector = getInjector(); + DigestAlgorithmFactory factory = injector.getInstance(DigestAlgorithmFactory.class); + DigestAlgorithm result = factory.getDigestAlgorithm("XXHash32", 12345); + + assertThat(result).isInstanceOf(XXHash32Provider.Lz4XXHash32DigestAlgorithm.class); + + DigestAlgorithmProvider xxhash32AlgorithmProvider = injector.getInstance(Key.get(DigestAlgorithmProvider.class, Names.named("xxhash32"))); + verify(xxhash32AlgorithmProvider, times(1)).get(12345); + } + + @Test + void testUnsupportedAlgorithm() + { + Injector injector = getInjector(); + DigestAlgorithmFactory factory = injector.getInstance(DigestAlgorithmFactory.class); + + assertThatThrownBy(() -> factory.getDigestAlgorithm("SHA256", null)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Unsupported digest algorithm SHA256"); + } + + + @Test + void testEmptyAlgorithmName() + { + Injector injector = getInjector(); + DigestAlgorithmFactory factory = injector.getInstance(DigestAlgorithmFactory.class); + + assertThatThrownBy(() -> factory.getDigestAlgorithm(null, null)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Digest algorithm name cannot be null or empty"); + + assertThatThrownBy(() -> factory.getDigestAlgorithm("", null)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Digest algorithm name cannot be null or empty"); + + assertThatThrownBy(() -> factory.getDigestAlgorithm(" ", null)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Digest algorithm name cannot be null or empty"); + } + + @Test + void testValidateDigestAlgorithm() + { + // Valid algorithms should not throw (case-insensitive) + DigestAlgorithmFactory.validateAlgorithmName("MD5"); + DigestAlgorithmFactory.validateAlgorithmName("XXHash32"); + DigestAlgorithmFactory.validateAlgorithmName("md5"); + DigestAlgorithmFactory.validateAlgorithmName("xxhash32"); + DigestAlgorithmFactory.validateAlgorithmName("XXHASH32"); + + // Invalid algorithm should throw with descriptive message + assertThatThrownBy(() -> DigestAlgorithmFactory.validateAlgorithmName("SHA256")) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Unsupported digest algorithm SHA256"); + + // Null algorithm should throw + //noinspection DataFlowIssue + assertThatThrownBy(() -> DigestAlgorithmFactory.validateAlgorithmName(null)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("cannot be null or empty"); + + // Empty string should throw + assertThatThrownBy(() -> DigestAlgorithmFactory.validateAlgorithmName("")) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("cannot be null or empty"); + + // Blank string should throw + assertThatThrownBy(() -> DigestAlgorithmFactory.validateAlgorithmName(" ")) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("cannot be null or empty"); + } + + Injector getInjector() + { + return Guice.createInjector(new DigestAlgorithmProviderTestModule()); + } +} diff --git a/server/src/test/java/org/apache/cassandra/sidecar/utils/MD5DigestVerifierTest.java b/server/src/test/java/org/apache/cassandra/sidecar/utils/MD5DigestVerifierTest.java index fb6de409b..54407d2ff 100644 --- a/server/src/test/java/org/apache/cassandra/sidecar/utils/MD5DigestVerifierTest.java +++ b/server/src/test/java/org/apache/cassandra/sidecar/utils/MD5DigestVerifierTest.java @@ -114,14 +114,14 @@ static class ExposeAsyncFileMD5DigestVerifier extends MD5DigestVerifier public ExposeAsyncFileMD5DigestVerifier(FileSystem fs, MD5Digest md5Digest) { - super(fs, md5Digest, new JdkMd5DigestProvider.JdkMD5Digest()); + super(fs, md5Digest, new JdkMd5DigestProvider.JdkMD5DigestAlgorithm()); } @Override protected Future calculateDigest(AsyncFile file) { this.file = file; - return super.calculateDigest(file); + return AsyncFileDigestCalculator.calculateDigest(file, new JdkMd5DigestProvider.JdkMD5DigestAlgorithm()); } } } diff --git a/server/src/test/java/org/apache/cassandra/sidecar/utils/TestFileUtils.java b/server/src/test/java/org/apache/cassandra/sidecar/utils/TestFileUtils.java index 764c87f6f..4f2a42e69 100644 --- a/server/src/test/java/org/apache/cassandra/sidecar/utils/TestFileUtils.java +++ b/server/src/test/java/org/apache/cassandra/sidecar/utils/TestFileUtils.java @@ -18,12 +18,14 @@ package org.apache.cassandra.sidecar.utils; +import java.io.File; import java.io.IOException; import java.io.OutputStream; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.nio.file.attribute.FileTime; import java.util.concurrent.ThreadLocalRandom; /** @@ -35,7 +37,7 @@ public class TestFileUtils * Writes random data to a file with name {@code filename} under the specified {@code directory} with * the specified size in bytes. * - * @param directory the directory where to + * @param directory the directory in which files needs to be created * @param fileName the name of the desired file to create * @param sizeInBytes the size of the files in bytes * @return the path of the file that was recently created @@ -76,7 +78,7 @@ public static void createFile(String content, String first, String... more) thro Path parent = file.getParent(); if (null == parent) { - throw new IOException("Parent doesn't exists for file " + file); + throw new IOException("Parent doesn't exist for file " + file); } Files.createDirectories(parent); Files.write(file, content.getBytes(StandardCharsets.UTF_8)); @@ -95,4 +97,25 @@ public static void createDirectory(String first, String... more) throws IOExcept Path dir = Paths.get(first, more); Files.createDirectories(dir); } + + public static void createFile(String file, int size, long lastModifiedTime) throws IOException + { + createFile(new File(file), size, lastModifiedTime); + } + + public static void createFile(File f, int size, long lastModifiedTime) throws IOException + { + Path p = f.toPath(); + Files.createDirectories(p.getParent()); + Files.createFile(p); + byte[] bytes = new byte[size]; + ThreadLocalRandom.current().nextBytes(bytes); + Files.write(p, bytes); + Files.setLastModifiedTime(p, FileTime.fromMillis(lastModifiedTime)); + } + + public static void createDirectory(File f) throws IOException + { + Files.createDirectories(f.toPath()); + } } diff --git a/server/src/test/java/org/apache/cassandra/sidecar/utils/XXHash32DigestVerifierTest.java b/server/src/test/java/org/apache/cassandra/sidecar/utils/XXHash32DigestVerifierTest.java index 21efed982..f58ebff7a 100644 --- a/server/src/test/java/org/apache/cassandra/sidecar/utils/XXHash32DigestVerifierTest.java +++ b/server/src/test/java/org/apache/cassandra/sidecar/utils/XXHash32DigestVerifierTest.java @@ -138,7 +138,7 @@ static class ExposeAsyncFileXXHash32DigestVerifier extends XXHash32DigestVerifie public ExposeAsyncFileXXHash32DigestVerifier(FileSystem fs, XXHash32Digest digest) { - super(fs, digest, new XXHash32Provider.Lz4XXHash32(maybeGetSeedOrDefault(digest))); + super(fs, digest, new XXHash32Provider.Lz4XXHash32DigestAlgorithm(maybeGetSeedOrDefault(digest))); } @Override diff --git a/server/src/test/resources/config/sidecar_live_migration.yaml b/server/src/test/resources/config/sidecar_live_migration.yaml index 51cacfdc8..d3ff3fb86 100644 --- a/server/src/test/resources/config/sidecar_live_migration.yaml +++ b/server/src/test/resources/config/sidecar_live_migration.yaml @@ -211,4 +211,4 @@ live_migration: - ${DATA_FILE_DIR}/*/*/snapshots migration_map: localhost1: localhost4 - max_concurrent_downloads: 20 + max_concurrent_file_requests: 20