diff --git a/.github/workflows/java_coverage.yml b/.github/workflows/java_coverage.yml index 0ad1e25ed..89f65b2c2 100644 --- a/.github/workflows/java_coverage.yml +++ b/.github/workflows/java_coverage.yml @@ -19,7 +19,8 @@ jobs: java-version: "11" distribution: "adopt" - name: Generate coverage report - run: mvn test --file ./java/pom.xml + working-directory: ./java + run: mvn test --file ./pom.xml - name: Test summary uses: test-summary/action@v1 with: diff --git a/java/client-apis/src/main/java/org/apache/rocketmq/client/apis/SessionCredentials.java b/java/client-apis/src/main/java/org/apache/rocketmq/client/apis/SessionCredentials.java index 5328eab40..b1c32a9e3 100644 --- a/java/client-apis/src/main/java/org/apache/rocketmq/client/apis/SessionCredentials.java +++ b/java/client-apis/src/main/java/org/apache/rocketmq/client/apis/SessionCredentials.java @@ -19,26 +19,39 @@ import static com.google.common.base.Preconditions.checkNotNull; +import java.time.Duration; import java.util.Optional; /** * Session credentials used in service authentications. */ public class SessionCredentials { + private static final Duration EXPIRATION_BUFFER_TIME = Duration.ofSeconds(1); private final String accessKey; private final String accessSecret; private final String securityToken; + private final long expiredTimestampMillis; + + public SessionCredentials(String accessKey, String accessSecret, String securityToken, + long expiredTimestampMillis) { + this.accessKey = checkNotNull(accessKey, "accessKey should not be null"); + this.accessSecret = checkNotNull(accessSecret, "accessSecret should not be null"); + this.securityToken = checkNotNull(securityToken, "securityToken should not be null"); + this.expiredTimestampMillis = expiredTimestampMillis; + } public SessionCredentials(String accessKey, String accessSecret, String securityToken) { this.accessKey = checkNotNull(accessKey, "accessKey should not be null"); this.accessSecret = checkNotNull(accessSecret, "accessSecret should not be null"); this.securityToken = checkNotNull(securityToken, "securityToken should not be null"); + this.expiredTimestampMillis = Long.MAX_VALUE; } public SessionCredentials(String accessKey, String accessSecret) { this.accessKey = checkNotNull(accessKey, "accessKey should not be null"); this.accessSecret = checkNotNull(accessSecret, "accessSecret should not be null"); this.securityToken = null; + this.expiredTimestampMillis = Long.MAX_VALUE; } public String getAccessKey() { @@ -52,4 +65,8 @@ public String getAccessSecret() { public Optional tryGetSecurityToken() { return null == securityToken ? Optional.empty() : Optional.of(securityToken); } + + public boolean expiredSoon() { + return System.currentTimeMillis() + EXPIRATION_BUFFER_TIME.toMillis() > expiredTimestampMillis; + } } diff --git a/java/client-apis/src/main/java/org/apache/rocketmq/client/apis/consumer/TopicMessageQueueChangeListener.java b/java/client-apis/src/main/java/org/apache/rocketmq/client/apis/consumer/TopicMessageQueueChangeListener.java new file mode 100644 index 000000000..0b8ac46c4 --- /dev/null +++ b/java/client-apis/src/main/java/org/apache/rocketmq/client/apis/consumer/TopicMessageQueueChangeListener.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.client.apis.consumer; + +import java.util.Set; +import org.apache.rocketmq.client.apis.message.MessageQueue; + +public interface TopicMessageQueueChangeListener { + /** + * This method will be invoked in the condition of queue numbers changed, These scenarios occur when the topic is + * expanded or shrunk. + * + * @param topic the topic to listen. + * @param messageQueues latest message queues of the topic. + */ + void onChanged(String topic, Set messageQueues); +} \ No newline at end of file diff --git a/java/client-apis/src/main/java/org/apache/rocketmq/client/apis/message/MessageQueue.java b/java/client-apis/src/main/java/org/apache/rocketmq/client/apis/message/MessageQueue.java new file mode 100644 index 000000000..5331db40c --- /dev/null +++ b/java/client-apis/src/main/java/org/apache/rocketmq/client/apis/message/MessageQueue.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.client.apis.message; + +public interface MessageQueue { + /** + * Topic of the current message queue. + */ + String getTopic(); +} \ No newline at end of file diff --git a/java/client-apis/src/main/java/org/apache/rocketmq/client/apis/producer/ProducerBuilder.java b/java/client-apis/src/main/java/org/apache/rocketmq/client/apis/producer/ProducerBuilder.java index cc4bdc6f0..13b6e65bd 100644 --- a/java/client-apis/src/main/java/org/apache/rocketmq/client/apis/producer/ProducerBuilder.java +++ b/java/client-apis/src/main/java/org/apache/rocketmq/client/apis/producer/ProducerBuilder.java @@ -48,7 +48,7 @@ public interface ProducerBuilder { * ArrayList topicList = new ArrayList<>(); * topicList.add("topicA"); * topicList.add("topicB"); - * producerBuilder.setTopics(topicList); + * producerBuilder.setTopics(topicList.toArray(new String[0])); * } * * @param topics topics to send/prepare. diff --git a/java/client/src/main/java/org/apache/rocketmq/client/java/exception/StatusChecker.java b/java/client/src/main/java/org/apache/rocketmq/client/java/exception/StatusChecker.java index 76f5a225c..dcd3d7ce3 100644 --- a/java/client/src/main/java/org/apache/rocketmq/client/java/exception/StatusChecker.java +++ b/java/client/src/main/java/org/apache/rocketmq/client/java/exception/StatusChecker.java @@ -18,6 +18,8 @@ package org.apache.rocketmq.client.java.exception; import apache.rocketmq.v2.Code; +import apache.rocketmq.v2.GetOffsetRequest; +import apache.rocketmq.v2.PullMessageRequest; import apache.rocketmq.v2.ReceiveMessageRequest; import apache.rocketmq.v2.Status; import org.apache.rocketmq.client.apis.ClientException; @@ -61,6 +63,11 @@ public static void check(Status status, RpcFuture future) throws ClientExc case CLIENT_ID_REQUIRED: case ILLEGAL_POLLING_TIME: throw new BadRequestException(codeNumber, requestId, statusMessage); + case ILLEGAL_OFFSET: + if (future.getRequest() instanceof PullMessageRequest) { + return; + } + // fall through on purpose. case UNAUTHORIZED: throw new UnauthorizedException(codeNumber, requestId, statusMessage); case PAYMENT_REQUIRED: @@ -71,11 +78,19 @@ public static void check(Status status, RpcFuture future) throws ClientExc if (future.getRequest() instanceof ReceiveMessageRequest) { return; } + if (future.getRequest() instanceof PullMessageRequest) { + return; + } // fall through on purpose. case NOT_FOUND: case TOPIC_NOT_FOUND: case CONSUMER_GROUP_NOT_FOUND: throw new NotFoundException(codeNumber, requestId, statusMessage); + case OFFSET_NOT_FOUND: + if (future.getRequest() instanceof GetOffsetRequest) { + return; + } + // fall through on purpose. case PAYLOAD_TOO_LARGE: case MESSAGE_BODY_TOO_LARGE: throw new PayloadTooLargeException(codeNumber, requestId, statusMessage); diff --git a/java/client/src/main/java/org/apache/rocketmq/client/java/hook/MessageHookPoints.java b/java/client/src/main/java/org/apache/rocketmq/client/java/hook/MessageHookPoints.java index 4f7ac523f..53a82dba2 100644 --- a/java/client/src/main/java/org/apache/rocketmq/client/java/hook/MessageHookPoints.java +++ b/java/client/src/main/java/org/apache/rocketmq/client/java/hook/MessageHookPoints.java @@ -26,6 +26,10 @@ public enum MessageHookPoints { * The hook point of message reception. */ RECEIVE, + /** + * The hook point of message pulling. + */ + PULL, /** * The hook point of message consumption. */ diff --git a/java/client/src/main/java/org/apache/rocketmq/client/java/impl/ClientImpl.java b/java/client/src/main/java/org/apache/rocketmq/client/java/impl/ClientImpl.java index 846f0ce9e..e39c72e46 100644 --- a/java/client/src/main/java/org/apache/rocketmq/client/java/impl/ClientImpl.java +++ b/java/client/src/main/java/org/apache/rocketmq/client/java/impl/ClientImpl.java @@ -588,7 +588,7 @@ public void onFailure(Throwable t) { public void doStats() { } - private ListenableFuture fetchTopicRoute(final String topic) { + protected ListenableFuture fetchTopicRoute(final String topic) { final ListenableFuture future0 = fetchTopicRoute0(topic); final ListenableFuture future = Futures.transformAsync(future0, topicRouteData -> onTopicRouteDataFetched(topic, topicRouteData), MoreExecutors.directExecutor()); diff --git a/java/client/src/main/java/org/apache/rocketmq/client/java/impl/ClientManager.java b/java/client/src/main/java/org/apache/rocketmq/client/java/impl/ClientManager.java index ce7d2e1b0..b8f65dfec 100644 --- a/java/client/src/main/java/org/apache/rocketmq/client/java/impl/ClientManager.java +++ b/java/client/src/main/java/org/apache/rocketmq/client/java/impl/ClientManager.java @@ -25,12 +25,18 @@ import apache.rocketmq.v2.EndTransactionResponse; import apache.rocketmq.v2.ForwardMessageToDeadLetterQueueRequest; import apache.rocketmq.v2.ForwardMessageToDeadLetterQueueResponse; +import apache.rocketmq.v2.GetOffsetRequest; +import apache.rocketmq.v2.GetOffsetResponse; import apache.rocketmq.v2.HeartbeatRequest; import apache.rocketmq.v2.HeartbeatResponse; import apache.rocketmq.v2.NotifyClientTerminationRequest; import apache.rocketmq.v2.NotifyClientTerminationResponse; +import apache.rocketmq.v2.PullMessageRequest; +import apache.rocketmq.v2.PullMessageResponse; import apache.rocketmq.v2.QueryAssignmentRequest; import apache.rocketmq.v2.QueryAssignmentResponse; +import apache.rocketmq.v2.QueryOffsetRequest; +import apache.rocketmq.v2.QueryOffsetResponse; import apache.rocketmq.v2.QueryRouteRequest; import apache.rocketmq.v2.QueryRouteResponse; import apache.rocketmq.v2.ReceiveMessageRequest; @@ -38,6 +44,8 @@ import apache.rocketmq.v2.SendMessageRequest; import apache.rocketmq.v2.SendMessageResponse; import apache.rocketmq.v2.TelemetryCommand; +import apache.rocketmq.v2.UpdateOffsetRequest; +import apache.rocketmq.v2.UpdateOffsetResponse; import com.google.common.util.concurrent.AbstractIdleService; import io.grpc.stub.StreamObserver; import java.time.Duration; @@ -147,6 +155,18 @@ public abstract RpcFuture ackMessage(Endp ForwardMessageToDeadLetterQueueResponse> forwardMessageToDeadLetterQueue(Endpoints endpoints, ForwardMessageToDeadLetterQueueRequest request, Duration duration); + public abstract RpcFuture> pullMessage(Endpoints endpoints, + PullMessageRequest request, Duration duration); + + public abstract RpcFuture updateOffset(Endpoints endpoints, + UpdateOffsetRequest request, Duration duration); + + public abstract RpcFuture getOffset(Endpoints endpoints, + GetOffsetRequest request, Duration duration); + + public abstract RpcFuture queryOffset(Endpoints endpoints, + QueryOffsetRequest request, Duration duration); + /** * Submit transaction resolution asynchronously, the method ensures no throwable. * diff --git a/java/client/src/main/java/org/apache/rocketmq/client/java/impl/ClientManagerImpl.java b/java/client/src/main/java/org/apache/rocketmq/client/java/impl/ClientManagerImpl.java index 2ae5f5e42..1d7c9daa6 100644 --- a/java/client/src/main/java/org/apache/rocketmq/client/java/impl/ClientManagerImpl.java +++ b/java/client/src/main/java/org/apache/rocketmq/client/java/impl/ClientManagerImpl.java @@ -25,12 +25,18 @@ import apache.rocketmq.v2.EndTransactionResponse; import apache.rocketmq.v2.ForwardMessageToDeadLetterQueueRequest; import apache.rocketmq.v2.ForwardMessageToDeadLetterQueueResponse; +import apache.rocketmq.v2.GetOffsetRequest; +import apache.rocketmq.v2.GetOffsetResponse; import apache.rocketmq.v2.HeartbeatRequest; import apache.rocketmq.v2.HeartbeatResponse; import apache.rocketmq.v2.NotifyClientTerminationRequest; import apache.rocketmq.v2.NotifyClientTerminationResponse; +import apache.rocketmq.v2.PullMessageRequest; +import apache.rocketmq.v2.PullMessageResponse; import apache.rocketmq.v2.QueryAssignmentRequest; import apache.rocketmq.v2.QueryAssignmentResponse; +import apache.rocketmq.v2.QueryOffsetRequest; +import apache.rocketmq.v2.QueryOffsetResponse; import apache.rocketmq.v2.QueryRouteRequest; import apache.rocketmq.v2.QueryRouteResponse; import apache.rocketmq.v2.ReceiveMessageRequest; @@ -38,6 +44,8 @@ import apache.rocketmq.v2.SendMessageRequest; import apache.rocketmq.v2.SendMessageResponse; import apache.rocketmq.v2.TelemetryCommand; +import apache.rocketmq.v2.UpdateOffsetRequest; +import apache.rocketmq.v2.UpdateOffsetResponse; import com.google.common.util.concurrent.ListenableFuture; import com.google.errorprone.annotations.concurrent.GuardedBy; import io.grpc.Metadata; @@ -283,8 +291,7 @@ public RpcFuture ackMessage(Endpoints end @Override public RpcFuture - changeInvisibleDuration(Endpoints endpoints, ChangeInvisibleDurationRequest request, - Duration duration) { + changeInvisibleDuration(Endpoints endpoints, ChangeInvisibleDurationRequest request, Duration duration) { try { final Metadata metadata = client.sign(); final Context context = new Context(endpoints, metadata); @@ -313,6 +320,66 @@ public RpcFuture ackMessage(Endpoints end } } + @Override + public RpcFuture> pullMessage(Endpoints endpoints, + PullMessageRequest request, Duration duration) { + try { + final Metadata metadata = client.sign(); + final Context context = new Context(endpoints, metadata); + final RpcClient rpcClient = getRpcClient(endpoints); + final ListenableFuture> future = + rpcClient.pullMessage(metadata, request, asyncWorker, duration); + return new RpcFuture<>(context, request, future); + } catch (Throwable t) { + return new RpcFuture<>(t); + } + } + + @Override + public RpcFuture updateOffset(Endpoints endpoints, + UpdateOffsetRequest request, Duration duration) { + try { + final Metadata metadata = client.sign(); + final Context context = new Context(endpoints, metadata); + final RpcClient rpcClient = getRpcClient(endpoints); + final ListenableFuture future = + rpcClient.updateOffset(metadata, request, asyncWorker, duration); + return new RpcFuture<>(context, request, future); + } catch (Throwable t) { + return new RpcFuture<>(t); + } + } + + @Override + public RpcFuture getOffset(Endpoints endpoints, GetOffsetRequest request, + Duration duration) { + try { + final Metadata metadata = client.sign(); + final Context context = new Context(endpoints, metadata); + final RpcClient rpcClient = getRpcClient(endpoints); + final ListenableFuture future = + rpcClient.getOffset(metadata, request, asyncWorker, duration); + return new RpcFuture<>(context, request, future); + } catch (Throwable t) { + return new RpcFuture<>(t); + } + } + + @Override + public RpcFuture queryOffset(Endpoints endpoints, + QueryOffsetRequest request, Duration duration) { + try { + final Metadata metadata = client.sign(); + final Context context = new Context(endpoints, metadata); + final RpcClient rpcClient = getRpcClient(endpoints); + final ListenableFuture future = + rpcClient.queryOffset(metadata, request, asyncWorker, duration); + return new RpcFuture<>(context, request, future); + } catch (Throwable t) { + return new RpcFuture<>(t); + } + } + @Override public RpcFuture endTransaction(Endpoints endpoints, EndTransactionRequest request, Duration duration) { @@ -395,9 +462,9 @@ protected void startUp() { () -> { try { log.info("Start to log statistics, clientVersion={}, clientWrapperVersion={}, " - + "clientEndpoints={}, os description=[{}], java description=[{}], clientId={}", + + "clientEndpoints={}, os description=[{}], java environment=[{}], clientId={}", MetadataUtils.getVersion(), MetadataUtils.getWrapperVersion(), client.getEndpoints(), - Utilities.getOsDescription(), Utilities.getJavaDescription(), clientId); + Utilities.getOsDescription(), Utilities.getJavaEnvironmentSummary(), clientId); client.doStats(); } catch (Throwable t) { log.error("Exception raised during statistics logging, clientId={}", clientId, t); diff --git a/java/client/src/main/java/org/apache/rocketmq/client/java/impl/ClientType.java b/java/client/src/main/java/org/apache/rocketmq/client/java/impl/ClientType.java index d514a8ac7..a9709c6da 100644 --- a/java/client/src/main/java/org/apache/rocketmq/client/java/impl/ClientType.java +++ b/java/client/src/main/java/org/apache/rocketmq/client/java/impl/ClientType.java @@ -20,7 +20,8 @@ public enum ClientType { PRODUCER, PUSH_CONSUMER, - SIMPLE_CONSUMER; + SIMPLE_CONSUMER, + PULL_CONSUMER; public apache.rocketmq.v2.ClientType toProtobuf() { if (PRODUCER.equals(this)) { @@ -32,6 +33,9 @@ public apache.rocketmq.v2.ClientType toProtobuf() { if (SIMPLE_CONSUMER.equals(this)) { return apache.rocketmq.v2.ClientType.SIMPLE_CONSUMER; } + if (PULL_CONSUMER.equals(this)) { + return apache.rocketmq.v2.ClientType.PULL_CONSUMER; + } return apache.rocketmq.v2.ClientType.CLIENT_TYPE_UNSPECIFIED; } } diff --git a/java/client/src/main/java/org/apache/rocketmq/client/java/impl/consumer/BatchMessageViews.java b/java/client/src/main/java/org/apache/rocketmq/client/java/impl/consumer/BatchMessageViews.java new file mode 100644 index 000000000..507b952e9 --- /dev/null +++ b/java/client/src/main/java/org/apache/rocketmq/client/java/impl/consumer/BatchMessageViews.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.client.java.impl.consumer; + +import java.util.List; +import org.apache.rocketmq.client.java.message.MessageViewImpl; + +public class BatchMessageViews { + private final List messageViewList; + private final PullProcessQueueImpl processQueue; + + public BatchMessageViews(List messageViewList, PullProcessQueueImpl processQueue) { + this.messageViewList = messageViewList; + this.processQueue = processQueue; + } + + public List getMessageViewList() { + return messageViewList; + } + + public long getOffset() { + return messageViewList.get(messageViewList.size() - 1).getOffset(); + } + + public PullProcessQueueImpl getProcessQueue() { + return processQueue; + } +} diff --git a/java/client/src/main/java/org/apache/rocketmq/client/java/impl/consumer/ConsumeService.java b/java/client/src/main/java/org/apache/rocketmq/client/java/impl/consumer/ConsumeService.java index 40adb9fb4..d1376f1a5 100644 --- a/java/client/src/main/java/org/apache/rocketmq/client/java/impl/consumer/ConsumeService.java +++ b/java/client/src/main/java/org/apache/rocketmq/client/java/impl/consumer/ConsumeService.java @@ -55,7 +55,7 @@ public ConsumeService(ClientId clientId, MessageListener messageListener, Thread this.scheduler = scheduler; } - public abstract void consume(ProcessQueue pq, List messageViews); + public abstract void consume(PushProcessQueue pq, List messageViews); public ListenableFuture consume(MessageViewImpl messageView) { return consume(messageView, Duration.ZERO); diff --git a/java/client/src/main/java/org/apache/rocketmq/client/java/impl/consumer/ConsumeTask.java b/java/client/src/main/java/org/apache/rocketmq/client/java/impl/consumer/ConsumeTask.java index 90e34c5a1..0372d5951 100644 --- a/java/client/src/main/java/org/apache/rocketmq/client/java/impl/consumer/ConsumeTask.java +++ b/java/client/src/main/java/org/apache/rocketmq/client/java/impl/consumer/ConsumeTask.java @@ -62,6 +62,9 @@ public ConsumeResult call() { messageInterceptor.doBefore(context, generalMessages); try { consumeResult = messageListener.consume(messageView); + if (null == consumeResult) { + consumeResult = ConsumeResult.FAILURE; + } } catch (Throwable t) { log.error("Message listener raised an exception while consuming messages, clientId={}", clientId, t); // If exception was thrown during the period of message consumption, mark it as failure. diff --git a/java/client/src/main/java/org/apache/rocketmq/client/java/impl/consumer/ConsumerImpl.java b/java/client/src/main/java/org/apache/rocketmq/client/java/impl/consumer/ConsumerImpl.java index a807fd289..a381e5888 100644 --- a/java/client/src/main/java/org/apache/rocketmq/client/java/impl/consumer/ConsumerImpl.java +++ b/java/client/src/main/java/org/apache/rocketmq/client/java/impl/consumer/ConsumerImpl.java @@ -24,12 +24,21 @@ import apache.rocketmq.v2.ChangeInvisibleDurationResponse; import apache.rocketmq.v2.Code; import apache.rocketmq.v2.FilterType; +import apache.rocketmq.v2.GetOffsetRequest; +import apache.rocketmq.v2.GetOffsetResponse; import apache.rocketmq.v2.Message; import apache.rocketmq.v2.NotifyClientTerminationRequest; +import apache.rocketmq.v2.PullMessageRequest; +import apache.rocketmq.v2.PullMessageResponse; +import apache.rocketmq.v2.QueryOffsetPolicy; +import apache.rocketmq.v2.QueryOffsetRequest; +import apache.rocketmq.v2.QueryOffsetResponse; import apache.rocketmq.v2.ReceiveMessageRequest; import apache.rocketmq.v2.ReceiveMessageResponse; import apache.rocketmq.v2.Resource; import apache.rocketmq.v2.Status; +import apache.rocketmq.v2.UpdateOffsetRequest; +import apache.rocketmq.v2.UpdateOffsetResponse; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; @@ -122,6 +131,58 @@ protected ListenableFuture receiveMessage(ReceiveMessageRe } } + @SuppressWarnings("SameParameterValue") + protected ListenableFuture pullMessage(PullMessageRequest request, MessageQueueImpl mq, + Duration awaitDuration) { + List messages = new ArrayList<>(); + try { + final Endpoints endpoints = mq.getBroker().getEndpoints(); + final Duration tolerance = clientConfiguration.getRequestTimeout(); + final Duration timeout = awaitDuration.plus(tolerance); + final ClientManager clientManager = this.getClientManager(); + final RpcFuture> future = + clientManager.pullMessage(endpoints, request, timeout); + return Futures.transformAsync(future, responses -> { + Status status = Status.newBuilder().setCode(Code.INTERNAL_SERVER_ERROR) + .setMessage("status was not set by server") + .build(); + List messageList = new ArrayList<>(); + long nextOffset = 0; + for (PullMessageResponse response : responses) { + switch (response.getContentCase()) { + case STATUS: + status = response.getStatus(); + break; + case MESSAGE: + messageList.add(response.getMessage()); + break; + case NEXT_OFFSET: + nextOffset = response.getNextOffset(); + break; + default: + log.warn("[Bug] Not recognized content for pull message response, mq={}, clientId={}," + + " response={}", mq, clientId, response); + } + } + for (Message message : messageList) { + final MessageViewImpl view = MessageViewImpl.fromProtobuf(message, mq); + messages.add(view); + } + StatusChecker.check(status, future); + if (Code.ILLEGAL_OFFSET.equals(status.getCode())) { + log.warn("Offset is illegal, requested offset={}, next offset={}, mq={}, clientId={}", + request.getOffset(), nextOffset, mq, clientId); + } + final PullMessageResult pullMessageResult = new PullMessageResult(endpoints, messages, nextOffset); + return Futures.immediateFuture(pullMessageResult); + }, MoreExecutors.directExecutor()); + } catch (Throwable t) { + // Should never reach here. + log.error("[Bug] Exception raised during message pulling, mq={}, clientId={}", mq, clientId, t); + return Futures.immediateFailedFuture(t); + } + } + private AckMessageRequest wrapAckMessageRequest(MessageViewImpl messageView) { final Resource topicResource = Resource.newBuilder().setName(messageView.getTopic()).build(); final AckMessageEntry entry = AckMessageEntry.newBuilder() @@ -210,8 +271,7 @@ public void onFailure(Throwable t) { MessageHookPointsStatus.ERROR); doAfter(context0, generalMessages); log.error("Exception raised while changing message invisible duration, messageId={}, endpoints={}, " - + "clientId={}", - messageId, endpoints, clientId, t); + + "clientId={}", messageId, endpoints, clientId, t); } }, MoreExecutors.directExecutor()); @@ -242,6 +302,74 @@ private apache.rocketmq.v2.FilterExpression wrapFilterExpression(FilterExpressio return expressionBuilder.build(); } + protected RpcFuture queryOffset(MessageQueueImpl mq, + OffsetPolicy offsetPolicy) { + return queryOffset(mq, offsetPolicy, null); + } + + protected RpcFuture queryOffset(MessageQueueImpl mq, + Long timestampMillis) { + return queryOffset(mq, null, timestampMillis); + } + + protected RpcFuture getOffset(MessageQueueImpl mq) { + final GetOffsetRequest request = wrapGetOffsetRequest(mq); + final Duration requestTimeout = clientConfiguration.getRequestTimeout(); + return this.getClientManager().getOffset(mq.getBroker().getEndpoints(), request, requestTimeout); + } + + protected RpcFuture updateOffset(MessageQueueImpl mq, long offset) { + final UpdateOffsetRequest request = wrapUpdateOffsetRequest(mq, offset); + final Duration requestTimeout = clientConfiguration.getRequestTimeout(); + return this.getClientManager().updateOffset(mq.getBroker().getEndpoints(), request, requestTimeout); + } + + private RpcFuture queryOffset(MessageQueueImpl mq, + OffsetPolicy offsetPolicy, Long timestampMillis) { + QueryOffsetRequest request = null == offsetPolicy ? wrapQueryOffsetRequest(mq, timestampMillis) : + wrapQueryOffsetRequest(mq, offsetPolicy); + final Duration requestTimeout = clientConfiguration.getRequestTimeout(); + return this.getClientManager().queryOffset(mq.getBroker().getEndpoints(), request, requestTimeout); + } + + QueryOffsetRequest wrapQueryOffsetRequest(MessageQueueImpl mq, Long timestampInMillis) { + return QueryOffsetRequest.newBuilder().setMessageQueue(mq.toProtobuf()) + .setQueryOffsetPolicy(QueryOffsetPolicy.TIMESTAMP).setTimestamp(Timestamps.fromMillis(timestampInMillis)) + .build(); + } + + GetOffsetRequest wrapGetOffsetRequest(MessageQueueImpl mq) { + return GetOffsetRequest.newBuilder().setMessageQueue(mq.toProtobuf()).setGroup(getProtobufGroup()).build(); + } + + UpdateOffsetRequest wrapUpdateOffsetRequest(MessageQueueImpl mq, long offset) { + return UpdateOffsetRequest.newBuilder().setMessageQueue(mq.toProtobuf()).setGroup(getProtobufGroup()) + .setOffset(offset).build(); + } + + QueryOffsetRequest wrapQueryOffsetRequest(MessageQueueImpl mq, OffsetPolicy offsetPolicy) { + final QueryOffsetRequest.Builder builder = QueryOffsetRequest.newBuilder().setMessageQueue(mq.toProtobuf()); + switch (offsetPolicy) { + case BEGINNING: + builder.setQueryOffsetPolicy(QueryOffsetPolicy.BEGINNING); + break; + case END: + builder.setQueryOffsetPolicy(QueryOffsetPolicy.END); + break; + default: + log.error("Unrecognized offset policy={}", offsetPolicy); + } + return builder.build(); + } + + PullMessageRequest wrapPullMessageRequest(long offset, int batchSize, MessageQueueImpl mq, + FilterExpression filterExpression) { + return PullMessageRequest.newBuilder().setGroup(getProtobufGroup()).setMessageQueue(mq.toProtobuf()) + .setLongPollingTimeout(Durations.fromNanos(PullProcessQueueImpl.LONG_POLLING_TIMEOUT.toNanos())) + .setBatchSize(batchSize).setOffset(offset).setFilterExpression(wrapFilterExpression(filterExpression)) + .build(); + } + ReceiveMessageRequest wrapReceiveMessageRequest(int batchSize, MessageQueueImpl mq, FilterExpression filterExpression, Duration longPollingTimeout, String attemptId) { attemptId = null == attemptId ? UUID.randomUUID().toString() : attemptId; diff --git a/java/client/src/main/java/org/apache/rocketmq/client/java/impl/consumer/FifoConsumeService.java b/java/client/src/main/java/org/apache/rocketmq/client/java/impl/consumer/FifoConsumeService.java index 07fb6238d..0e05cf881 100644 --- a/java/client/src/main/java/org/apache/rocketmq/client/java/impl/consumer/FifoConsumeService.java +++ b/java/client/src/main/java/org/apache/rocketmq/client/java/impl/consumer/FifoConsumeService.java @@ -43,11 +43,11 @@ public FifoConsumeService(ClientId clientId, MessageListener messageListener, } @Override - public void consume(ProcessQueue pq, List messageViews) { + public void consume(PushProcessQueue pq, List messageViews) { consumeIteratively(pq, messageViews.iterator()); } - public void consumeIteratively(ProcessQueue pq, Iterator iterator) { + public void consumeIteratively(PushProcessQueue pq, Iterator iterator) { if (!iterator.hasNext()) { return; } diff --git a/java/client/src/main/java/org/apache/rocketmq/client/java/impl/consumer/OffsetPolicy.java b/java/client/src/main/java/org/apache/rocketmq/client/java/impl/consumer/OffsetPolicy.java new file mode 100644 index 000000000..41d0d3257 --- /dev/null +++ b/java/client/src/main/java/org/apache/rocketmq/client/java/impl/consumer/OffsetPolicy.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.client.java.impl.consumer; + +public enum OffsetPolicy { + BEGINNING, + END, +} diff --git a/java/client/src/main/java/org/apache/rocketmq/client/java/impl/consumer/ProcessQueue.java b/java/client/src/main/java/org/apache/rocketmq/client/java/impl/consumer/ProcessQueue.java index 48b56122b..466e3e7d1 100644 --- a/java/client/src/main/java/org/apache/rocketmq/client/java/impl/consumer/ProcessQueue.java +++ b/java/client/src/main/java/org/apache/rocketmq/client/java/impl/consumer/ProcessQueue.java @@ -17,55 +17,8 @@ package org.apache.rocketmq.client.java.impl.consumer; -import com.google.common.util.concurrent.ListenableFuture; -import org.apache.rocketmq.client.apis.consumer.ConsumeResult; -import org.apache.rocketmq.client.apis.consumer.PushConsumer; -import org.apache.rocketmq.client.java.message.MessageViewImpl; import org.apache.rocketmq.client.java.route.MessageQueueImpl; -/** - * Process queue is a cache to store fetched messages from remote for {@link PushConsumer}. - * - *

{@link PushConsumer} queries assignments periodically and converts them into message queues, each message queue is - * mapped into one process queue to fetch message from remote. If the message queue is removed from the newest - * assignment, the corresponding process queue is marked as expired soon, which means its lifecycle is over. - * - *

A standard procedure to cache/erase message

- * - * - *

- * phase 1: Fetch 32 messages successfully from remote. - * 

- *  32 in ┌─────────────────────────┐
- * ───────►           32            │
- *        └─────────────────────────┘
- *             cached messages = 32
- *
- * phase 2: consuming 1 message. - * 
- *        ┌─────────────────────┐   ┌───┐
- *        │          31         ├───► 1 │ consuming
- *        └─────────────────────┘   └───┘
- *             cached messages = 32
- *
- * phase 3: {@link #eraseMessage(MessageViewImpl, ConsumeResult)} with 1 messages and its consume result. - * 
- *        ┌─────────────────────┐   ┌───┐ 1 consumed
- *        │          31         ├───► 0 ├───────────►
- *        └─────────────────────┘   └───┘
- *            cached messages = 31
- *
- * - *

Especially, there are some different processing procedures for FIFO consumption. The server ensures that the - * next batch of messages will not be obtained by the client until the previous batch of messages is confirmed to be - * consumed successfully or not. In detail, the server confirms the success of consumption by message being - * successfully acknowledged, and confirms the consumption failure by being successfully forwarding to the dead - * letter queue, thus the client should try to ensure it succeeded in acknowledgement or forwarding to the dead - * letter queue as possible. - * - *

Considering the different workflow of FIFO consumption, {@link #eraseFifoMessage(MessageViewImpl, ConsumeResult)} - * and {@link #discardFifoMessage(MessageViewImpl)} is provided. - */ public interface ProcessQueue { /** * Get the mapped message queue. @@ -81,47 +34,12 @@ public interface ProcessQueue { void drop(); /** - * {@link ProcessQueue} would be regarded as expired if no fetch message for a long time. + * {@link PushProcessQueue} would be regarded as expired if no fetch message for a long time. * * @return if it is expired. */ boolean expired(); - /** - * Start to fetch messages from remote immediately. - */ - void fetchMessageImmediately(); - - /** - * Erase messages(Non-FIFO-consume-mode) which have been consumed properly. - * - * @param messageView the message to erase. - * @param consumeResult consume result. - */ - void eraseMessage(MessageViewImpl messageView, ConsumeResult consumeResult); - - /** - * Erase message(FIFO-consume-mode) which have been consumed properly. - * - * @param messageView the message to erase. - * @param consumeResult consume status. - */ - ListenableFuture eraseFifoMessage(MessageViewImpl messageView, ConsumeResult consumeResult); - - /** - * Discard the message(Non-FIFO-consume-mode) which could not be consumed properly. - * - * @param messageView the message to discard. - */ - void discardMessage(MessageViewImpl messageView); - - /** - * Discard the message(FIFO-consume-mode) which could not consumed properly. - * - * @param messageView the FIFO message to discard. - */ - void discardFifoMessage(MessageViewImpl messageView); - /** * Get the count of cached messages. * diff --git a/java/client/src/main/java/org/apache/rocketmq/client/java/impl/consumer/PullConsumerImpl.java b/java/client/src/main/java/org/apache/rocketmq/client/java/impl/consumer/PullConsumerImpl.java new file mode 100644 index 000000000..5dbecddbe --- /dev/null +++ b/java/client/src/main/java/org/apache/rocketmq/client/java/impl/consumer/PullConsumerImpl.java @@ -0,0 +1,571 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.client.java.impl.consumer; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; + +import apache.rocketmq.v2.ClientType; +import apache.rocketmq.v2.Code; +import apache.rocketmq.v2.GetOffsetRequest; +import apache.rocketmq.v2.GetOffsetResponse; +import apache.rocketmq.v2.HeartbeatRequest; +import apache.rocketmq.v2.QueryOffsetRequest; +import apache.rocketmq.v2.QueryOffsetResponse; +import apache.rocketmq.v2.Status; +import apache.rocketmq.v2.UpdateOffsetRequest; +import apache.rocketmq.v2.UpdateOffsetResponse; +import com.google.common.base.Predicate; +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.stream.Collectors; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.rocketmq.client.apis.ClientConfiguration; +import org.apache.rocketmq.client.apis.ClientException; +import org.apache.rocketmq.client.apis.consumer.FilterExpression; +import org.apache.rocketmq.client.apis.consumer.TopicMessageQueueChangeListener; +import org.apache.rocketmq.client.apis.message.MessageQueue; +import org.apache.rocketmq.client.apis.message.MessageView; +import org.apache.rocketmq.client.java.exception.StatusChecker; +import org.apache.rocketmq.client.java.impl.Settings; +import org.apache.rocketmq.client.java.message.MessageViewImpl; +import org.apache.rocketmq.client.java.message.protocol.Resource; +import org.apache.rocketmq.client.java.misc.CacheBlockingListQueue; +import org.apache.rocketmq.client.java.misc.ExcludeFromJacocoGeneratedReport; +import org.apache.rocketmq.client.java.misc.Utilities; +import org.apache.rocketmq.client.java.route.MessageQueueImpl; +import org.apache.rocketmq.client.java.route.TopicRouteData; +import org.apache.rocketmq.client.java.rpc.RpcFuture; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@SuppressWarnings({"UnstableApiUsage", "NullableProblems", "UnusedReturnValue"}) +public class PullConsumerImpl extends ConsumerImpl { + private static final Logger log = LoggerFactory.getLogger(PullConsumerImpl.class); + private static final Duration AUTO_COMMIT_DELAY = Duration.ofSeconds(1); + + final PullMessageQueuesScanner scanner; + + private final String consumerGroup; + + private final boolean autoCommitEnabled; + + private final Duration autoCommitInterval; + + private final PullSubscriptionSettings pullSubscriptionSettings; + private final int maxCacheMessageCountTotalQueue; + + private final int maxCacheMessageCountEachQueue; + + private final int maxCacheMessageSizeInBytesTotalQueue; + private final int maxCacheMessageSizeInBytesEachQueue; + private volatile ConcurrentMap subscriptions; + + private final ConcurrentMap processQueueTable; + private final ConcurrentMap processQueueLocks; + private final CacheBlockingListQueue blockingListQueue; + + private final AtomicLong pullTimes; + private final AtomicLong pulledMessagesQuantity; + + private final ConcurrentMap> topicMessageQueuesCache; + private final ConcurrentMap topicMessageQueueChangeListenerMap; + + public PullConsumerImpl(ClientConfiguration clientConfiguration, String consumerGroup, + boolean autoCommitEnabled, Duration autoCommitInterval, int maxCacheMessageCountTotalQueue, + int maxCacheMessageCountEachQueue, int maxCacheMessageSizeInBytesTotalQueue, + int maxCacheMessageSizeInBytesEachQueue) { + super(clientConfiguration, consumerGroup, new HashSet<>()); + this.consumerGroup = consumerGroup; + this.autoCommitEnabled = autoCommitEnabled; + Resource groupResource = new Resource(consumerGroup); + this.pullSubscriptionSettings = new PullSubscriptionSettings(clientId, endpoints, groupResource, + clientConfiguration.getRequestTimeout()); + this.autoCommitInterval = autoCommitInterval; + this.maxCacheMessageCountTotalQueue = maxCacheMessageCountTotalQueue; + this.maxCacheMessageCountEachQueue = maxCacheMessageCountEachQueue; + this.maxCacheMessageSizeInBytesTotalQueue = maxCacheMessageSizeInBytesTotalQueue; + this.maxCacheMessageSizeInBytesEachQueue = maxCacheMessageSizeInBytesEachQueue; + this.subscriptions = new ConcurrentHashMap<>(); + this.processQueueTable = new ConcurrentHashMap<>(); + this.processQueueLocks = new ConcurrentHashMap<>(); + this.blockingListQueue = new CacheBlockingListQueue<>(); + this.scanner = new PullMessageQueuesScanner(this); + this.pullTimes = new AtomicLong(0); + this.pulledMessagesQuantity = new AtomicLong(0); + this.topicMessageQueuesCache = new ConcurrentHashMap<>(); + this.topicMessageQueueChangeListenerMap = new ConcurrentHashMap<>(); + } + + private ReadWriteLock getProcessQueueReadWriteLock(MessageQueueImpl mq) { + return processQueueLocks.computeIfAbsent(mq, messageQueue -> new ReentrantReadWriteLock()); + } + + void cacheMessages(PullProcessQueue processQueue, List messageViews) { + log.debug("Cached {} messages, mq={}, pq={}, clientId={}", messageViews.size(), processQueue.getMessageQueue(), + processQueue, clientId); + blockingListQueue.cache(processQueue, messageViews); + } + + public List peekCachedMessages(PullProcessQueue processQueue) { + return blockingListQueue.peek(processQueue); + } + + Map getSubscriptions() { + return subscriptions; + } + + ConcurrentMap getProcessQueueTable() { + return processQueueTable; + } + + AtomicLong getPullTimes() { + return pullTimes; + } + + public AtomicLong getPulledMessagesQuantity() { + return pulledMessagesQuantity; + } + + public String getConsumerGroup() { + return consumerGroup; + } + + public void registerMessageQueueChangeListenerByTopic(String topic, TopicMessageQueueChangeListener listener) + throws ClientException { + checkNotNull(topic, "topic should not be null"); + checkNotNull(listener, "listener should not be null"); + if (!this.isRunning()) { + log.error("Unable to register message queue listener because pull consumer is not running," + + " state={}, clientId={}", this.state(), clientId); + throw new IllegalStateException("Pull consumer is not running now"); + } + if (topicMessageQueueChangeListenerMap.containsKey(topic)) { + log.info("The listener of topic={} has been registered, the new listener will replace the existing one", + topic); + } + topicMessageQueueChangeListenerMap.put(topic, listener); + fetchMessageQueues(topic); + } + + int getMaxCacheMessageCountEachQueue() { + int size = processQueueTable.size(); + if (size < 1) { + size = 1; + } + return Math.min(Math.max(1, maxCacheMessageCountTotalQueue / size), maxCacheMessageCountEachQueue); + } + + int getMaxCacheMessageSizeInBytesEachQueue() { + int size = processQueueTable.size(); + if (size < 1) { + size = 1; + } + return Math.min(Math.max(1, maxCacheMessageSizeInBytesTotalQueue / size), maxCacheMessageSizeInBytesEachQueue); + } + + public Collection fetchMessageQueues(String topic) throws ClientException { + checkNotNull(topic, "topic should not be null"); + if (!this.isRunning()) { + log.error("Unable to fetch message queue because pull consumer is not running, state={}, clientId={}", + this.state(), clientId); + throw new IllegalStateException("Pull consumer is not running now"); + } + final ListenableFuture future = fetchTopicRoute(topic); + final TopicRouteData topicRouteData = handleClientFuture(future); + final List queues = transformTopicRouteData(topicRouteData); + return new ArrayList<>(queues); + } + + void tryPullMessageByMessageQueueImmediately(MessageQueueImpl mq, FilterExpression filterExpression) { + final Optional pq = createProcessQueue(mq, filterExpression); + pq.ifPresent(PullProcessQueue::pullMessageImmediately); + } + + void tryPullMessageByMessageQueueImmediately(MessageQueueImpl mq, FilterExpression filterExpression, long offset) { + final Optional pq = createProcessQueue(mq, filterExpression, offset); + pq.ifPresent(PullProcessQueue::pullMessageImmediately); + } + + Optional createProcessQueue(MessageQueueImpl mq, FilterExpression filterExpression) { + final Lock lock = getProcessQueueReadWriteLock(mq).readLock(); + lock.lock(); + try { + final PullProcessQueueImpl processQueue = new PullProcessQueueImpl(this, mq, filterExpression); + final PullProcessQueue previous = processQueueTable.putIfAbsent(mq, processQueue); + if (null != previous) { + return Optional.empty(); + } + return Optional.of(processQueue); + } finally { + lock.unlock(); + } + } + + Optional createProcessQueue(MessageQueueImpl mq, FilterExpression filterExpression, long offset) { + final Lock lock = getProcessQueueReadWriteLock(mq).readLock(); + lock.lock(); + try { + final PullProcessQueueImpl processQueue = new PullProcessQueueImpl(this, mq, filterExpression, + offset); + final PullProcessQueue previous = processQueueTable.putIfAbsent(mq, processQueue); + if (null != previous) { + return Optional.empty(); + } + return Optional.of(processQueue); + } finally { + lock.unlock(); + } + } + + void dropProcessQueue(MessageQueueImpl mq) { + final Lock lock = getProcessQueueReadWriteLock(mq).readLock(); + lock.lock(); + try { + final PullProcessQueue pq = processQueueTable.remove(mq); + if (null != pq) { + pq.drop(); + blockingListQueue.drop(pq); + } + } finally { + lock.unlock(); + } + } + + public void assign(Collection mqs) { + Map subscriptions = new HashMap<>(); + for (MessageQueue mq : mqs) { + subscriptions.put(mq, FilterExpression.SUB_ALL); + } + assign(subscriptions); + } + + void assign(Map subscriptions) { + assign0(subscriptions); + // Notify the scanner to scan the assigned queue immediately. + scanner.signal(); + } + + /** + * Replace the current subscriptions with the latest subscriptions. + */ + void assign0(Map subscriptions) { + checkNotNull(subscriptions, "subscriptions to be assigned should not be null"); + checkArgument(!subscriptions.isEmpty(), "subscription should not be empty"); + if (!this.isRunning()) { + log.error("Unable to assign subscription because pull consumer is not running, state={}, clientId={}", + this.state(), clientId); + throw new IllegalStateException("Pull consumer is not running now"); + } + ConcurrentMap temp = new ConcurrentHashMap<>(); + for (Map.Entry entry : subscriptions.entrySet()) { + final MessageQueue mq = entry.getKey(); + final FilterExpression filterExpression = entry.getValue(); + temp.put((MessageQueueImpl) mq, filterExpression); + } + this.subscriptions = temp; + } + + public List poll(Duration timeout) throws InterruptedException { + checkNotNull(timeout, "timeout should not be null"); + if (!this.isRunning()) { + log.error("Unable to pull message because pull consumer is not running, state={}, clientId={}", + this.state(), clientId); + throw new IllegalStateException("Pull consumer is not running now"); + } + final Pair> pair = blockingListQueue.poll(timeout); + if (null == pair) { + return new ArrayList<>(); + } + // Update the latest consumed offset. + final PullProcessQueue pq = pair.getKey(); + final List value = pair.getValue(); + final long offset = value.get(value.size() - 1).getOffset(); + pq.updateConsumedOffset(offset); + return new ArrayList<>(value); + } + + public void seek(MessageQueue messageQueue, long offset) { + checkNotNull(messageQueue, "messageQueue should not be null"); + if (!this.isRunning()) { + log.error("Unable to seek because pull consumer is not running, state={}, clientId={}", + this.state(), clientId); + throw new IllegalStateException("Pull consumer is not running now"); + } + MessageQueueImpl mq = (MessageQueueImpl) messageQueue; + if (!subscriptions.containsKey(mq)) { + throw new IllegalArgumentException("The message queue is not contained in the assigned list"); + } + seek0(mq, offset); + } + + private void seek0(MessageQueueImpl mq, long offset) { + final Lock lock = getProcessQueueReadWriteLock(mq).writeLock(); + lock.lock(); + try { + final FilterExpression filterExpression = subscriptions.get(mq); + if (null == filterExpression) { + throw new IllegalArgumentException("The message queue is not contained in the assigned list"); + } + log.info("Seek to the offset={}, mq={}, filterExpression={}, clientId={}", offset, mq, filterExpression, + clientId); + dropProcessQueue(mq); + tryPullMessageByMessageQueueImmediately(mq, filterExpression, offset); + } finally { + lock.unlock(); + } + } + + public void pause(Collection messageQueues) { + checkNotNull(messageQueues, "messageQueues should not be null"); + checkArgument(!messageQueues.isEmpty(), "message queues should not be empty"); + if (!this.isRunning()) { + log.error("Unable to pause because pull consumer is not running, state={}, clientId={}", this.state(), + clientId); + throw new IllegalStateException("Pull consumer is not running now"); + } + for (MessageQueue mq : messageQueues) { + final PullProcessQueue pq = processQueueTable.get((MessageQueueImpl) mq); + if (null != pq) { + pq.pause(); + log.info("Message queue is paused to pull, mq={}, clientId={}", mq, clientId); + } + } + } + + public void resume(Collection messageQueues) { + checkNotNull(messageQueues, "messageQueues should not be null"); + checkArgument(!messageQueues.isEmpty(), "message queues should not be null"); + if (!this.isRunning()) { + log.error("Unable to resume because pull consumer is not running, state={}, clientId={}", this.state(), + clientId); + throw new IllegalStateException("Pull consumer is not running now"); + } + for (MessageQueue mq : messageQueues) { + final PullProcessQueue pq = processQueueTable.get((MessageQueueImpl) mq); + pq.resume(); + } + } + + public Optional offsetForTimestamp(MessageQueue messageQueue, long timestamp) throws ClientException { + checkNotNull(messageQueue, "messageQueue should not be null"); + if (!this.isRunning()) { + log.error("Unable to query offset because pull consumer is not running, state={}, clientId={}", + this.state(), clientId); + throw new IllegalStateException("Pull consumer is not running now"); + } + final RpcFuture future = queryOffset((MessageQueueImpl) messageQueue, + timestamp); + final QueryOffsetResponse response = handleClientFuture(future); + final Status status = response.getStatus(); + StatusChecker.check(status, future); + return Optional.of(response.getOffset()); + } + + public Optional committed(MessageQueue messageQueue) throws ClientException { + checkNotNull(messageQueue, "messageQueue should not be null"); + if (!this.isRunning()) { + log.error("Unable to query offset because pull consumer is not running, state={}, clientId={}", + this.state(), clientId); + throw new IllegalStateException("Pull consumer is not running now"); + } + final RpcFuture future = getOffset((MessageQueueImpl) messageQueue); + final GetOffsetResponse response = handleClientFuture(future); + final Status status = response.getStatus(); + StatusChecker.check(status, future); + return Optional.of(response.getOffset()); + } + + public List> commit0() { + if (!this.isRunning()) { + log.error("Unable to commit offset because pull consumer is not running, state={}, clientId={}", + this.state(), clientId); + throw new IllegalStateException("Pull consumer is not running now"); + } + List> futures = new ArrayList<>(); + for (Map.Entry entry : processQueueTable.entrySet()) { + final MessageQueueImpl mq = entry.getKey(); + final PullProcessQueue pq = entry.getValue(); + final long offset = pq.getConsumedOffset(); + if (offset < 0) { + continue; + } + final RpcFuture future = updateOffset(mq, offset); + Futures.addCallback(future, new FutureCallback() { + @Override + public void onSuccess(UpdateOffsetResponse response) { + final Status status = response.getStatus(); + final Code code = status.getCode(); + if (Code.OK.equals(code)) { + log.info("Update offset successfully, mq={}, offset={}, clientId={}, consumerGroup={}", mq, + offset, clientId, consumerGroup); + return; + } + log.info("Failed to update offset, mq={}, offset={}, clientId={}, consumerGroup={}, code={}, " + + "status message=[{}]", mq, offset, clientId, consumerGroup, code, status.getMessage()); + } + + @Override + public void onFailure(Throwable t) { + log.error("Failed to update offset, mq={}, offset={}, clientId={}, consumerGroup={}", mq, offset, + clientId, consumerGroup, t); + } + }, MoreExecutors.directExecutor()); + futures.add(future); + } + return futures; + } + + public void commit() throws ClientException { + List> futures = commit0(); + final ListenableFuture> future0 = Futures.allAsList(futures); + final List responses = handleClientFuture(future0); + for (int i = 0; i < futures.size(); i++) { + final RpcFuture future = futures.get(i); + final UpdateOffsetResponse response = responses.get(i); + StatusChecker.check(response.getStatus(), future); + } + } + + public void seekToBegin(MessageQueue messageQueue) throws ClientException { + checkNotNull(messageQueue, "messageQueue should not be null"); + if (!this.isRunning()) { + log.error("Unable to seek because pull consumer is not running, state={}, clientId={}", + this.state(), clientId); + throw new IllegalStateException("Pull consumer is not running now"); + } + MessageQueueImpl mq = (MessageQueueImpl) messageQueue; + if (!subscriptions.containsKey(mq)) { + throw new IllegalArgumentException("The message queue is not contained in the assigned list"); + } + log.info("Seek to the beginning offset, mq={}, clientId={}", messageQueue, clientId); + final RpcFuture future = queryOffset(mq, OffsetPolicy.BEGINNING); + final QueryOffsetResponse response = handleClientFuture(future); + StatusChecker.check(response.getStatus(), future); + seek0(mq, response.getOffset()); + } + + public void seekToEnd(MessageQueue messageQueue) throws ClientException { + checkNotNull(messageQueue, "messageQueue should not be null"); + if (!this.isRunning()) { + log.error("Unable to seek because pull consumer is not running, state={}, clientId={}", + this.state(), clientId); + throw new IllegalStateException("Pull consumer is not running now"); + } + MessageQueueImpl mq = (MessageQueueImpl) messageQueue; + if (!subscriptions.containsKey(mq)) { + throw new IllegalArgumentException("The message queue is not contained in the assigned list"); + } + log.info("Seek to the end offset, mq={}, clientId={}", messageQueue, clientId); + final RpcFuture future = queryOffset(mq, OffsetPolicy.END); + final QueryOffsetResponse response = handleClientFuture(future); + StatusChecker.check(response.getStatus(), future); + seek0(mq, response.getOffset()); + } + + @Override + protected void startUp() throws Exception { + log.info("Begin to start the rocketmq pull consumer, clientId={}", clientId); + super.startUp(); + scanner.startUp(); + if (autoCommitEnabled) { + this.getScheduler().scheduleWithFixedDelay(() -> { + try { + commit(); + } catch (Throwable t) { + log.error("Failed to commit offset for pull consumer, clientId={}", clientId, t); + } + }, AUTO_COMMIT_DELAY.toNanos(), autoCommitInterval.toNanos(), TimeUnit.NANOSECONDS); + } + log.info("The rocketmq pull consumer starts successfully, clientId={}", clientId); + } + + @Override + protected void shutDown() throws InterruptedException { + log.info("Begin to shutdown the rocketmq pull consumer, clientId={}", clientId); + scanner.shutDown(); + super.shutDown(); + log.info("Shutdown the rocketmq pull consumer successfully, clientId={}", clientId); + } + + public void close() { + this.stopAsync().awaitTerminated(); + } + + @Override + public Settings getSettings() { + return pullSubscriptionSettings; + } + + @Override + public HeartbeatRequest wrapHeartbeatRequest() { + return HeartbeatRequest.newBuilder().setGroup(getProtobufGroup()) + .setClientType(ClientType.PULL_CONSUMER).build(); + } + + private List transformTopicRouteData(TopicRouteData topicRouteData) { + return topicRouteData.getMessageQueues().stream() + .filter((Predicate) mq -> mq.getPermission().isReadable() && + Utilities.MASTER_BROKER_ID == mq.getBroker().getId()) + .collect(Collectors.toList()); + } + + public void onTopicRouteDataUpdate0(String topic, TopicRouteData topicRouteData) { + final List newMqs = transformTopicRouteData(topicRouteData); + Set newMqSet = new HashSet<>(newMqs); + synchronized (topicMessageQueuesCache) { + final List oldMqs = topicMessageQueuesCache.get(topic); + Set oldMqSet = null == oldMqs ? null : new HashSet<>(oldMqs); + if (!newMqSet.equals(oldMqSet)) { + final TopicMessageQueueChangeListener listener = topicMessageQueueChangeListenerMap.get(topic); + if (null != listener) { + listener.onChanged(topic, newMqSet); + } + } + topicMessageQueuesCache.put(topic, newMqs); + } + } + + @ExcludeFromJacocoGeneratedReport + @Override + public void doStats() { + final long pullTimes = this.pullTimes.getAndSet(0); + final long pulledMessagesQuantity = this.pulledMessagesQuantity.getAndSet(0); + log.info("clientId={}, consumerGroup={}, pullTimes={}, pulledMessageQuantity={}", clientId, consumerGroup, + pullTimes, pulledMessagesQuantity); + processQueueTable.values().forEach(ProcessQueue::doStats); + } +} diff --git a/java/client/src/main/java/org/apache/rocketmq/client/java/impl/consumer/PullMessageQueuesScanner.java b/java/client/src/main/java/org/apache/rocketmq/client/java/impl/consumer/PullMessageQueuesScanner.java new file mode 100644 index 000000000..cbd46a249 --- /dev/null +++ b/java/client/src/main/java/org/apache/rocketmq/client/java/impl/consumer/PullMessageQueuesScanner.java @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.client.java.impl.consumer; + +import com.google.common.util.concurrent.AbstractIdleService; +import java.time.Duration; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import org.apache.rocketmq.client.apis.consumer.FilterExpression; +import org.apache.rocketmq.client.java.misc.ClientId; +import org.apache.rocketmq.client.java.route.MessageQueueImpl; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * This class represents a scanner to detect any changes made to the assigned message queues by the pull consumer. + */ +public class PullMessageQueuesScanner extends AbstractIdleService { + private static final Logger log = LoggerFactory.getLogger(PullMessageQueuesScanner.class); + private static final Duration SCAN_INITIAL_DELAY = Duration.ofSeconds(0); + private static final Duration SCAN_PERIOD = Duration.ofSeconds(3); + private final PullConsumerImpl consumer; + private final AtomicBoolean scanTaskInQueueFlag; + private final AtomicBoolean scannerStarted; + private final ClientId clientId; + + public PullMessageQueuesScanner(PullConsumerImpl consumer) { + this.consumer = consumer; + this.scanTaskInQueueFlag = new AtomicBoolean(false); + this.scannerStarted = new AtomicBoolean(false); + this.clientId = consumer.getClientId(); + } + + /** + * Starts the scanning process and logs a successful message if the process starts correctly. + */ + @Override + protected void startUp() { + log.info("Begin to start the pull message queues scanner, clientId={}", clientId); + log.info("The pull message queues scanner starts successfully, clientId={}", clientId); + } + + void signal() { + if (scannerStarted.compareAndSet(false, true)) { + log.info("Begin to execute pull message queues scan task periodically, clientId={}", clientId); + consumer.getScheduler().scheduleWithFixedDelay(this::signal0, SCAN_INITIAL_DELAY.toNanos(), + SCAN_PERIOD.toNanos(), TimeUnit.NANOSECONDS); + } + signal0(); + } + + private void signal0() { + if (scanTaskInQueueFlag.compareAndSet(false, true)) { + consumer.getScheduler().submit(this::scan); + } + } + + private void scan() { + scanTaskInQueueFlag.compareAndSet(true, false); + scan0(); + } + + /** + * Shuts down the scanning process and logs a successful message once the shutdown is complete. + */ + @Override + protected void shutDown() { + log.info("Begin to shutdown the pull message queues scanner, clientId={}", clientId); + log.info("Shutdown the pull message queues scanner successfully, clientId={}", clientId); + } + + /** + * Scans the assigned message queues to the pull consumer to detect any changes made and act accordingly. + */ + private void scan0() { + final ConcurrentMap processQueueTable = consumer.getProcessQueueTable(); + final Map subscriptions = consumer.getSubscriptions(); + final Set latest = subscriptions.keySet(); + final Set existed = processQueueTable.keySet(); + + if (latest.isEmpty() && existed.isEmpty()) { + log.debug("Message queues are empty, clientId={}", clientId); + return; + } + + if (!latest.equals(existed)) { + log.info("Message queues have been changed, {} => {}, clientId={}", existed, subscriptions, clientId); + } + + Set activeMqs = new HashSet<>(); + for (Map.Entry entry : processQueueTable.entrySet()) { + final MessageQueueImpl mq = entry.getKey(); + final PullProcessQueue pq = entry.getValue(); + if (!subscriptions.containsKey(mq)) { + consumer.dropProcessQueue(mq); + continue; + } + if (null != pq && pq.expired()) { + consumer.dropProcessQueue(mq); + continue; + } + activeMqs.add(mq); + } + for (Map.Entry entry : subscriptions.entrySet()) { + final MessageQueueImpl mq = entry.getKey(); + if (activeMqs.contains(mq)) { + continue; + } + final FilterExpression filterExpression = entry.getValue(); + consumer.tryPullMessageByMessageQueueImmediately(mq, filterExpression); + } + } +} diff --git a/java/client/src/main/java/org/apache/rocketmq/client/java/impl/consumer/PullMessageResult.java b/java/client/src/main/java/org/apache/rocketmq/client/java/impl/consumer/PullMessageResult.java new file mode 100644 index 000000000..cbb05991c --- /dev/null +++ b/java/client/src/main/java/org/apache/rocketmq/client/java/impl/consumer/PullMessageResult.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.client.java.impl.consumer; + +import java.util.ArrayList; +import java.util.List; +import org.apache.rocketmq.client.apis.message.MessageView; +import org.apache.rocketmq.client.java.message.MessageViewImpl; +import org.apache.rocketmq.client.java.route.Endpoints; + +public class PullMessageResult { + private final Endpoints endpoints; + private final List messages; + private final long nextOffset; + + public PullMessageResult(Endpoints endpoints, List messages, long nextOffset) { + this.endpoints = endpoints; + this.messages = messages; + this.nextOffset = nextOffset; + } + + public List getMessageViews() { + return new ArrayList<>(messages); + } + + public Endpoints getEndpoints() { + return endpoints; + } + + public List getMessageViewImpls() { + return messages; + } + + public long getNextOffset() { + return nextOffset; + } +} diff --git a/java/client/src/main/java/org/apache/rocketmq/client/java/impl/consumer/PullProcessQueue.java b/java/client/src/main/java/org/apache/rocketmq/client/java/impl/consumer/PullProcessQueue.java new file mode 100644 index 000000000..9e3784939 --- /dev/null +++ b/java/client/src/main/java/org/apache/rocketmq/client/java/impl/consumer/PullProcessQueue.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.client.java.impl.consumer; + +/** + * The PullProcessQueue interface is used for pull-consumer to control the message consuming process. + */ +public interface PullProcessQueue extends ProcessQueue { + + /** + * Resume message pulling, if it has been paused. + */ + void resume(); + + /** + * Pause message pulling. + */ + void pause(); + + /** + * Pull all available messages immediately. + */ + void pullMessageImmediately(); + + void updateConsumedOffset(long offset); + + /** + * Get the offset of the consumed message. + * + * @return The offset of the consumed message. + */ + long getConsumedOffset(); +} \ No newline at end of file diff --git a/java/client/src/main/java/org/apache/rocketmq/client/java/impl/consumer/PullProcessQueueImpl.java b/java/client/src/main/java/org/apache/rocketmq/client/java/impl/consumer/PullProcessQueueImpl.java new file mode 100644 index 000000000..ed4e07071 --- /dev/null +++ b/java/client/src/main/java/org/apache/rocketmq/client/java/impl/consumer/PullProcessQueueImpl.java @@ -0,0 +1,353 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.client.java.impl.consumer; + +import apache.rocketmq.v2.Code; +import apache.rocketmq.v2.GetOffsetRequest; +import apache.rocketmq.v2.GetOffsetResponse; +import apache.rocketmq.v2.PullMessageRequest; +import apache.rocketmq.v2.QueryOffsetRequest; +import apache.rocketmq.v2.QueryOffsetResponse; +import apache.rocketmq.v2.Status; +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; +import java.time.Duration; +import java.util.List; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; +import org.apache.rocketmq.client.apis.consumer.FilterExpression; +import org.apache.rocketmq.client.java.exception.StatusChecker; +import org.apache.rocketmq.client.java.exception.TooManyRequestsException; +import org.apache.rocketmq.client.java.message.MessageViewImpl; +import org.apache.rocketmq.client.java.misc.ClientId; +import org.apache.rocketmq.client.java.misc.ExcludeFromJacocoGeneratedReport; +import org.apache.rocketmq.client.java.route.Endpoints; +import org.apache.rocketmq.client.java.route.MessageQueueImpl; +import org.apache.rocketmq.client.java.rpc.RpcFuture; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@SuppressWarnings({"NullableProblems", "UnstableApiUsage"}) +public class PullProcessQueueImpl implements PullProcessQueue { + static final Duration LONG_POLLING_TIMEOUT = Duration.ofSeconds(30); + + private static final Logger log = LoggerFactory.getLogger(PullProcessQueueImpl.class); + + private static final int PULL_BATCH_SIZE = 32; + + private static final Duration PULL_FLOW_CONTROL_BACKOFF_DELAY = Duration.ofMillis(20); + + private static final Duration PULL_FAILURE_BACKOFF_DELAY = Duration.ofSeconds(1); + + private static final Duration QUERY_OFFSET_BACKOFF_DELAY = Duration.ofSeconds(3); + private static final Duration PAUSE_CHECK_PERIOD = Duration.ofSeconds(1); + + private final PullConsumerImpl consumer; + private final FilterExpression filterExpression; + private volatile boolean dropped; + private final MessageQueueImpl mq; + + private final AtomicLong pullTimes; + + private final AtomicLong pulledMessagesQuantity; + + private volatile long activityNanoTime = System.nanoTime(); + private volatile long cacheFullNanoTime = Long.MIN_VALUE; + + private final AtomicBoolean paused; + + private final long initialOffset; + private volatile long consumedOffset; + + public PullProcessQueueImpl(PullConsumerImpl consumer, MessageQueueImpl mq, FilterExpression filterExpression) { + this(consumer, mq, filterExpression, -1); + } + + public PullProcessQueueImpl(PullConsumerImpl consumer, MessageQueueImpl mq, FilterExpression filterExpression, + long initialOffset) { + this.consumer = consumer; + this.dropped = false; + this.mq = mq; + this.filterExpression = filterExpression; + this.pullTimes = new AtomicLong(); + this.paused = new AtomicBoolean(false); + this.initialOffset = initialOffset; + this.pulledMessagesQuantity = new AtomicLong(); + } + + public void pause() { + paused.compareAndSet(false, true); + } + + public void resume() { + paused.compareAndSet(true, false); + } + + @Override + public MessageQueueImpl getMessageQueue() { + return mq; + } + + @Override + public void drop() { + this.dropped = true; + log.info("Process queue was dropped, pq={}, clientId={}", this, consumer.getClientId()); + } + + @Override + public boolean expired() { + final Duration requestTimeout = consumer.getClientConfiguration().getRequestTimeout(); + final Duration maxIdleDuration = LONG_POLLING_TIMEOUT.plus(requestTimeout).multipliedBy(3); + final Duration idleDuration = Duration.ofNanos(System.nanoTime() - activityNanoTime); + if (idleDuration.compareTo(maxIdleDuration) < 0) { + return false; + } + final Duration afterCacheFullDuration = Duration.ofNanos(System.nanoTime() - cacheFullNanoTime); + if (afterCacheFullDuration.compareTo(maxIdleDuration) < 0) { + return false; + } + log.warn("Process queue is idle, idleDuration={}, maxIdleDuration={}, afterCacheFullDuration={}, mq={}, " + + "clientId={}", idleDuration, maxIdleDuration, afterCacheFullDuration, mq, consumer.getClientId()); + return true; + } + + @Override + public long getCachedMessageCount() { + return consumer.peekCachedMessages(this).size(); + } + + @Override + public long getCachedMessageBytes() { + final List messages = consumer.peekCachedMessages(this); + long bodyBytes = 0; + for (MessageViewImpl message : messages) { + bodyBytes += message.getBody().remaining(); + } + return bodyBytes; + } + + @ExcludeFromJacocoGeneratedReport + @Override + public void doStats() { + final long pullTimes = this.pullTimes.getAndSet(0); + final long pulledMessagesQuantity = this.pulledMessagesQuantity.getAndSet(0); + log.info("Process queue stats: clientId={}, mq={}, pullTimes={}, pulledMessagesQuantity={}, " + + "cachedMessageCount={}, cachedMessageBytes={}", consumer.getClientId(), mq, pullTimes, + pulledMessagesQuantity, this.getCachedMessageCount(), this.getCachedMessageBytes()); + } + + public boolean isCacheFull() { + final int cachedMessageCountThreshold = consumer.getMaxCacheMessageCountEachQueue(); + final long actualMessageQuantity = this.getCachedMessageCount(); + final ClientId clientId = consumer.getClientId(); + if (cachedMessageCountThreshold < actualMessageQuantity) { + log.warn("Process queue total cached messages quantity exceeds the threshold, threshold={}, actual={}, " + + "mq={}, clientId={}", cachedMessageCountThreshold, actualMessageQuantity, mq, clientId); + cacheFullNanoTime = System.nanoTime(); + return true; + } + final int cachedMessageSizeInBytesThreshold = consumer.getMaxCacheMessageSizeInBytesEachQueue(); + final long actualMessageBytes = this.getCachedMessageBytes(); + if (cachedMessageSizeInBytesThreshold < actualMessageBytes) { + log.warn("Process queue total cached messages memory exceeds the threshold, threshold={} bytes, " + + "actual={} bytes, mq={}, clientId={}", cachedMessageSizeInBytesThreshold, actualMessageBytes, mq, + clientId); + cacheFullNanoTime = System.nanoTime(); + return true; + } + return false; + } + + public void pullMessageImmediately(long offset) { + final ClientId clientId = consumer.getClientId(); + if (!consumer.isRunning()) { + log.info("Stop to pull message because consumer is not running, mq={}, offset={}, clientId={}", + mq, offset, clientId); + return; + } + if (dropped) { + log.info("Process queue has been dropped, no longer pull message, mq={}, pq={}, clientId={}", mq, + this, clientId); + return; + } + if (this.isCacheFull()) { + log.warn("Process queue cache is full, would receive message later, mq={}, clientId={}", mq, clientId); + return; + } + if (paused.get()) { + log.debug("Process queue pulling is paused, mq={}, clientId={}", mq, clientId); + final ScheduledExecutorService scheduler = consumer.getScheduler(); + scheduler.schedule(() -> pullMessageImmediately(offset), PAUSE_CHECK_PERIOD.toNanos(), + TimeUnit.NANOSECONDS); + return; + } + try { + final PullMessageRequest request = consumer.wrapPullMessageRequest(offset, PULL_BATCH_SIZE, mq, + filterExpression); + activityNanoTime = System.nanoTime(); + final ListenableFuture future = consumer.pullMessage(request, mq, LONG_POLLING_TIMEOUT); + final Endpoints endpoints = mq.getBroker().getEndpoints(); + Futures.addCallback(future, new FutureCallback() { + @Override + public void onSuccess(PullMessageResult result) { + try { + onPullMessageResult(result, offset); + } catch (Throwable t) { + // Should never reach here. + log.error("[Bug] Exception raised while handling pull request, mq={}, endpoints={}, " + + "clientId={}", mq, endpoints, clientId, t); + onPullMessageException(offset, t); + } + } + + @Override + public void onFailure(Throwable t) { + log.error("Exception raised during message pulling, mq={}, endpoints={}, clientId={}", mq, + endpoints, clientId, t); + onPullMessageException(offset, t); + } + }, MoreExecutors.directExecutor()); + pullTimes.getAndIncrement(); + consumer.getPullTimes().getAndIncrement(); + } catch (Throwable t) { + log.error("Exception raised during message pulling, mq={}, clientId={}", mq, clientId, t); + } + } + + private void onPullMessageException(long offset, Throwable t) { + Duration delay = t instanceof TooManyRequestsException ? PULL_FLOW_CONTROL_BACKOFF_DELAY : + PULL_FAILURE_BACKOFF_DELAY; + pullMessageLater(offset, delay); + } + + private void pullMessageLater(long offset, Duration delay) { + final ClientId clientId = consumer.getClientId(); + final ScheduledExecutorService scheduler = consumer.getScheduler(); + try { + log.info("Try to pull message later, mq={}, delay={}, clientId={}", mq, delay, clientId); + scheduler.schedule(() -> pullMessageImmediately(offset), delay.toNanos(), TimeUnit.NANOSECONDS); + } catch (Throwable t) { + if (scheduler.isShutdown()) { + return; + } + // Should never reach here. + log.error("[Bug] Failed to schedule message pulling request, mq={}, clientId={}", mq, clientId, t); + onPullMessageException(offset, t); + } + } + + void onPullMessageResult(PullMessageResult result, long offset) { + final ClientId clientId = consumer.getClientId(); + if (paused.get()) { + log.info("Discard pull result because process queue pulling is paused, mq={}, pq={}, offset={}, " + + "clientId={}", mq, this, offset, clientId); + pullMessageImmediately(offset); + return; + } + if (dropped) { + log.info("Discard pull result because process queue is dropped, mq={}, pq={}, clientId={}", mq, this, + clientId); + return; + } + final List messages = result.getMessageViewImpls(); + if (!messages.isEmpty()) { + pulledMessagesQuantity.getAndAdd(messages.size()); + consumer.getPulledMessagesQuantity().getAndAdd(messages.size()); + consumer.cacheMessages(this, messages); + } + pullMessageImmediately(result.getNextOffset()); + } + + @Override + public void pullMessageImmediately() { + final ClientId clientId = consumer.getClientId(); + if (initialOffset >= 0) { + log.info("Start to pull message immediately because offset is appointed already, mq={}, initialOffset={}," + + " clientId={}", mq, initialOffset, clientId); + pullMessageImmediately(initialOffset); + return; + } + if (!consumer.isRunning()) { + log.info("Stop to pull message because consumer is not running, mq={}, initialOffset={}, clientId={}", mq, + initialOffset, clientId); + return; + } + if (dropped) { + log.info("Process queue has been dropped, no longer receive message, mq={}, initialOffset={}, " + + "pq={}, clientId={}", mq, initialOffset, this, clientId); + return; + } + final ScheduledExecutorService scheduler = consumer.getScheduler(); + if (paused.get()) { + log.debug("Process queue pulling is paused, mq={}, initialOffset={}, clientId={}", + mq, initialOffset, clientId); + scheduler.schedule(() -> pullMessageImmediately(), PAUSE_CHECK_PERIOD.toNanos(), TimeUnit.NANOSECONDS); + return; + } + final RpcFuture future0 = consumer.getOffset(mq); + final ListenableFuture future = Futures.transformAsync(future0, response -> { + final Status status = response.getStatus(); + StatusChecker.check(status, future0); + if (Code.OFFSET_NOT_FOUND.equals(status.getCode())) { + final RpcFuture future1 = + consumer.queryOffset(mq, OffsetPolicy.END); + return Futures.transformAsync(consumer.queryOffset(mq, OffsetPolicy.END), input -> { + StatusChecker.check(input.getStatus(), future1); + final long offset = input.getOffset(); + log.info("Query offset using end policy from remote successfully, consumerGroup={}, mq={}," + + " offset={}, clientId={}", consumer.getConsumerGroup(), mq, offset, clientId); + return Futures.immediateFuture(offset); + }, MoreExecutors.directExecutor()); + } + final long offset = response.getOffset(); + log.info("Get offset by consumerGroup from remote successfully, consumerGroup={}, mq={}, offset={}, " + + "clientId={}", consumer.getConsumerGroup(), mq, offset, clientId); + return Futures.immediateFuture(offset); + }, MoreExecutors.directExecutor()); + Futures.addCallback(future, new FutureCallback() { + @Override + public void onSuccess(Long offset) { + consumedOffset = offset; + log.info("Start to pull message immediately because offset is fetched from remote successfully," + + " mq={}, offset={}, clientId={}", mq, consumedOffset, clientId); + pullMessageImmediately(consumedOffset); + } + + @Override + public void onFailure(Throwable t) { + log.info("Exception raised while fetching offset from remote, mq={}, clientId={}", mq, clientId, t); + scheduler.schedule(() -> pullMessageImmediately(), QUERY_OFFSET_BACKOFF_DELAY.toNanos(), + TimeUnit.NANOSECONDS); + } + }, MoreExecutors.directExecutor()); + } + + @Override + public void updateConsumedOffset(long offset) { + this.consumedOffset = offset; + } + + @Override + public long getConsumedOffset() { + return consumedOffset; + } +} diff --git a/java/client/src/main/java/org/apache/rocketmq/client/java/impl/consumer/PullSubscriptionSettings.java b/java/client/src/main/java/org/apache/rocketmq/client/java/impl/consumer/PullSubscriptionSettings.java new file mode 100644 index 000000000..5bb878b25 --- /dev/null +++ b/java/client/src/main/java/org/apache/rocketmq/client/java/impl/consumer/PullSubscriptionSettings.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.client.java.impl.consumer; + +import apache.rocketmq.v2.Subscription; +import com.google.common.base.MoreObjects; +import com.google.protobuf.util.Durations; +import java.time.Duration; +import org.apache.rocketmq.client.java.impl.ClientType; +import org.apache.rocketmq.client.java.impl.Settings; +import org.apache.rocketmq.client.java.impl.UserAgent; +import org.apache.rocketmq.client.java.message.protocol.Resource; +import org.apache.rocketmq.client.java.misc.ClientId; +import org.apache.rocketmq.client.java.misc.ExcludeFromJacocoGeneratedReport; +import org.apache.rocketmq.client.java.route.Endpoints; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class PullSubscriptionSettings extends Settings { + private static final Logger log = LoggerFactory.getLogger(PullSubscriptionSettings.class); + private final Resource group; + + private volatile Duration longPollingTimeout = Duration.ofSeconds(30); + + public PullSubscriptionSettings(ClientId clientId, Endpoints endpoints, Resource group, Duration requestTimeout) { + super(clientId, ClientType.PULL_CONSUMER, endpoints, requestTimeout); + this.group = group; + } + + public Duration getLongPollingTimeout() { + return longPollingTimeout; + } + + public apache.rocketmq.v2.Settings toProtobuf() { + Subscription subscription = Subscription.newBuilder().setGroup(group.toProtobuf()).build(); + return apache.rocketmq.v2.Settings.newBuilder().setAccessPoint(accessPoint.toProtobuf()) + .setClientType(clientType.toProtobuf()).setRequestTimeout(Durations.fromNanos(requestTimeout.toNanos())) + .setSubscription(subscription).setUserAgent(UserAgent.INSTANCE.toProtoBuf()).build(); + } + + @Override + public void sync(apache.rocketmq.v2.Settings settings) { + final apache.rocketmq.v2.Settings.PubSubCase pubSubCase = settings.getPubSubCase(); + if (!apache.rocketmq.v2.Settings.PubSubCase.SUBSCRIPTION.equals(pubSubCase)) { + log.error("[Bug] Issued settings not match with the client type, clientId={}, pubSubCase={}, " + + "clientType={}", clientId, pubSubCase, clientType); + return; + } + final Subscription subscription = settings.getSubscription(); + this.longPollingTimeout = Duration.ofNanos(Durations.toNanos(subscription.getLongPollingTimeout())); + } + + @ExcludeFromJacocoGeneratedReport + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("group", group) + .add("longPollingTimeout", longPollingTimeout) + .add("clientId", clientId) + .add("clientType", clientType) + .add("accessPoint", accessPoint) + .add("retryPolicy", retryPolicy) + .add("requestTimeout", requestTimeout) + .toString(); + } +} diff --git a/java/client/src/main/java/org/apache/rocketmq/client/java/impl/consumer/PulledMessageList.java b/java/client/src/main/java/org/apache/rocketmq/client/java/impl/consumer/PulledMessageList.java new file mode 100644 index 000000000..346eacc06 --- /dev/null +++ b/java/client/src/main/java/org/apache/rocketmq/client/java/impl/consumer/PulledMessageList.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.client.java.impl.consumer; + +import java.util.List; +import org.apache.rocketmq.client.java.message.MessageViewImpl; + +public class PulledMessageList { + private final PullProcessQueueImpl processQueue; + private final List messageViews; + + public PulledMessageList(PullProcessQueueImpl processQueue, List messageViews) { + this.processQueue = processQueue; + this.messageViews = messageViews; + } + + public PullProcessQueueImpl getProcessQueue() { + return processQueue; + } + + public List getMessageViews() { + return messageViews; + } +} diff --git a/java/client/src/main/java/org/apache/rocketmq/client/java/impl/consumer/PushConsumerImpl.java b/java/client/src/main/java/org/apache/rocketmq/client/java/impl/consumer/PushConsumerImpl.java index 295367ace..27a4858cd 100644 --- a/java/client/src/main/java/org/apache/rocketmq/client/java/impl/consumer/PushConsumerImpl.java +++ b/java/client/src/main/java/org/apache/rocketmq/client/java/impl/consumer/PushConsumerImpl.java @@ -94,12 +94,12 @@ class PushConsumerImpl extends ConsumerImpl implements PushConsumer { final AtomicLong consumptionOkQuantity; final AtomicLong consumptionErrorQuantity; - private final ClientConfiguration clientConfiguration; + Map subscriptionExpressions; + MessageListener messageListener; + private final PushSubscriptionSettings pushSubscriptionSettings; private final String consumerGroup; - private final Map subscriptionExpressions; private final ConcurrentMap cacheAssignments; - private final MessageListener messageListener; private final int maxCacheMessageCount; private final int maxCacheMessageSizeInBytes; @@ -126,7 +126,6 @@ public PushConsumerImpl(ClientConfiguration clientConfiguration, String consumer Map subscriptionExpressions, MessageListener messageListener, int maxCacheMessageCount, int maxCacheMessageSizeInBytes, int consumptionThreadCount) { super(clientConfiguration, consumerGroup, subscriptionExpressions.keySet()); - this.clientConfiguration = clientConfiguration; Resource groupResource = new Resource(consumerGroup); this.pushSubscriptionSettings = new PushSubscriptionSettings(clientId, endpoints, groupResource, clientConfiguration.getRequestTimeout(), subscriptionExpressions); @@ -285,7 +284,8 @@ ListenableFuture queryAssignment(final String topic) { } /** - * Drop {@link ProcessQueue} by {@link MessageQueueImpl}, {@link ProcessQueue} must be removed before it is dropped. + * Drop {@link PushProcessQueue} by {@link MessageQueueImpl}, {@link PushProcessQueue} must be removed before + * it is dropped. * * @param mq message queue. */ @@ -308,8 +308,9 @@ void dropProcessQueue(MessageQueueImpl mq) { * @param filterExpression filter expression of topic. * @return optional process queue. */ - protected Optional createProcessQueue(MessageQueueImpl mq, final FilterExpression filterExpression) { - final ProcessQueueImpl processQueue = new ProcessQueueImpl(this, mq, filterExpression); + protected Optional createProcessQueue(MessageQueueImpl mq, + final FilterExpression filterExpression) { + final PushProcessQueueImpl processQueue = new PushProcessQueueImpl(this, mq, filterExpression); final ProcessQueue previous = processQueueTable.putIfAbsent(mq, processQueue); if (null != previous) { return Optional.empty(); @@ -323,7 +324,6 @@ public HeartbeatRequest wrapHeartbeatRequest() { .setClientType(ClientType.PUSH_CONSUMER).build(); } - @VisibleForTesting void syncProcessQueue(String topic, Assignments assignments, FilterExpression filterExpression) { Set latest = new HashSet<>(); @@ -334,7 +334,6 @@ void syncProcessQueue(String topic, Assignments assignments, FilterExpression fi } Set activeMqs = new HashSet<>(); - for (Map.Entry entry : processQueueTable.entrySet()) { final MessageQueueImpl mq = entry.getKey(); final ProcessQueue pq = entry.getValue(); @@ -343,8 +342,7 @@ void syncProcessQueue(String topic, Assignments assignments, FilterExpression fi } if (!latest.contains(mq)) { - log.info("Drop message queue according to the latest assignmentList, mq={}, clientId={}", mq, - clientId); + log.info("Drop message queue according to the latest assignmentList, mq={}, clientId={}", mq, clientId); dropProcessQueue(mq); continue; } @@ -361,10 +359,10 @@ void syncProcessQueue(String topic, Assignments assignments, FilterExpression fi if (activeMqs.contains(mq)) { continue; } - final Optional optionalProcessQueue = createProcessQueue(mq, filterExpression); + final Optional optionalProcessQueue = createProcessQueue(mq, filterExpression); if (optionalProcessQueue.isPresent()) { - log.info("Start to fetch message from remote, mq={}, clientId={}", mq, clientId); - optionalProcessQueue.get().fetchMessageImmediately(); + log.info("Start to receive message from remote, mq={}, clientId={}", mq, clientId); + optionalProcessQueue.get().receiveMessageImmediately(); } } } diff --git a/java/client/src/main/java/org/apache/rocketmq/client/java/impl/consumer/PushProcessQueue.java b/java/client/src/main/java/org/apache/rocketmq/client/java/impl/consumer/PushProcessQueue.java new file mode 100644 index 000000000..255d4a240 --- /dev/null +++ b/java/client/src/main/java/org/apache/rocketmq/client/java/impl/consumer/PushProcessQueue.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.client.java.impl.consumer; + +import com.google.common.util.concurrent.ListenableFuture; +import org.apache.rocketmq.client.apis.consumer.ConsumeResult; +import org.apache.rocketmq.client.apis.consumer.PushConsumer; +import org.apache.rocketmq.client.java.message.MessageViewImpl; + +/** + * Process queue is a cache to store fetched messages from remote for {@link PushConsumer}. + * + *

{@link PushConsumer} queries assignments periodically and converts them into message queues, each message queue is + * mapped into one process queue to fetch message from remote. If the message queue is removed from the newest + * assignment, the corresponding process queue is marked as expired soon, which means its lifecycle is over. + * + *

A standard procedure to cache/erase message

+ * + * + *

+ * phase 1: Fetch 32 messages successfully from remote. + * 

+ *  32 in ┌─────────────────────────┐
+ * ───────►           32            │
+ *        └─────────────────────────┘
+ *             cached messages = 32
+ *
+ * phase 2: consuming 1 message. + * 
+ *        ┌─────────────────────┐   ┌───┐
+ *        │          31         ├───► 1 │ consuming
+ *        └─────────────────────┘   └───┘
+ *             cached messages = 32
+ *
+ * phase 3: {@link #eraseMessage(MessageViewImpl, ConsumeResult)} with 1 messages and its consume result. + * 
+ *        ┌─────────────────────┐   ┌───┐ 1 consumed
+ *        │          31         ├───► 0 ├───────────►
+ *        └─────────────────────┘   └───┘
+ *            cached messages = 31
+ *
+ * + *

Especially, there are some different processing procedures for FIFO consumption. The server ensures that the + * next batch of messages will not be obtained by the client until the previous batch of messages is confirmed to be + * consumed successfully or not. In detail, the server confirms the success of consumption by message being + * successfully acknowledged, and confirms the consumption failure by being successfully forwarding to the dead + * letter queue, thus the client should try to ensure it succeeded in acknowledgement or forwarding to the dead + * letter queue as possible. + * + *

Considering the different workflow of FIFO consumption, {@link #eraseFifoMessage(MessageViewImpl, ConsumeResult)} + * and {@link #discardFifoMessage(MessageViewImpl)} is provided. + */ +public interface PushProcessQueue extends ProcessQueue { + /** + * Start to receive messages from remote immediately. + */ + void receiveMessageImmediately(); + + /** + * Erase messages(Non-FIFO-consume-mode) which have been consumed properly. + * + * @param messageView the message to erase. + * @param consumeResult consume result. + */ + void eraseMessage(MessageViewImpl messageView, ConsumeResult consumeResult); + + /** + * Erase message(FIFO-consume-mode) which have been consumed properly. + * + * @param messageView the message to erase. + * @param consumeResult consume status. + */ + ListenableFuture eraseFifoMessage(MessageViewImpl messageView, ConsumeResult consumeResult); + + /** + * Discard the message(Non-FIFO-consume-mode) which could not be consumed properly. + * + * @param messageView the message to discard. + */ + void discardMessage(MessageViewImpl messageView); + + /** + * Discard the message(FIFO-consume-mode) which could not consumed properly. + * + * @param messageView the FIFO message to discard. + */ + void discardFifoMessage(MessageViewImpl messageView); + +} diff --git a/java/client/src/main/java/org/apache/rocketmq/client/java/impl/consumer/ProcessQueueImpl.java b/java/client/src/main/java/org/apache/rocketmq/client/java/impl/consumer/PushProcessQueueImpl.java similarity index 97% rename from java/client/src/main/java/org/apache/rocketmq/client/java/impl/consumer/ProcessQueueImpl.java rename to java/client/src/main/java/org/apache/rocketmq/client/java/impl/consumer/PushProcessQueueImpl.java index c43b4a10f..b9df98bff 100644 --- a/java/client/src/main/java/org/apache/rocketmq/client/java/impl/consumer/ProcessQueueImpl.java +++ b/java/client/src/main/java/org/apache/rocketmq/client/java/impl/consumer/PushProcessQueueImpl.java @@ -67,19 +67,19 @@ import org.slf4j.LoggerFactory; /** - * Default implementation of {@link ProcessQueue}. + * Default implementation of {@link PushProcessQueue}. * - *

Apart from the basic part mentioned in {@link ProcessQueue}, this implementation + *

Apart from the basic part mentioned in {@link PushProcessQueue}, this implementation * - * @see ProcessQueue + * @see PushProcessQueue */ @SuppressWarnings({"NullableProblems", "UnstableApiUsage"}) -class ProcessQueueImpl implements ProcessQueue { +class PushProcessQueueImpl implements PushProcessQueue { static final Duration FORWARD_FIFO_MESSAGE_TO_DLQ_FAILURE_BACKOFF_DELAY = Duration.ofSeconds(1); static final Duration ACK_MESSAGE_FAILURE_BACKOFF_DELAY = Duration.ofSeconds(1); static final Duration CHANGE_INVISIBLE_DURATION_FAILURE_BACKOFF_DELAY = Duration.ofSeconds(1); - private static final Logger log = LoggerFactory.getLogger(ProcessQueueImpl.class); + private static final Logger log = LoggerFactory.getLogger(PushProcessQueueImpl.class); private static final Duration RECEIVING_FLOW_CONTROL_BACKOFF_DELAY = Duration.ofMillis(20); private static final Duration RECEIVING_FAILURE_BACKOFF_DELAY = Duration.ofSeconds(1); @@ -88,7 +88,8 @@ class ProcessQueueImpl implements ProcessQueue { private final PushConsumerImpl consumer; /** - * Dropped means {@link ProcessQueue} is deprecated, which means no message would be fetched from remote anymore. + * Dropped means {@link PushProcessQueue} is deprecated, which means no message would be fetched from remote + * anymore. */ private volatile boolean dropped; private final MessageQueueImpl mq; @@ -109,7 +110,7 @@ class ProcessQueueImpl implements ProcessQueue { private volatile long activityNanoTime = System.nanoTime(); private volatile long cacheFullNanoTime = Long.MIN_VALUE; - public ProcessQueueImpl(PushConsumerImpl consumer, MessageQueueImpl mq, FilterExpression filterExpression) { + public PushProcessQueueImpl(PushConsumerImpl consumer, MessageQueueImpl mq, FilterExpression filterExpression) { this.consumer = consumer; this.dropped = false; this.mq = mq; @@ -149,7 +150,7 @@ public boolean expired() { return true; } - void cacheMessages(List messageList) { + public void cacheMessages(List messageList) { cachedMessageLock.writeLock().lock(); try { for (MessageViewImpl messageView : messageList) { @@ -167,11 +168,6 @@ private int getReceptionBatchSize() { return Math.min(bufferSize, consumer.getPushConsumerSettings().getReceiveBatchSize()); } - @Override - public void fetchMessageImmediately() { - receiveMessageImmediately(); - } - /** * Receive message later by message queue. * @@ -221,7 +217,7 @@ public void receiveMessage(String attemptId) { receiveMessageImmediately(attemptId); } - private void receiveMessageImmediately() { + public void receiveMessageImmediately() { receiveMessageImmediately(this.generateAttemptId()); } @@ -495,7 +491,6 @@ public ListenableFuture eraseFifoMessage(MessageViewImpl messageView, Cons return future; } - private ListenableFuture forwardToDeadLetterQueue(final MessageViewImpl messageView) { final SettableFuture future = SettableFuture.create(); forwardToDeadLetterQueue(messageView, 1, future); @@ -665,7 +660,7 @@ public void doStats() { final long receptionTimes = this.receptionTimes.getAndSet(0); final long receivedMessagesQuantity = this.receivedMessagesQuantity.getAndSet(0); log.info("Process queue stats: clientId={}, mq={}, receptionTimes={}, receivedMessageQuantity={}, " - + "cachedMessageCount={}, cachedMessageBytes={}", consumer.getClientId(), mq, receptionTimes, + + "cachedMessageCount={}, cachedMessageBytes={}", consumer.getClientId(), mq, receptionTimes, receivedMessagesQuantity, this.getCachedMessageCount(), this.getCachedMessageBytes()); } } diff --git a/java/client/src/main/java/org/apache/rocketmq/client/java/impl/consumer/PushSubscriptionSettings.java b/java/client/src/main/java/org/apache/rocketmq/client/java/impl/consumer/PushSubscriptionSettings.java index 70338b0cd..fd870827b 100644 --- a/java/client/src/main/java/org/apache/rocketmq/client/java/impl/consumer/PushSubscriptionSettings.java +++ b/java/client/src/main/java/org/apache/rocketmq/client/java/impl/consumer/PushSubscriptionSettings.java @@ -21,6 +21,7 @@ import apache.rocketmq.v2.RetryPolicy; import apache.rocketmq.v2.Subscription; import apache.rocketmq.v2.SubscriptionEntry; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.MoreObjects; import com.google.protobuf.util.Durations; import java.time.Duration; @@ -44,9 +45,9 @@ public class PushSubscriptionSettings extends Settings { private static final Logger log = LoggerFactory.getLogger(PushSubscriptionSettings.class); + private volatile Boolean fifo = null; private final Resource group; private final Map subscriptionExpressions; - private volatile Boolean fifo = false; private volatile int receiveBatchSize = 32; private volatile Duration longPollingTimeout = Duration.ofSeconds(30); @@ -57,6 +58,11 @@ public PushSubscriptionSettings(ClientId clientId, Endpoints endpoints, Resource this.subscriptionExpressions = subscriptionExpression; } + @VisibleForTesting + public void setFifo(boolean fifo) { + this.fifo = fifo; + } + public boolean isFifo() { return fifo; } @@ -93,11 +99,21 @@ public apache.rocketmq.v2.Settings toProtobuf() { SubscriptionEntry.newBuilder().setTopic(topic).setExpression(expressionBuilder.build()).build(); subscriptionEntries.add(subscriptionEntry); } - Subscription subscription = - Subscription.newBuilder().setGroup(group.toProtobuf()).addAllSubscriptions(subscriptionEntries).build(); - return apache.rocketmq.v2.Settings.newBuilder().setAccessPoint(accessPoint.toProtobuf()) - .setClientType(clientType.toProtobuf()).setRequestTimeout(Durations.fromNanos(requestTimeout.toNanos())) - .setSubscription(subscription).setUserAgent(UserAgent.INSTANCE.toProtoBuf()).build(); + + final Subscription.Builder builder = Subscription.newBuilder().setGroup(group.toProtobuf()) + .addAllSubscriptions(subscriptionEntries); + if (null != fifo) { + builder.setFifo(fifo); + } + final Subscription subscription = builder.build(); + final apache.rocketmq.v2.Settings.Builder settingsBuilder = apache.rocketmq.v2.Settings.newBuilder() + .setAccessPoint(accessPoint.toProtobuf()).setClientType(clientType.toProtobuf()) + .setRequestTimeout(Durations.fromNanos(requestTimeout.toNanos())) + .setSubscription(subscription).setUserAgent(UserAgent.INSTANCE.toProtoBuf()); + if (null == retryPolicy) { + return settingsBuilder.build(); + } + return settingsBuilder.setBackoffPolicy(retryPolicy.toProtobuf()).build(); } @Override diff --git a/java/client/src/main/java/org/apache/rocketmq/client/java/impl/consumer/StandardConsumeService.java b/java/client/src/main/java/org/apache/rocketmq/client/java/impl/consumer/StandardConsumeService.java index 2775077ae..fda533ae8 100644 --- a/java/client/src/main/java/org/apache/rocketmq/client/java/impl/consumer/StandardConsumeService.java +++ b/java/client/src/main/java/org/apache/rocketmq/client/java/impl/consumer/StandardConsumeService.java @@ -43,7 +43,7 @@ public StandardConsumeService(ClientId clientId, MessageListener messageListener } @Override - public void consume(ProcessQueue pq, List messageViews) { + public void consume(PushProcessQueue pq, List messageViews) { for (MessageViewImpl messageView : messageViews) { // Discard corrupted message. if (messageView.isCorrupted()) { diff --git a/java/client/src/main/java/org/apache/rocketmq/client/java/misc/CacheBlockingListQueue.java b/java/client/src/main/java/org/apache/rocketmq/client/java/misc/CacheBlockingListQueue.java new file mode 100644 index 000000000..336f3afe5 --- /dev/null +++ b/java/client/src/main/java/org/apache/rocketmq/client/java/misc/CacheBlockingListQueue.java @@ -0,0 +1,175 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.client.java.misc; + +import static com.google.common.base.Preconditions.checkNotNull; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.locks.ReentrantReadWriteLock; +import org.apache.commons.lang3.tuple.Pair; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * CacheBlockingListQueue is a custom implementation of a linked blocking queue which caches pairs of keys and lists + * of elements. It is an extension of {@link CustomLinkedBlockingQueue}. It is thread-safe and supports operations + * for caching, polling, and dropping items. + * + * @param The type of the key used for caching. + * @param The type of the elements present in the list. + */ +public class CacheBlockingListQueue extends CustomLinkedBlockingQueue>> { + private static final long serialVersionUID = 3505724726193519379L; + private static final Logger log = LoggerFactory.getLogger(CacheBlockingListQueue.class); + private final Map>> cacheMap; + private final ReentrantReadWriteLock lock; + + /** + * Constructs a new CacheBlockingListQueue with an empty cache map and a read-write lock. + */ + public CacheBlockingListQueue() { + this.cacheMap = new HashMap<>(); + this.lock = new ReentrantReadWriteLock(); + } + + public List peek(T t) { + lock.readLock().lock(); + try { + final Pair> pair = cacheMap.get(t); + if (null == pair) { + return new ArrayList<>(); + } + final List value = pair.getValue(); + if (null == value) { + return new ArrayList<>(); + } + return Collections.unmodifiableList(value); + } finally { + lock.readLock().unlock(); + } + } + + /** + * Caches a key with a linked list of elements as a pair. + * + * @param t The key to be cached. + * @param linkedElement The list of elements to be associated with the key. + */ + @SuppressWarnings("UnusedReturnValue") + public void cache(T t, List linkedElement) { + final Pair> pair = Pair.of(t, linkedElement); + this.offer(pair); + } + + /** + * Retrieves and removes next available queue from the head of the queue, + * waiting up to the specified duration if necessary. + * + * @param duration Time duration to wait before giving up. + * @throws InterruptedException if the current thread is interrupted while waiting. + */ + public Pair> poll(Duration duration) throws InterruptedException { + return poll(duration.toNanos(), TimeUnit.NANOSECONDS); + } + + /** + * Removes the key and its associated list of elements from the cache, if it exists. + * + * @param key The key to be dropped from the cache. + */ + @SuppressWarnings("ResultOfMethodCallIgnored") + public void drop(T key) { + fullyLock(); + lock.writeLock().lock(); + try { + final Pair> queue = cacheMap.remove(key); + remove(queue); + } finally { + lock.writeLock().unlock(); + fullyUnlock(); + } + } + + @Override + protected Pair> dequeue() { + lock.writeLock().lock(); + try { + final Pair> queue = super.dequeue(); + final T key = queue.getKey(); + cacheMap.remove(key); + return queue; + } finally { + lock.writeLock().unlock(); + } + } + + /** + * Inserts the specified element at the tail of the queue, respecting the key constraints. + * This method is thread-safe and achieves its goal by acquiring a write lock. + * + * @param e The element to be added to the tail of the queue. + * @return true if the element is added successfully, false otherwise. + */ + @SuppressWarnings("NullableProblems") + @Override + public boolean offer(Pair> e) { + checkNotNull(e); + final T key = e.getKey(); + putLock.lock(); + lock.writeLock().lock(); + final int c; + try { + final Pair> exist = cacheMap.get(key); + if (exist != null) { + log.debug("Key exists in the cache, key={}", key); + exist.getValue().addAll(e.getValue()); + return false; + } + log.debug("Key doesn't exist in the cache, key={}", key); + cacheMap.put(key, e); + final AtomicInteger count = this.count; + if (count.get() == capacity) { + return false; + } + final CustomLinkedBlockingQueue.Node>> node = new CustomLinkedBlockingQueue.Node<>(e); + if (count.get() == capacity) { + return false; + } + enqueue(node); + c = count.getAndIncrement(); + if (c + 1 < capacity) { + notFull.signal(); + } + } finally { + lock.writeLock().unlock(); + putLock.unlock(); + } + if (c == 0) { + signalNotEmpty(); + } + return true; + + } +} diff --git a/java/client/src/main/java/org/apache/rocketmq/client/java/misc/CustomLinkedBlockingQueue.java b/java/client/src/main/java/org/apache/rocketmq/client/java/misc/CustomLinkedBlockingQueue.java new file mode 100644 index 000000000..da77cc05c --- /dev/null +++ b/java/client/src/main/java/org/apache/rocketmq/client/java/misc/CustomLinkedBlockingQueue.java @@ -0,0 +1,1117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.client.java.misc; + +import java.io.Serializable; +import java.util.AbstractQueue; +import java.util.Collection; +import java.util.Iterator; +import java.util.NoSuchElementException; +import java.util.Objects; +import java.util.Spliterator; +import java.util.Spliterators; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.locks.Condition; +import java.util.concurrent.locks.ReentrantLock; +import java.util.function.Consumer; +import java.util.function.Predicate; + +/** + * CustomLinkedBlockingQueue is an extension of the {@link LinkedBlockingQueue} class, + * providing additional functionality and access to protected or public methods. + * This class is designed for situations where some of the original methods need + * to be customized or extended to cater to specific use-cases, while maintaining + * the thread-safety and overall queue functionality of the LinkedBlockingQueue. + */ +@SuppressWarnings("ALL") +@ExcludeFromJacocoGeneratedReport +public class CustomLinkedBlockingQueue extends AbstractQueue implements BlockingQueue, Serializable { + private static final long serialVersionUID = -7874526962038421584L; + + /** + * Linked list node class. + */ + static class Node { + E item; + + /** + * One of: + * - the real successor Node + * - this Node, meaning the successor is head.next + * - null, meaning there is no successor (this is the last node) + */ + CustomLinkedBlockingQueue.Node next; + + Node(E x) { + item = x; + } + } + + /** + * The capacity bound, or Integer.MAX_VALUE if none + */ + protected final int capacity; + + /** + * Current number of elements + */ + protected final AtomicInteger count = new AtomicInteger(); + + /** + * Head of linked list. + * Invariant: head.item == null + */ + transient CustomLinkedBlockingQueue.Node head; + + /** + * Tail of linked list. + * Invariant: last.next == null + */ + private transient CustomLinkedBlockingQueue.Node last; + + /** + * Lock held by take, poll, etc + */ + private final ReentrantLock takeLock = new ReentrantLock(); + + /** + * Wait queue for waiting takes + */ + private final Condition notEmpty = takeLock.newCondition(); + + /** + * Lock held by put, offer, etc + */ + protected final ReentrantLock putLock = new ReentrantLock(); + + /** + * Wait queue for waiting puts + */ + protected final Condition notFull = putLock.newCondition(); + + /** + * Signals a waiting take. Called only from put/offer (which do not + * otherwise ordinarily lock takeLock.) + */ + protected void signalNotEmpty() { + final ReentrantLock takeLock = this.takeLock; + takeLock.lock(); + try { + notEmpty.signal(); + } finally { + takeLock.unlock(); + } + } + + /** + * Signals a waiting put. Called only from take/poll. + */ + private void signalNotFull() { + final ReentrantLock putLock = this.putLock; + putLock.lock(); + try { + notFull.signal(); + } finally { + putLock.unlock(); + } + } + + /** + * Links node at end of queue. + * + * @param node the node + */ + protected void enqueue(CustomLinkedBlockingQueue.Node node) { + // assert putLock.isHeldByCurrentThread(); + // assert last.next == null; + last = last.next = node; + } + + /** + * Removes a node from head of queue. + * + * @return the node + */ + protected E dequeue() { + // assert takeLock.isHeldByCurrentThread(); + // assert head.item == null; + CustomLinkedBlockingQueue.Node h = head; + CustomLinkedBlockingQueue.Node first = h.next; + h.next = h; // help GC + head = first; + E x = first.item; + first.item = null; + return x; + } + + /** + * Locks to prevent both puts and takes. + */ + void fullyLock() { + putLock.lock(); + takeLock.lock(); + } + + /** + * Unlocks to allow both puts and takes. + */ + void fullyUnlock() { + takeLock.unlock(); + putLock.unlock(); + } + + /** + * Creates a {@code LinkedBlockingQueue} with a capacity of + * {@link Integer#MAX_VALUE}. + */ + public CustomLinkedBlockingQueue() { + this(Integer.MAX_VALUE); + } + + /** + * Creates a {@code LinkedBlockingQueue} with the given (fixed) capacity. + * + * @param capacity the capacity of this queue + * @throws IllegalArgumentException if {@code capacity} is not greater + * than zero + */ + public CustomLinkedBlockingQueue(int capacity) { + if (capacity <= 0) + throw new IllegalArgumentException(); + this.capacity = capacity; + last = head = new CustomLinkedBlockingQueue.Node(null); + } + + /** + * Creates a {@code LinkedBlockingQueue} with a capacity of + * {@link Integer#MAX_VALUE}, initially containing the elements of the + * given collection, + * added in traversal order of the collection's iterator. + * + * @param c the collection of elements to initially contain + * @throws NullPointerException if the specified collection or any + * of its elements are null + */ + public CustomLinkedBlockingQueue(Collection c) { + this(Integer.MAX_VALUE); + final ReentrantLock putLock = this.putLock; + putLock.lock(); // Never contended, but necessary for visibility + try { + int n = 0; + for (E e : c) { + if (e == null) + throw new NullPointerException(); + if (n == capacity) + throw new IllegalStateException("Queue full"); + enqueue(new CustomLinkedBlockingQueue.Node(e)); + ++n; + } + count.set(n); + } finally { + putLock.unlock(); + } + } + + // this doc comment is overridden to remove the reference to collections + // greater in size than Integer.MAX_VALUE + + /** + * Returns the number of elements in this queue. + * + * @return the number of elements in this queue + */ + public int size() { + return count.get(); + } + + // this doc comment is a modified copy of the inherited doc comment, + // without the reference to unlimited queues. + + /** + * Returns the number of additional elements that this queue can ideally + * (in the absence of memory or resource constraints) accept without + * blocking. This is always equal to the initial capacity of this queue + * less the current {@code size} of this queue. + * + *

Note that you cannot always tell if an attempt to insert + * an element will succeed by inspecting {@code remainingCapacity} + * because it may be the case that another thread is about to + * insert or remove an element. + */ + public int remainingCapacity() { + return capacity - count.get(); + } + + /** + * Inserts the specified element at the tail of this queue, waiting if + * necessary for space to become available. + * + * @throws InterruptedException {@inheritDoc} + * @throws NullPointerException {@inheritDoc} + */ + public void put(E e) throws InterruptedException { + if (e == null) + throw new NullPointerException(); + final int c; + final CustomLinkedBlockingQueue.Node node = new CustomLinkedBlockingQueue.Node(e); + final ReentrantLock putLock = this.putLock; + final AtomicInteger count = this.count; + putLock.lockInterruptibly(); + try { + /* + * Note that count is used in wait guard even though it is + * not private by lock. This works because count can + * only decrease at this point (all other puts are shut + * out by lock), and we (or some other waiting put) are + * signalled if it ever changes from capacity. Similarly + * for all other uses of count in other wait guards. + */ + while (count.get() == capacity) { + notFull.await(); + } + enqueue(node); + c = count.getAndIncrement(); + if (c + 1 < capacity) + notFull.signal(); + } finally { + putLock.unlock(); + } + if (c == 0) + signalNotEmpty(); + } + + /** + * Inserts the specified element at the tail of this queue, waiting if + * necessary up to the specified wait time for space to become available. + * + * @return {@code true} if successful, or {@code false} if + * the specified waiting time elapses before space is available + * @throws InterruptedException {@inheritDoc} + * @throws NullPointerException {@inheritDoc} + */ + public boolean offer(E e, long timeout, TimeUnit unit) + throws InterruptedException { + + if (e == null) + throw new NullPointerException(); + long nanos = unit.toNanos(timeout); + final int c; + final ReentrantLock putLock = this.putLock; + final AtomicInteger count = this.count; + putLock.lockInterruptibly(); + try { + while (count.get() == capacity) { + if (nanos <= 0L) + return false; + nanos = notFull.awaitNanos(nanos); + } + enqueue(new CustomLinkedBlockingQueue.Node(e)); + c = count.getAndIncrement(); + if (c + 1 < capacity) + notFull.signal(); + } finally { + putLock.unlock(); + } + if (c == 0) + signalNotEmpty(); + return true; + } + + /** + * Inserts the specified element at the tail of this queue if it is + * possible to do so immediately without exceeding the queue's capacity, + * returning {@code true} upon success and {@code false} if this queue + * is full. + * When using a capacity-restricted queue, this method is generally + * preferable to method {@link BlockingQueue#add add}, which can fail to + * insert an element only by throwing an exception. + * + * @throws NullPointerException if the specified element is null + */ + public boolean offer(E e) { + if (e == null) + throw new NullPointerException(); + final AtomicInteger count = this.count; + if (count.get() == capacity) + return false; + final int c; + final CustomLinkedBlockingQueue.Node node = new CustomLinkedBlockingQueue.Node(e); + final ReentrantLock putLock = this.putLock; + putLock.lock(); + try { + if (count.get() == capacity) + return false; + enqueue(node); + c = count.getAndIncrement(); + if (c + 1 < capacity) + notFull.signal(); + } finally { + putLock.unlock(); + } + if (c == 0) + signalNotEmpty(); + return true; + } + + public E take() throws InterruptedException { + final E x; + final int c; + final AtomicInteger count = this.count; + final ReentrantLock takeLock = this.takeLock; + takeLock.lockInterruptibly(); + try { + while (count.get() == 0) { + notEmpty.await(); + } + x = dequeue(); + c = count.getAndDecrement(); + if (c > 1) + notEmpty.signal(); + } finally { + takeLock.unlock(); + } + if (c == capacity) + signalNotFull(); + return x; + } + + public E poll(long timeout, TimeUnit unit) throws InterruptedException { + final E x; + final int c; + long nanos = unit.toNanos(timeout); + final AtomicInteger count = this.count; + final ReentrantLock takeLock = this.takeLock; + takeLock.lockInterruptibly(); + try { + while (count.get() == 0) { + if (nanos <= 0L) + return null; + nanos = notEmpty.awaitNanos(nanos); + } + x = dequeue(); + c = count.getAndDecrement(); + if (c > 1) + notEmpty.signal(); + } finally { + takeLock.unlock(); + } + if (c == capacity) + signalNotFull(); + return x; + } + + public E poll() { + final AtomicInteger count = this.count; + if (count.get() == 0) + return null; + final E x; + final int c; + final ReentrantLock takeLock = this.takeLock; + takeLock.lock(); + try { + if (count.get() == 0) + return null; + x = dequeue(); + c = count.getAndDecrement(); + if (c > 1) + notEmpty.signal(); + } finally { + takeLock.unlock(); + } + if (c == capacity) + signalNotFull(); + return x; + } + + public E peek() { + final AtomicInteger count = this.count; + if (count.get() == 0) + return null; + final ReentrantLock takeLock = this.takeLock; + takeLock.lock(); + try { + return (count.get() > 0) ? head.next.item : null; + } finally { + takeLock.unlock(); + } + } + + /** + * Unlinks interior Node p with predecessor pred. + */ + void unlink( + CustomLinkedBlockingQueue.Node p, CustomLinkedBlockingQueue.Node pred) { + // assert putLock.isHeldByCurrentThread(); + // assert takeLock.isHeldByCurrentThread(); + // p.next is not changed, to allow iterators that are + // traversing p to maintain their weak-consistency guarantee. + p.item = null; + pred.next = p.next; + if (last == p) + last = pred; + if (count.getAndDecrement() == capacity) + notFull.signal(); + } + + /** + * Removes a single instance of the specified element from this queue, + * if it is present. More formally, removes an element {@code e} such + * that {@code o.equals(e)}, if this queue contains one or more such + * elements. + * Returns {@code true} if this queue contained the specified element + * (or equivalently, if this queue changed as a result of the call). + * + * @param o element to be removed from this queue, if present + * @return {@code true} if this queue changed as a result of the call + */ + public boolean remove(Object o) { + if (o == null) + return false; + fullyLock(); + try { + for (CustomLinkedBlockingQueue.Node pred = head, p = pred.next; + p != null; + pred = p, p = p.next) { + if (o.equals(p.item)) { + unlink(p, pred); + return true; + } + } + return false; + } finally { + fullyUnlock(); + } + } + + /** + * Returns {@code true} if this queue contains the specified element. + * More formally, returns {@code true} if and only if this queue contains + * at least one element {@code e} such that {@code o.equals(e)}. + * + * @param o object to be checked for containment in this queue + * @return {@code true} if this queue contains the specified element + */ + public boolean contains(Object o) { + if (o == null) + return false; + fullyLock(); + try { + for (CustomLinkedBlockingQueue.Node p = head.next; p != null; p = p.next) + if (o.equals(p.item)) + return true; + return false; + } finally { + fullyUnlock(); + } + } + + /** + * Returns an array containing all of the elements in this queue, in + * proper sequence. + * + *

The returned array will be "safe" in that no references to it are + * maintained by this queue. (In other words, this method must allocate + * a new array). The caller is thus free to modify the returned array. + * + *

This method acts as bridge between array-based and collection-based + * APIs. + * + * @return an array containing all of the elements in this queue + */ + public Object[] toArray() { + fullyLock(); + try { + int size = count.get(); + Object[] a = new Object[size]; + int k = 0; + for (CustomLinkedBlockingQueue.Node p = head.next; p != null; p = p.next) + a[k++] = p.item; + return a; + } finally { + fullyUnlock(); + } + } + + /** + * Returns an array containing all of the elements in this queue, in + * proper sequence; the runtime type of the returned array is that of + * the specified array. If the queue fits in the specified array, it + * is returned therein. Otherwise, a new array is allocated with the + * runtime type of the specified array and the size of this queue. + * + *

If this queue fits in the specified array with room to spare + * (i.e., the array has more elements than this queue), the element in + * the array immediately following the end of the queue is set to + * {@code null}. + * + *

Like the {@link #toArray()} method, this method acts as bridge between + * array-based and collection-based APIs. Further, this method allows + * precise control over the runtime type of the output array, and may, + * under certain circumstances, be used to save allocation costs. + * + *

Suppose {@code x} is a queue known to contain only strings. + * The following code can be used to dump the queue into a newly + * allocated array of {@code String}: + * + * 

 {@code String[] y = x.toArray(new String[0]);}
+ *

+ * Note that {@code toArray(new Object[0])} is identical in function to + * {@code toArray()}. + * + * @param a the array into which the elements of the queue are to + * be stored, if it is big enough; otherwise, a new array of the + * same runtime type is allocated for this purpose + * @return an array containing all of the elements in this queue + * @throws ArrayStoreException if the runtime type of the specified array + * is not a supertype of the runtime type of every element in + * this queue + * @throws NullPointerException if the specified array is null + */ + @SuppressWarnings("unchecked") + public T[] toArray(T[] a) { + fullyLock(); + try { + int size = count.get(); + if (a.length < size) + a = (T[]) java.lang.reflect.Array.newInstance + (a.getClass().getComponentType(), size); + + int k = 0; + for (CustomLinkedBlockingQueue.Node p = head.next; p != null; p = p.next) + a[k++] = (T) p.item; + if (a.length > k) + a[k] = null; + return a; + } finally { + fullyUnlock(); + } + } + + /** + * Atomically removes all of the elements from this queue. + * The queue will be empty after this call returns. + */ + public void clear() { + fullyLock(); + try { + for (CustomLinkedBlockingQueue.Node p, h = head; (p = h.next) != null; h = p) { + h.next = h; + p.item = null; + } + head = last; + // assert head.item == null && head.next == null; + if (count.getAndSet(0) == capacity) + notFull.signal(); + } finally { + fullyUnlock(); + } + } + + /** + * @throws UnsupportedOperationException {@inheritDoc} + * @throws ClassCastException {@inheritDoc} + * @throws NullPointerException {@inheritDoc} + * @throws IllegalArgumentException {@inheritDoc} + */ + public int drainTo(Collection c) { + return drainTo(c, Integer.MAX_VALUE); + } + + /** + * @throws UnsupportedOperationException {@inheritDoc} + * @throws ClassCastException {@inheritDoc} + * @throws NullPointerException {@inheritDoc} + * @throws IllegalArgumentException {@inheritDoc} + */ + public int drainTo(Collection c, int maxElements) { + Objects.requireNonNull(c); + if (c == this) + throw new IllegalArgumentException(); + if (maxElements <= 0) + return 0; + boolean signalNotFull = false; + final ReentrantLock takeLock = this.takeLock; + takeLock.lock(); + try { + int n = Math.min(maxElements, count.get()); + // count.get provides visibility to first n Nodes + CustomLinkedBlockingQueue.Node h = head; + int i = 0; + try { + while (i < n) { + CustomLinkedBlockingQueue.Node p = h.next; + c.add(p.item); + p.item = null; + h.next = h; + h = p; + ++i; + } + return n; + } finally { + // Restore invariants even if c.add() threw + if (i > 0) { + // assert h.item == null; + head = h; + signalNotFull = (count.getAndAdd(-i) == capacity); + } + } + } finally { + takeLock.unlock(); + if (signalNotFull) + signalNotFull(); + } + } + + /** + * Used for any element traversal that is not entirely under lock. + * Such traversals must handle both: + * - dequeued nodes (p.next == p) + * - (possibly multiple) interior removed nodes (p.item == null) + */ + CustomLinkedBlockingQueue.Node succ(CustomLinkedBlockingQueue.Node p) { + if (p == (p = p.next)) + p = head.next; + return p; + } + + /** + * Returns an iterator over the elements in this queue in proper sequence. + * The elements will be returned in order from first (head) to last (tail). + * + *

The returned iterator is + * weakly consistent. + * + * @return an iterator over the elements in this queue in proper sequence + */ + public Iterator iterator() { + return new CustomLinkedBlockingQueue.Itr(); + } + + /** + * Weakly-consistent iterator. + *

+ * Lazily updated ancestor field provides expected O(1) remove(), + * but still O(n) in the worst case, whenever the saved ancestor + * is concurrently deleted. + */ + private class Itr implements Iterator { + private CustomLinkedBlockingQueue.Node next; // Node holding nextItem + private E nextItem; // next item to hand out + private CustomLinkedBlockingQueue.Node lastRet; + private CustomLinkedBlockingQueue.Node ancestor; // Helps unlink lastRet on remove() + + Itr() { + fullyLock(); + try { + if ((next = head.next) != null) + nextItem = next.item; + } finally { + fullyUnlock(); + } + } + + public boolean hasNext() { + return next != null; + } + + public E next() { + CustomLinkedBlockingQueue.Node p; + if ((p = next) == null) + throw new NoSuchElementException(); + lastRet = p; + E x = nextItem; + fullyLock(); + try { + E e = null; + for (p = p.next; p != null && (e = p.item) == null; ) + p = succ(p); + next = p; + nextItem = e; + } finally { + fullyUnlock(); + } + return x; + } + + public void forEachRemaining(Consumer action) { + // A variant of forEachFrom + Objects.requireNonNull(action); + CustomLinkedBlockingQueue.Node p; + if ((p = next) == null) + return; + lastRet = p; + next = null; + final int batchSize = 64; + Object[] es = null; + int n, len = 1; + do { + fullyLock(); + try { + if (es == null) { + p = p.next; + for (CustomLinkedBlockingQueue.Node q = p; q != null; q = succ(q)) + if (q.item != null && ++len == batchSize) + break; + es = new Object[len]; + es[0] = nextItem; + nextItem = null; + n = 1; + } else + n = 0; + for (; p != null && n < len; p = succ(p)) + if ((es[n] = p.item) != null) { + lastRet = p; + n++; + } + } finally { + fullyUnlock(); + } + for (int i = 0; i < n; i++) { + @SuppressWarnings("unchecked") E e = (E) es[i]; + action.accept(e); + } + } + while (n > 0 && p != null); + } + + public void remove() { + CustomLinkedBlockingQueue.Node p = lastRet; + if (p == null) + throw new IllegalStateException(); + lastRet = null; + fullyLock(); + try { + if (p.item != null) { + if (ancestor == null) + ancestor = head; + ancestor = findPred(p, ancestor); + unlink(p, ancestor); + } + } finally { + fullyUnlock(); + } + } + } + + /** + * A customized variant of Spliterators.IteratorSpliterator. + * Keep this class in sync with (very similar) LBDSpliterator. + */ + private final class LBQSpliterator implements Spliterator { + static final int MAX_BATCH = 1 << 25; // max batch array size; + CustomLinkedBlockingQueue.Node current; // current node; null until initialized + int batch; // batch size for splits + boolean exhausted; // true when no more nodes + long est = size(); // size estimate + + LBQSpliterator() { + } + + public long estimateSize() { + return est; + } + + public Spliterator trySplit() { + CustomLinkedBlockingQueue.Node h; + if (!exhausted && + ((h = current) != null || (h = head.next) != null) + && h.next != null) { + int n = batch = Math.min(batch + 1, MAX_BATCH); + Object[] a = new Object[n]; + int i = 0; + CustomLinkedBlockingQueue.Node p = current; + fullyLock(); + try { + if (p != null || (p = head.next) != null) + for (; p != null && i < n; p = succ(p)) + if ((a[i] = p.item) != null) + i++; + } finally { + fullyUnlock(); + } + if ((current = p) == null) { + est = 0L; + exhausted = true; + } else if ((est -= i) < 0L) + est = 0L; + if (i > 0) + return Spliterators.spliterator + (a, 0, i, (Spliterator.ORDERED | + Spliterator.NONNULL | + Spliterator.CONCURRENT)); + } + return null; + } + + public boolean tryAdvance(Consumer action) { + Objects.requireNonNull(action); + if (!exhausted) { + E e = null; + fullyLock(); + try { + CustomLinkedBlockingQueue.Node p; + if ((p = current) != null || (p = head.next) != null) + do { + e = p.item; + p = succ(p); + } + while (e == null && p != null); + if ((current = p) == null) + exhausted = true; + } finally { + fullyUnlock(); + } + if (e != null) { + action.accept(e); + return true; + } + } + return false; + } + + public void forEachRemaining(Consumer action) { + Objects.requireNonNull(action); + if (!exhausted) { + exhausted = true; + CustomLinkedBlockingQueue.Node p = current; + current = null; + forEachFrom(action, p); + } + } + + public int characteristics() { + return (Spliterator.ORDERED | + Spliterator.NONNULL | + Spliterator.CONCURRENT); + } + } + + /** + * Returns a {@link Spliterator} over the elements in this queue. + * + *

The returned spliterator is + * weakly consistent. + * + *

The {@code Spliterator} reports {@link Spliterator#CONCURRENT}, + * {@link Spliterator#ORDERED}, and {@link Spliterator#NONNULL}. + * + * @return a {@code Spliterator} over the elements in this queue + * @implNote The {@code Spliterator} implements {@code trySplit} to permit limited + * parallelism. + * @since 1.8 + */ + public Spliterator spliterator() { + return new CustomLinkedBlockingQueue.LBQSpliterator(); + } + + /** + * @throws NullPointerException {@inheritDoc} + */ + public void forEach(Consumer action) { + Objects.requireNonNull(action); + forEachFrom(action, null); + } + + /** + * Runs action on each element found during a traversal starting at p. + * If p is null, traversal starts at head. + */ + void forEachFrom(Consumer action, CustomLinkedBlockingQueue.Node p) { + // Extract batches of elements while holding the lock; then + // run the action on the elements while not + final int batchSize = 64; // max number of elements per batch + Object[] es = null; // container for batch of elements + int n, len = 0; + do { + fullyLock(); + try { + if (es == null) { + if (p == null) + p = head.next; + for (CustomLinkedBlockingQueue.Node q = p; q != null; q = succ(q)) + if (q.item != null && ++len == batchSize) + break; + es = new Object[len]; + } + for (n = 0; p != null && n < len; p = succ(p)) + if ((es[n] = p.item) != null) + n++; + } finally { + fullyUnlock(); + } + for (int i = 0; i < n; i++) { + @SuppressWarnings("unchecked") E e = (E) es[i]; + action.accept(e); + } + } + while (n > 0 && p != null); + } + + /** + * @throws NullPointerException {@inheritDoc} + */ + public boolean removeIf(Predicate filter) { + Objects.requireNonNull(filter); + return bulkRemove(filter); + } + + /** + * @throws NullPointerException {@inheritDoc} + */ + public boolean removeAll(Collection c) { + Objects.requireNonNull(c); + return bulkRemove(e -> c.contains(e)); + } + + /** + * @throws NullPointerException {@inheritDoc} + */ + public boolean retainAll(Collection c) { + Objects.requireNonNull(c); + return bulkRemove(e -> !c.contains(e)); + } + + /** + * Returns the predecessor of live node p, given a node that was + * once a live ancestor of p (or head); allows unlinking of p. + */ + CustomLinkedBlockingQueue.Node findPred( + CustomLinkedBlockingQueue.Node p, CustomLinkedBlockingQueue.Node ancestor) { + // assert p.item != null; + if (ancestor.item == null) + ancestor = head; + // Fails with NPE if precondition not satisfied + for (CustomLinkedBlockingQueue.Node q; (q = ancestor.next) != p; ) + ancestor = q; + return ancestor; + } + + /** + * Implementation of bulk remove methods. + */ + @SuppressWarnings("unchecked") + private boolean bulkRemove(Predicate filter) { + boolean removed = false; + CustomLinkedBlockingQueue.Node p = null, ancestor = head; + CustomLinkedBlockingQueue.Node[] nodes = null; + int n, len = 0; + do { + // 1. Extract batch of up to 64 elements while holding the lock. + fullyLock(); + try { + if (nodes == null) { // first batch; initialize + p = head.next; + for (CustomLinkedBlockingQueue.Node q = p; q != null; q = succ(q)) + if (q.item != null && ++len == 64) + break; + nodes = (CustomLinkedBlockingQueue.Node[]) new CustomLinkedBlockingQueue.Node[len]; + } + for (n = 0; p != null && n < len; p = succ(p)) + nodes[n++] = p; + } finally { + fullyUnlock(); + } + + // 2. Run the filter on the elements while lock is free. + long deathRow = 0L; // "bitset" of size 64 + for (int i = 0; i < n; i++) { + final E e; + if ((e = nodes[i].item) != null && filter.test(e)) + deathRow |= 1L << i; + } + + // 3. Remove any filtered elements while holding the lock. + if (deathRow != 0) { + fullyLock(); + try { + for (int i = 0; i < n; i++) { + final CustomLinkedBlockingQueue.Node q; + if ((deathRow & (1L << i)) != 0L + && (q = nodes[i]).item != null) { + ancestor = findPred(q, ancestor); + unlink(q, ancestor); + removed = true; + } + nodes[i] = null; // help GC + } + } finally { + fullyUnlock(); + } + } + } + while (n > 0 && p != null); + return removed; + } + + /** + * Saves this queue to a stream (that is, serializes it). + * + * @param s the stream + * @throws java.io.IOException if an I/O error occurs + * @serialData The capacity is emitted (int), followed by all of + * its elements (each an {@code Object}) in the proper order, + * followed by a null + */ + private void writeObject(java.io.ObjectOutputStream s) + throws java.io.IOException { + + fullyLock(); + try { + // Write out any hidden stuff, plus capacity + s.defaultWriteObject(); + + // Write out all elements in the proper order. + for (CustomLinkedBlockingQueue.Node p = head.next; p != null; p = p.next) + s.writeObject(p.item); + + // Use trailing null as sentinel + s.writeObject(null); + } finally { + fullyUnlock(); + } + } + + /** + * Reconstitutes this queue from a stream (that is, deserializes it). + * + * @param s the stream + * @throws ClassNotFoundException if the class of a serialized object + * could not be found + * @throws java.io.IOException if an I/O error occurs + */ + private void readObject(java.io.ObjectInputStream s) + throws java.io.IOException, ClassNotFoundException { + // Read in capacity, and any hidden stuff + s.defaultReadObject(); + + count.set(0); + last = head = new CustomLinkedBlockingQueue.Node(null); + + // Read in all elements and place in queue + for (; ; ) { + @SuppressWarnings("unchecked") + E item = (E) s.readObject(); + if (item == null) + break; + add(item); + } + } +} diff --git a/java/client/src/main/java/org/apache/rocketmq/client/java/misc/ExcludeFromJacocoGeneratedReport.java b/java/client/src/main/java/org/apache/rocketmq/client/java/misc/ExcludeFromJacocoGeneratedReport.java index 5f8c142ef..578fbb099 100644 --- a/java/client/src/main/java/org/apache/rocketmq/client/java/misc/ExcludeFromJacocoGeneratedReport.java +++ b/java/client/src/main/java/org/apache/rocketmq/client/java/misc/ExcludeFromJacocoGeneratedReport.java @@ -30,6 +30,6 @@ * set to METHOD, indicating that it can only be applied to methods. */ @Retention(RetentionPolicy.RUNTIME) -@Target(ElementType.METHOD) +@Target({ElementType.METHOD, ElementType.TYPE}) public @interface ExcludeFromJacocoGeneratedReport { } diff --git a/java/client/src/main/java/org/apache/rocketmq/client/java/misc/Utilities.java b/java/client/src/main/java/org/apache/rocketmq/client/java/misc/Utilities.java index 35916a5b6..7a25cc8ad 100644 --- a/java/client/src/main/java/org/apache/rocketmq/client/java/misc/Utilities.java +++ b/java/client/src/main/java/org/apache/rocketmq/client/java/misc/Utilities.java @@ -298,9 +298,21 @@ public static String getOsVersion() { } } + public static String getJavaRuntimeName() { + return System.getProperty("java.runtime.name"); + } + + public static String getJavaRuntimeVersion() { + return System.getProperty("java.runtime.version"); + } + public static String getJavaDescription() { return System.getProperty("java.vm.vendor") + " " + System.getProperty("java.vm.name") + " " + System.getProperty("java.vm.version"); } + + public static String getJavaEnvironmentSummary() { + return getJavaRuntimeName() + "/" + getJavaRuntimeVersion() + "/" + getJavaDescription(); + } } diff --git a/java/client/src/main/java/org/apache/rocketmq/client/java/route/MessageQueueImpl.java b/java/client/src/main/java/org/apache/rocketmq/client/java/route/MessageQueueImpl.java index de36c5efd..b6e63a28d 100644 --- a/java/client/src/main/java/org/apache/rocketmq/client/java/route/MessageQueueImpl.java +++ b/java/client/src/main/java/org/apache/rocketmq/client/java/route/MessageQueueImpl.java @@ -21,10 +21,11 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import org.apache.rocketmq.client.apis.message.MessageQueue; import org.apache.rocketmq.client.java.message.MessageType; import org.apache.rocketmq.client.java.message.protocol.Resource; -public class MessageQueueImpl { +public class MessageQueueImpl implements MessageQueue { private final Resource topicResource; private final Broker broker; private final int queueId; diff --git a/java/client/src/main/java/org/apache/rocketmq/client/java/rpc/RpcClient.java b/java/client/src/main/java/org/apache/rocketmq/client/java/rpc/RpcClient.java index 4c5e8b76b..0727a22e8 100644 --- a/java/client/src/main/java/org/apache/rocketmq/client/java/rpc/RpcClient.java +++ b/java/client/src/main/java/org/apache/rocketmq/client/java/rpc/RpcClient.java @@ -25,12 +25,18 @@ import apache.rocketmq.v2.EndTransactionResponse; import apache.rocketmq.v2.ForwardMessageToDeadLetterQueueRequest; import apache.rocketmq.v2.ForwardMessageToDeadLetterQueueResponse; +import apache.rocketmq.v2.GetOffsetRequest; +import apache.rocketmq.v2.GetOffsetResponse; import apache.rocketmq.v2.HeartbeatRequest; import apache.rocketmq.v2.HeartbeatResponse; import apache.rocketmq.v2.NotifyClientTerminationRequest; import apache.rocketmq.v2.NotifyClientTerminationResponse; +import apache.rocketmq.v2.PullMessageRequest; +import apache.rocketmq.v2.PullMessageResponse; import apache.rocketmq.v2.QueryAssignmentRequest; import apache.rocketmq.v2.QueryAssignmentResponse; +import apache.rocketmq.v2.QueryOffsetRequest; +import apache.rocketmq.v2.QueryOffsetResponse; import apache.rocketmq.v2.QueryRouteRequest; import apache.rocketmq.v2.QueryRouteResponse; import apache.rocketmq.v2.ReceiveMessageRequest; @@ -38,6 +44,8 @@ import apache.rocketmq.v2.SendMessageRequest; import apache.rocketmq.v2.SendMessageResponse; import apache.rocketmq.v2.TelemetryCommand; +import apache.rocketmq.v2.UpdateOffsetRequest; +import apache.rocketmq.v2.UpdateOffsetResponse; import com.google.common.util.concurrent.ListenableFuture; import io.grpc.Metadata; import io.grpc.stub.StreamObserver; @@ -162,6 +170,18 @@ ListenableFuture changeInvisibleDuration(Metada ListenableFuture forwardMessageToDeadLetterQueue( Metadata metadata, ForwardMessageToDeadLetterQueueRequest request, Executor executor, Duration duration); + ListenableFuture> pullMessage(Metadata metadata, PullMessageRequest request, + Executor executor, Duration duration); + + ListenableFuture updateOffset(Metadata metadata, UpdateOffsetRequest request, + Executor executor, Duration duration); + + ListenableFuture getOffset(Metadata metadata, GetOffsetRequest request, Executor executor, + Duration duration); + + ListenableFuture queryOffset(Metadata metadata, QueryOffsetRequest request, Executor executor, + Duration duration); + /** * Submit transaction resolution asynchronously. * diff --git a/java/client/src/main/java/org/apache/rocketmq/client/java/rpc/RpcClientImpl.java b/java/client/src/main/java/org/apache/rocketmq/client/java/rpc/RpcClientImpl.java index a40cd3989..2e60584f0 100644 --- a/java/client/src/main/java/org/apache/rocketmq/client/java/rpc/RpcClientImpl.java +++ b/java/client/src/main/java/org/apache/rocketmq/client/java/rpc/RpcClientImpl.java @@ -25,13 +25,19 @@ import apache.rocketmq.v2.EndTransactionResponse; import apache.rocketmq.v2.ForwardMessageToDeadLetterQueueRequest; import apache.rocketmq.v2.ForwardMessageToDeadLetterQueueResponse; +import apache.rocketmq.v2.GetOffsetRequest; +import apache.rocketmq.v2.GetOffsetResponse; import apache.rocketmq.v2.HeartbeatRequest; import apache.rocketmq.v2.HeartbeatResponse; import apache.rocketmq.v2.MessagingServiceGrpc; import apache.rocketmq.v2.NotifyClientTerminationRequest; import apache.rocketmq.v2.NotifyClientTerminationResponse; +import apache.rocketmq.v2.PullMessageRequest; +import apache.rocketmq.v2.PullMessageResponse; import apache.rocketmq.v2.QueryAssignmentRequest; import apache.rocketmq.v2.QueryAssignmentResponse; +import apache.rocketmq.v2.QueryOffsetRequest; +import apache.rocketmq.v2.QueryOffsetResponse; import apache.rocketmq.v2.QueryRouteRequest; import apache.rocketmq.v2.QueryRouteResponse; import apache.rocketmq.v2.ReceiveMessageRequest; @@ -39,6 +45,8 @@ import apache.rocketmq.v2.SendMessageRequest; import apache.rocketmq.v2.SendMessageResponse; import apache.rocketmq.v2.TelemetryCommand; +import apache.rocketmq.v2.UpdateOffsetRequest; +import apache.rocketmq.v2.UpdateOffsetResponse; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.SettableFuture; import io.grpc.ClientInterceptor; @@ -196,6 +204,57 @@ public ListenableFuture forwardMessageT .withDeadlineAfter(duration.toNanos(), TimeUnit.NANOSECONDS).forwardMessageToDeadLetterQueue(request); } + @Override + public ListenableFuture> pullMessage(Metadata metadata, PullMessageRequest request, + Executor executor, Duration duration) { + this.activityNanoTime = System.nanoTime(); + final SettableFuture> future = SettableFuture.create(); + List responses = new ArrayList<>(); + stub.withInterceptors(MetadataUtils.newAttachHeadersInterceptor(metadata)).withExecutor(executor) + .withDeadlineAfter(duration.toNanos(), TimeUnit.NANOSECONDS) + .pullMessage(request, new StreamObserver() { + @Override + public void onNext(PullMessageResponse response) { + responses.add(response); + } + + @Override + public void onError(Throwable t) { + future.setException(t); + } + + @Override + public void onCompleted() { + future.set(responses); + } + }); + return future; + } + + @Override + public ListenableFuture updateOffset(Metadata metadata, UpdateOffsetRequest request, + Executor executor, Duration duration) { + this.activityNanoTime = System.nanoTime(); + return futureStub.withInterceptors(MetadataUtils.newAttachHeadersInterceptor(metadata)).withExecutor(executor) + .withDeadlineAfter(duration.toNanos(), TimeUnit.NANOSECONDS).updateOffset(request); + } + + @Override + public ListenableFuture getOffset(Metadata metadata, GetOffsetRequest request, Executor executor, + Duration duration) { + this.activityNanoTime = System.nanoTime(); + return futureStub.withInterceptors(MetadataUtils.newAttachHeadersInterceptor(metadata)).withExecutor(executor) + .withDeadlineAfter(duration.toNanos(), TimeUnit.NANOSECONDS).getOffset(request); + } + + @Override + public ListenableFuture queryOffset(Metadata metadata, QueryOffsetRequest request, + Executor executor, Duration duration) { + this.activityNanoTime = System.nanoTime(); + return futureStub.withInterceptors(MetadataUtils.newAttachHeadersInterceptor(metadata)).withExecutor(executor) + .withDeadlineAfter(duration.toNanos(), TimeUnit.NANOSECONDS).queryOffset(request); + } + @Override public ListenableFuture endTransaction(Metadata metadata, EndTransactionRequest request, Executor executor, Duration duration) { diff --git a/java/client/src/test/java/org/apache/rocketmq/client/java/impl/ClientManagerImplTest.java b/java/client/src/test/java/org/apache/rocketmq/client/java/impl/ClientManagerImplTest.java index 46d82339a..bcfb2ff59 100644 --- a/java/client/src/test/java/org/apache/rocketmq/client/java/impl/ClientManagerImplTest.java +++ b/java/client/src/test/java/org/apache/rocketmq/client/java/impl/ClientManagerImplTest.java @@ -21,12 +21,15 @@ import apache.rocketmq.v2.ChangeInvisibleDurationRequest; import apache.rocketmq.v2.EndTransactionRequest; import apache.rocketmq.v2.ForwardMessageToDeadLetterQueueRequest; +import apache.rocketmq.v2.GetOffsetRequest; import apache.rocketmq.v2.HeartbeatRequest; import apache.rocketmq.v2.NotifyClientTerminationRequest; +import apache.rocketmq.v2.PullMessageRequest; import apache.rocketmq.v2.QueryAssignmentRequest; import apache.rocketmq.v2.QueryRouteRequest; import apache.rocketmq.v2.ReceiveMessageRequest; import apache.rocketmq.v2.SendMessageRequest; +import apache.rocketmq.v2.UpdateOffsetRequest; import io.grpc.Metadata; import java.time.Duration; import org.apache.rocketmq.client.java.misc.ClientId; @@ -120,6 +123,30 @@ public void testForwardMessageToDeadLetterQueue() { // Expect no exception thrown. } + @Test + public void testPullMessage() { + PullMessageRequest request = PullMessageRequest.newBuilder().build(); + CLIENT_MANAGER.pullMessage(fakeEndpoints(), request, Duration.ofSeconds(1)); + CLIENT_MANAGER.pullMessage(null, request, Duration.ofSeconds(1)); + // Expect no exception thrown. + } + + @Test + public void testUpdateOffset() { + UpdateOffsetRequest request = UpdateOffsetRequest.newBuilder().build(); + CLIENT_MANAGER.updateOffset(fakeEndpoints(), request, Duration.ofSeconds(1)); + CLIENT_MANAGER.updateOffset(null, request, Duration.ofSeconds(1)); + // Expect no exception thrown. + } + + @Test + public void testGetOffset() { + GetOffsetRequest request = GetOffsetRequest.newBuilder().build(); + CLIENT_MANAGER.getOffset(fakeEndpoints(), request, Duration.ofSeconds(1)); + CLIENT_MANAGER.getOffset(null, request, Duration.ofSeconds(1)); + // Expect no exception thrown. + } + @Test public void testEndTransaction() { EndTransactionRequest request = EndTransactionRequest.newBuilder().build(); diff --git a/java/client/src/test/java/org/apache/rocketmq/client/java/impl/consumer/ConsumeServiceTest.java b/java/client/src/test/java/org/apache/rocketmq/client/java/impl/consumer/ConsumeServiceTest.java index dbf8c89b2..3c9d44554 100644 --- a/java/client/src/test/java/org/apache/rocketmq/client/java/impl/consumer/ConsumeServiceTest.java +++ b/java/client/src/test/java/org/apache/rocketmq/client/java/impl/consumer/ConsumeServiceTest.java @@ -54,7 +54,7 @@ public void testConsumeSuccess() throws ExecutionException, InterruptedException final ConsumeService consumeService = new ConsumeService(clientId, messageListener, consumptionExecutor, interceptor, scheduler) { @Override - public void consume(ProcessQueue pq, List messageViews) { + public void consume(PushProcessQueue pq, List messageViews) { } }; final MessageViewImpl messageView = fakeMessageViewImpl(); @@ -69,7 +69,7 @@ public void testConsumeFailure() throws ExecutionException, InterruptedException final ConsumeService consumeService = new ConsumeService(clientId, messageListener, consumptionExecutor, interceptor, scheduler) { @Override - public void consume(ProcessQueue pq, List messageViews) { + public void consume(PushProcessQueue pq, List messageViews) { } }; final MessageViewImpl messageView = fakeMessageViewImpl(); @@ -86,7 +86,7 @@ public void testConsumeWithException() throws ExecutionException, InterruptedExc final ConsumeService consumeService = new ConsumeService(clientId, messageListener, consumptionExecutor, interceptor, scheduler) { @Override - public void consume(ProcessQueue pq, List messageViews) { + public void consume(PushProcessQueue pq, List messageViews) { } }; @@ -103,7 +103,7 @@ public void testConsumeWithDelay() throws ExecutionException, InterruptedExcepti consumptionExecutor, interceptor, scheduler) { @Override - public void consume(ProcessQueue pq, List messageViews) { + public void consume(PushProcessQueue pq, List messageViews) { } }; diff --git a/java/client/src/test/java/org/apache/rocketmq/client/java/impl/consumer/PullConsumerImplTest.java b/java/client/src/test/java/org/apache/rocketmq/client/java/impl/consumer/PullConsumerImplTest.java new file mode 100644 index 000000000..0821c1e10 --- /dev/null +++ b/java/client/src/test/java/org/apache/rocketmq/client/java/impl/consumer/PullConsumerImplTest.java @@ -0,0 +1,318 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.client.java.impl.consumer; + +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +import apache.rocketmq.v2.UpdateOffsetRequest; +import apache.rocketmq.v2.UpdateOffsetResponse; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import org.apache.rocketmq.client.apis.ClientConfiguration; +import org.apache.rocketmq.client.apis.ClientException; +import org.apache.rocketmq.client.apis.consumer.FilterExpression; +import org.apache.rocketmq.client.apis.consumer.TopicMessageQueueChangeListener; +import org.apache.rocketmq.client.apis.message.MessageQueue; +import org.apache.rocketmq.client.java.message.MessageViewImpl; +import org.apache.rocketmq.client.java.route.MessageQueueImpl; +import org.apache.rocketmq.client.java.route.TopicRouteData; +import org.apache.rocketmq.client.java.rpc.RpcFuture; +import org.apache.rocketmq.client.java.tool.TestBase; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.junit.MockitoJUnitRunner; + +@RunWith(MockitoJUnitRunner.class) +public class PullConsumerImplTest extends TestBase { + + private final ClientConfiguration clientConfiguration = ClientConfiguration.newBuilder() + .setEndpoints(FAKE_ENDPOINTS).build(); + + private PullConsumerImpl pullConsumer; + + @Before + public void init() { + pullConsumer = spy(new PullConsumerImpl(clientConfiguration, FAKE_CONSUMER_GROUP_0, false, + Duration.ofSeconds(3), Integer.MAX_VALUE, Integer.MAX_VALUE, Integer.MAX_VALUE, Integer.MAX_VALUE)); + } + + @Test + public void testGetSubscription() { + final Map map = pullConsumer.getSubscriptions(); + assertTrue(map.isEmpty()); + } + + @Test(expected = IllegalStateException.class) + public void testAssignBeforeStartup() { + final MessageQueue mq = fakeMessageQueueImpl0(); + List list = new ArrayList<>(); + list.add(mq); + pullConsumer.assign(list); + } + + @Test + public void testAssign() { + doReturn(true).when(pullConsumer).isRunning(); + final MessageQueue mq = fakeMessageQueueImpl0(); + List list = new ArrayList<>(); + list.add(mq); + pullConsumer.assign(list); + } + + @Test(expected = IllegalStateException.class) + public void testRegisterMessageQueueChangeListenerByTopicBeforeStartup() throws ClientException { + pullConsumer.registerMessageQueueChangeListenerByTopic("abc", (topic, messageQueues) -> { + }); + } + + @Test(expected = IllegalStateException.class) + public void testPollBeforeStartup() throws InterruptedException { + pullConsumer.poll(Duration.ofSeconds(3)); + } + + @Test(expected = IllegalStateException.class) + public void testSeekBeforeStartup() { + pullConsumer.seek(fakeMessageQueueImpl0(), 0); + } + + @Test(expected = IllegalStateException.class) + public void testPauseBeforeStartup() { + final ArrayList list = new ArrayList<>(); + list.add(fakeMessageQueueImpl0()); + pullConsumer.pause(list); + } + + @Test(expected = IllegalStateException.class) + public void testResumeBeforeStartup() { + final ArrayList list = new ArrayList<>(); + list.add(fakeMessageQueueImpl0()); + pullConsumer.resume(list); + } + + @Test(expected = IllegalStateException.class) + public void testOffsetForTimestampBeforeStartup() throws ClientException { + final MessageQueueImpl mq = fakeMessageQueueImpl0(); + pullConsumer.offsetForTimestamp(mq, System.currentTimeMillis()); + } + + @Test(expected = IllegalStateException.class) + public void testCommittedBeforeStartup() throws ClientException { + final MessageQueueImpl mq = fakeMessageQueueImpl0(); + pullConsumer.committed(mq); + } + + @Test(expected = IllegalStateException.class) + public void testCommitBeforeStartup() throws ClientException { + pullConsumer.commit(); + } + + @Test(expected = IllegalStateException.class) + public void testSeekToBeginBeforeStartup() throws ClientException { + pullConsumer.seekToBegin(fakeMessageQueueImpl0()); + } + + @Test(expected = IllegalStateException.class) + public void testSeekToEndBeforeStartup() throws ClientException { + pullConsumer.seekToEnd(fakeMessageQueueImpl0()); + } + + @Test + public void testCacheMessages() { + PullConsumerImpl pullConsumer = new PullConsumerImpl(clientConfiguration, FAKE_CONSUMER_GROUP_0, + false, Duration.ofSeconds(3), Integer.MAX_VALUE, Integer.MAX_VALUE, Integer.MAX_VALUE, Integer.MAX_VALUE); + final MessageQueueImpl messageQueueImpl = fakeMessageQueueImpl(FAKE_TOPIC_0); + final PullProcessQueueImpl pq = new PullProcessQueueImpl(pullConsumer, messageQueueImpl, + FilterExpression.SUB_ALL); + final MessageViewImpl messageView = fakeMessageViewImpl(messageQueueImpl); + List messageViewList = new ArrayList<>(); + messageViewList.add(messageView); + pullConsumer.cacheMessages(pq, messageViewList); + final List list = pullConsumer.peekCachedMessages(pq); + Assert.assertEquals(list.size(), 1); + } + + @Test + public void testGetConsumerGroup() { + PullConsumerImpl pullConsumer = new PullConsumerImpl(clientConfiguration, FAKE_CONSUMER_GROUP_0, + false, Duration.ofSeconds(3), Integer.MAX_VALUE, Integer.MAX_VALUE, Integer.MAX_VALUE, Integer.MAX_VALUE); + Assert.assertEquals(pullConsumer.getConsumerGroup(), FAKE_CONSUMER_GROUP_0); + } + + @Test + public void testGetMaxCacheMessageCountEachQueue() { + int maxCacheMessageCountTotalQueue = 128; + int maxCacheMessageCountEachQueue = 64; + PullConsumerImpl consumer = new PullConsumerImpl(clientConfiguration, FAKE_CONSUMER_GROUP_0, false, + Duration.ofSeconds(3), + maxCacheMessageCountTotalQueue, maxCacheMessageCountEachQueue, Integer.MAX_VALUE, Integer.MAX_VALUE); + Assert.assertEquals(maxCacheMessageCountEachQueue, consumer.getMaxCacheMessageCountEachQueue()); + + maxCacheMessageCountTotalQueue = 256; + maxCacheMessageCountEachQueue = 512; + consumer = new PullConsumerImpl(clientConfiguration, FAKE_CONSUMER_GROUP_0, false, Duration.ofSeconds(3), + maxCacheMessageCountTotalQueue, maxCacheMessageCountEachQueue, Integer.MAX_VALUE, Integer.MAX_VALUE); + Assert.assertEquals(maxCacheMessageCountTotalQueue, consumer.getMaxCacheMessageCountEachQueue()); + } + + @Test + public void testGetMaxCacheMessageSizeInBytesEachQueue() { + int maxCacheMessageSizeInBytesTotalQueue = 128; + int maxCacheMessageSizeInBytesEachQueue = 64; + PullConsumerImpl consumer = new PullConsumerImpl(clientConfiguration, FAKE_CONSUMER_GROUP_0, false, + Duration.ofSeconds(3), Integer.MAX_VALUE, Integer.MAX_VALUE, maxCacheMessageSizeInBytesTotalQueue, + maxCacheMessageSizeInBytesEachQueue); + Assert.assertEquals(maxCacheMessageSizeInBytesEachQueue, consumer.getMaxCacheMessageSizeInBytesEachQueue()); + + maxCacheMessageSizeInBytesTotalQueue = 256; + maxCacheMessageSizeInBytesEachQueue = 512; + consumer = new PullConsumerImpl(clientConfiguration, FAKE_CONSUMER_GROUP_0, false, Duration.ofSeconds(3), + Integer.MAX_VALUE, Integer.MAX_VALUE, maxCacheMessageSizeInBytesTotalQueue, + maxCacheMessageSizeInBytesEachQueue); + Assert.assertEquals(maxCacheMessageSizeInBytesTotalQueue, consumer.getMaxCacheMessageSizeInBytesEachQueue()); + } + + @Test + public void testCreateProcessQueue() { + PullConsumerImpl pullConsumer = new PullConsumerImpl(clientConfiguration, FAKE_CONSUMER_GROUP_0, + false, Duration.ofSeconds(3), Integer.MAX_VALUE, Integer.MAX_VALUE, Integer.MAX_VALUE, Integer.MAX_VALUE); + final MessageQueueImpl mq = fakeMessageQueueImpl(FAKE_TOPIC_0); + Optional pq = pullConsumer.createProcessQueue(mq, FilterExpression.SUB_ALL); + Assert.assertTrue(pq.isPresent()); + + pq = pullConsumer.createProcessQueue(mq, FilterExpression.SUB_ALL); + Assert.assertFalse(pq.isPresent()); + + pullConsumer.dropProcessQueue(mq); + pq = pullConsumer.createProcessQueue(mq, FilterExpression.SUB_ALL); + Assert.assertTrue(pq.isPresent()); + } + + @Test + public void testCreateProcessQueueWithOffset() { + PullConsumerImpl pullConsumer = new PullConsumerImpl(clientConfiguration, FAKE_CONSUMER_GROUP_0, + false, Duration.ofSeconds(3), Integer.MAX_VALUE, Integer.MAX_VALUE, Integer.MAX_VALUE, Integer.MAX_VALUE); + final MessageQueueImpl mq = fakeMessageQueueImpl(FAKE_TOPIC_0); + Optional pq = pullConsumer.createProcessQueue(mq, FilterExpression.SUB_ALL, 0); + Assert.assertTrue(pq.isPresent()); + + pq = pullConsumer.createProcessQueue(mq, FilterExpression.SUB_ALL, 0); + Assert.assertFalse(pq.isPresent()); + + pullConsumer.dropProcessQueue(mq); + pq = pullConsumer.createProcessQueue(mq, FilterExpression.SUB_ALL, 0); + Assert.assertTrue(pq.isPresent()); + } + + @Test + public void testOnTopicRouteDataUpdateWithNullListener() { + List pbMessageQueues = new ArrayList<>(); + pbMessageQueues.add(fakePbMessageQueue0()); + final TopicRouteData topicRouteData = new TopicRouteData(pbMessageQueues); + pullConsumer.onTopicRouteDataUpdate0(FAKE_TOPIC_0, topicRouteData); + } + + @Test + public void testOnTopicRouteDataUpdate() throws ClientException { + doReturn(true).when(pullConsumer).isRunning(); + final MessageQueueImpl mq = fakeMessageQueueImpl(FAKE_TOPIC_0); + List mqs = new ArrayList<>(); + mqs.add(mq); + doReturn(mqs).when(pullConsumer).fetchMessageQueues(anyString()); + TopicMessageQueueChangeListener listener = new TopicMessageQueueChangeListener() { + @Override + public void onChanged(String topic, Set messageQueues) { + + } + }; + listener = spy(listener); + pullConsumer.registerMessageQueueChangeListenerByTopic(FAKE_TOPIC_0, listener); + verify(pullConsumer, times(1)).fetchMessageQueues(anyString()); + } + + @Test(expected = IllegalArgumentException.class) + public void testSeekWithMessageQueueIsNotContained() { + doReturn(true).when(pullConsumer).isRunning(); + final MessageQueueImpl mq = fakeMessageQueueImpl(FAKE_TOPIC_0); + pullConsumer.seek(mq, 1); + } + + @Test + public void testSeek() { + doReturn(true).when(pullConsumer).isRunning(); + MessageQueueImpl mq = fakeMessageQueueImpl(FAKE_TOPIC_0); + List mqs = new ArrayList<>(); + mqs.add(mq); + pullConsumer.assign(mqs); + pullConsumer.seek(mq, 1); + verify(pullConsumer, times(1)).dropProcessQueue(any(MessageQueueImpl.class)); + verify(pullConsumer, times(1)) + .tryPullMessageByMessageQueueImmediately(any(MessageQueueImpl.class), any(FilterExpression.class), + anyLong()); + } + + @Test + public void testSeekToBegin() throws ClientException { + doReturn(true).when(pullConsumer).isRunning(); + final MessageQueueImpl mq = fakeMessageQueueImpl(FAKE_TOPIC_0); + List mqs = new ArrayList<>(); + mqs.add(mq); + pullConsumer.assign(mqs); + doReturn(okQueryOffsetResponseFuture()).when(pullConsumer).queryOffset(any(MessageQueueImpl.class), + any(OffsetPolicy.class)); + pullConsumer.seekToBegin(mq); + verify(pullConsumer, times(1)).queryOffset(any(MessageQueueImpl.class), + any(OffsetPolicy.class)); + } + + @Test + public void testSeekToEnd() throws ClientException { + doReturn(true).when(pullConsumer).isRunning(); + final MessageQueueImpl mq = fakeMessageQueueImpl(FAKE_TOPIC_0); + List mqs = new ArrayList<>(); + mqs.add(mq); + pullConsumer.assign(mqs); + doReturn(okQueryOffsetResponseFuture()).when(pullConsumer).queryOffset(any(MessageQueueImpl.class), + any(OffsetPolicy.class)); + pullConsumer.seekToEnd(mq); + verify(pullConsumer, times(1)).queryOffset(any(MessageQueueImpl.class), + any(OffsetPolicy.class)); + } + + @Test + public void testCommit() throws ClientException { + List> futures = new ArrayList<>(); + final RpcFuture future = okUpdateOffsetResponseFuture(); + futures.add(future); + doReturn(futures).when(pullConsumer).commit0(); + pullConsumer.commit(); + verify(pullConsumer, times(1)).commit0(); + } +} diff --git a/java/client/src/test/java/org/apache/rocketmq/client/java/impl/consumer/PullProcessQueueImplTest.java b/java/client/src/test/java/org/apache/rocketmq/client/java/impl/consumer/PullProcessQueueImplTest.java new file mode 100644 index 000000000..836b10718 --- /dev/null +++ b/java/client/src/test/java/org/apache/rocketmq/client/java/impl/consumer/PullProcessQueueImplTest.java @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.client.java.impl.consumer; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +import apache.rocketmq.v2.PullMessageRequest; +import com.google.common.util.concurrent.Futures; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import org.apache.rocketmq.client.apis.ClientConfiguration; +import org.apache.rocketmq.client.apis.consumer.FilterExpression; +import org.apache.rocketmq.client.java.message.MessageViewImpl; +import org.apache.rocketmq.client.java.route.Endpoints; +import org.apache.rocketmq.client.java.route.MessageQueueImpl; +import org.apache.rocketmq.client.java.tool.TestBase; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.junit.MockitoJUnitRunner; + +@RunWith(MockitoJUnitRunner.class) +public class PullProcessQueueImplTest extends TestBase { + private final ClientConfiguration clientConfiguration = ClientConfiguration.newBuilder() + .setEndpoints(FAKE_ENDPOINTS).build(); + + private final int maxCacheMessageCountEachQueue = 128; + private final int maxCacheMessageSizeInBytesEachQueue = 128; + + private PullConsumerImpl spyConsumer; + private PullProcessQueueImpl spyPq; + + @Before + public void init() { + int maxCacheMessageSizeInBytesTotalQueue = 128; + int maxCacheMessageCountTotalQueue = 128; + spyConsumer = spy(new PullConsumerImpl(clientConfiguration, FAKE_CONSUMER_GROUP_0, false, + Duration.ofSeconds(3), maxCacheMessageCountTotalQueue, maxCacheMessageCountEachQueue, + maxCacheMessageSizeInBytesTotalQueue, maxCacheMessageSizeInBytesEachQueue)); + spyPq = spy(new PullProcessQueueImpl(spyConsumer, fakeMessageQueueImpl0(), FilterExpression.SUB_ALL)); + } + + @Test + public void testExpired() { + Assert.assertFalse(spyPq.expired()); + } + + @Test + public void testIsCacheFullWhenNoMessage() { + Assert.assertFalse(spyPq.isCacheFull()); + } + + @Test + public void testIsCacheFullWhenMessageCountExceeds() { + doReturn(1L + maxCacheMessageCountEachQueue).when(spyPq).getCachedMessageCount(); + Assert.assertTrue(spyPq.isCacheFull()); + } + + @Test + public void testIsCacheFullWhenMessageBytesExceeds() { + doReturn(1L + maxCacheMessageSizeInBytesEachQueue).when(spyPq).getCachedMessageBytes(); + Assert.assertTrue(spyPq.isCacheFull()); + } + + @Test + public void testPullMessageImmediatelyWhenConsumerIsNotRunning() { + doReturn(false).when(spyConsumer).isRunning(); + spyPq.pullMessageImmediately(); + verify(spyConsumer, never()).getScheduler(); + } + + @Test + public void testPullMessageImmediatelyWithOffsetNotFound() { + doReturn(true).when(spyConsumer).isRunning(); + doReturn(okGetOffsetResponseFuture()).when(spyConsumer).getOffset(any(MessageQueueImpl.class)); + doNothing().when(spyPq).pullMessageImmediately(anyLong()); + spyPq.pullMessageImmediately(); + verify(spyPq, times(1)).pullMessageImmediately(anyLong()); + } + + @Test + public void testPullMessageImmediatelyWithOffsetWhenConsumerIsNotRunning() { + doReturn(false).when(spyConsumer).isRunning(); + spyPq.pullMessageImmediately(1); + verify(spyPq, never()).isCacheFull(); + } + + @Test + public void testPullMessageImmediatelyWithOffsetWithFullCache() { + doReturn(true).when(spyConsumer).isRunning(); + doReturn(true).when(spyPq).isCacheFull(); + spyPq.pullMessageImmediately(1); + verify(spyConsumer, never()).getScheduler(); + } + + @Test + public void testPullMessageImmediatelyWithOffset() { + doReturn(true).when(spyConsumer).isRunning(); + + final Endpoints endpoints = fakeEndpoints(); + final MessageViewImpl messageView = fakeMessageViewImpl(fakeMessageQueueImpl0()); + List messageViewList = new ArrayList<>(); + messageViewList.add(messageView); + long nextOffset = 100; + final PullMessageResult pullMessageResult = new PullMessageResult(endpoints, messageViewList, nextOffset); + + doReturn(Futures.immediateFuture(pullMessageResult)).when(spyConsumer) + .pullMessage(any(PullMessageRequest.class), any(MessageQueueImpl.class), any(Duration.class)); + doNothing().when(spyPq).onPullMessageResult(any(PullMessageResult.class), anyLong()); + spyPq.pullMessageImmediately(1); + verify(spyPq, times(1)).onPullMessageResult(any(PullMessageResult.class), anyLong()); + } +} diff --git a/java/client/src/test/java/org/apache/rocketmq/client/java/impl/consumer/ProcessQueueImplTest.java b/java/client/src/test/java/org/apache/rocketmq/client/java/impl/consumer/PushProcessQueueImplTest.java similarity index 95% rename from java/client/src/test/java/org/apache/rocketmq/client/java/impl/consumer/ProcessQueueImplTest.java rename to java/client/src/test/java/org/apache/rocketmq/client/java/impl/consumer/PushProcessQueueImplTest.java index d9aa61fbe..a8db924dd 100644 --- a/java/client/src/test/java/org/apache/rocketmq/client/java/impl/consumer/ProcessQueueImplTest.java +++ b/java/client/src/test/java/org/apache/rocketmq/client/java/impl/consumer/PushProcessQueueImplTest.java @@ -61,7 +61,7 @@ import org.mockito.junit.MockitoJUnitRunner; @RunWith(MockitoJUnitRunner.class) -public class ProcessQueueImplTest extends TestBase { +public class PushProcessQueueImplTest extends TestBase { @Mock private PushConsumerImpl pushConsumer; @Mock @@ -75,11 +75,11 @@ public class ProcessQueueImplTest extends TestBase { private final FilterExpression filterExpression = FilterExpression.SUB_ALL; - private ProcessQueueImpl processQueue; + private PushProcessQueueImpl processQueue; @Before public void setup() throws IllegalAccessException, NoSuchFieldException { - this.processQueue = new ProcessQueueImpl(pushConsumer, fakeMessageQueueImpl0(), filterExpression); + this.processQueue = new PushProcessQueueImpl(pushConsumer, fakeMessageQueueImpl0(), filterExpression); when(pushConsumer.isRunning()).thenReturn(true); this.consumptionOkQuantity = new AtomicLong(0); @@ -136,7 +136,7 @@ public void testReceiveMessageImmediately() { ReceiveMessageRequest request = ReceiveMessageRequest.newBuilder().build(); when(pushConsumer.wrapReceiveMessageRequest(anyInt(), any(MessageQueueImpl.class), any(FilterExpression.class), any(Duration.class), nullable(String.class))).thenReturn(request); - processQueue.fetchMessageImmediately(); + processQueue.receiveMessageImmediately(); await().atMost(Duration.ofSeconds(3)) .untilAsserted(() -> verify(pushConsumer, times(cachedMessagesCountThresholdPerQueue)) .receiveMessage(any(ReceiveMessageRequest.class), any(MessageQueueImpl.class), any(Duration.class))); @@ -166,7 +166,7 @@ public void testEraseMessageWithAckFailure() { processQueue.eraseMessage(messageView, ConsumeResult.SUCCESS); int ackTimes = 3; final Duration tolerance = Duration.ofMillis(500); - await().atMost(ProcessQueueImpl.ACK_MESSAGE_FAILURE_BACKOFF_DELAY.multipliedBy(ackTimes) + await().atMost(PushProcessQueueImpl.ACK_MESSAGE_FAILURE_BACKOFF_DELAY.multipliedBy(ackTimes) .plus(tolerance)).untilAsserted(() -> verify(pushConsumer, times(ackTimes)) .ackMessage(eq(messageView))); } @@ -185,7 +185,7 @@ public void testEraseMessageWithChangingInvisibleDurationFailure() { processQueue.eraseMessage(messageView, ConsumeResult.FAILURE); int ackTimes = 3; final Duration tolerance = Duration.ofMillis(500); - await().atMost(ProcessQueueImpl.CHANGE_INVISIBLE_DURATION_FAILURE_BACKOFF_DELAY.multipliedBy(ackTimes) + await().atMost(PushProcessQueueImpl.CHANGE_INVISIBLE_DURATION_FAILURE_BACKOFF_DELAY.multipliedBy(ackTimes) .plus(tolerance)).untilAsserted(() -> verify(pushConsumer, times(ackTimes)) .changeInvisibleDuration(any(MessageViewImpl.class), any(Duration.class))); } @@ -258,7 +258,7 @@ public void testEraseFifoMessageWithForwardingMessageToDeadLetterQueueFailure() processQueue.eraseFifoMessage(messageView, ConsumeResult.FAILURE); int forwardingToDeadLetterQueueTimes = 3; final Duration tolerance = Duration.ofMillis(500); - await().atMost(ProcessQueueImpl.FORWARD_FIFO_MESSAGE_TO_DLQ_FAILURE_BACKOFF_DELAY + await().atMost(PushProcessQueueImpl.FORWARD_FIFO_MESSAGE_TO_DLQ_FAILURE_BACKOFF_DELAY .multipliedBy(forwardingToDeadLetterQueueTimes).plus(tolerance)) .untilAsserted(() -> verify(pushConsumer, times(forwardingToDeadLetterQueueTimes)) .forwardMessageToDeadLetterQueue(any(MessageViewImpl.class))); @@ -279,7 +279,7 @@ public void testEraseFifoMessageWithForwardingMessageToDeadLetterQueueException( processQueue.eraseFifoMessage(messageView, ConsumeResult.FAILURE); int forwardingToDeadLetterQueueTimes = 3; final Duration tolerance = Duration.ofMillis(500); - await().atMost(ProcessQueueImpl.FORWARD_FIFO_MESSAGE_TO_DLQ_FAILURE_BACKOFF_DELAY + await().atMost(PushProcessQueueImpl.FORWARD_FIFO_MESSAGE_TO_DLQ_FAILURE_BACKOFF_DELAY .multipliedBy(forwardingToDeadLetterQueueTimes).plus(tolerance)) .untilAsserted(() -> verify(pushConsumer, times(forwardingToDeadLetterQueueTimes)) .forwardMessageToDeadLetterQueue(any(MessageViewImpl.class))); diff --git a/java/client/src/test/java/org/apache/rocketmq/client/java/misc/CacheBlockingListQueueTest.java b/java/client/src/test/java/org/apache/rocketmq/client/java/misc/CacheBlockingListQueueTest.java new file mode 100644 index 000000000..4770a3279 --- /dev/null +++ b/java/client/src/test/java/org/apache/rocketmq/client/java/misc/CacheBlockingListQueueTest.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.client.java.misc; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import org.apache.commons.lang3.tuple.Pair; +import org.junit.Test; + +public class CacheBlockingListQueueTest { + @Test + public void testCache() { + CacheBlockingListQueue cacheQueue = new CacheBlockingListQueue<>(); + + List data = new ArrayList<>(); + for (int i = 1; i <= 5; i++) { + data.add(i); + } + + cacheQueue.cache("A", data); + assertEquals(1, cacheQueue.size()); + } + + @Test + public void testPoll() throws InterruptedException { + CacheBlockingListQueue cacheQueue = new CacheBlockingListQueue<>(); + + List data = new ArrayList<>(); + for (int i = 1; i <= 5; i++) { + data.add(i); + } + + cacheQueue.cache("A", data); + assertEquals(1, cacheQueue.size()); + + final Pair> pair = cacheQueue.poll(Duration.ofSeconds(1)); + assertEquals(data, pair.getValue()); + } + + @Test + public void testDrop() { + CacheBlockingListQueue cacheQueue = new CacheBlockingListQueue<>(); + + List data = new ArrayList<>(); + for (int i = 1; i <= 5; i++) { + data.add(i); + } + + cacheQueue.cache("A", data); + assertEquals(1, cacheQueue.size()); + + cacheQueue.drop("A"); + assertEquals(0, cacheQueue.size()); + assertNull(cacheQueue.poll()); // Poll after dropping should return null + } +} \ No newline at end of file diff --git a/java/client/src/test/java/org/apache/rocketmq/client/java/misc/UtilitiesTest.java b/java/client/src/test/java/org/apache/rocketmq/client/java/misc/UtilitiesTest.java index b59c377a7..76ddfccb4 100644 --- a/java/client/src/test/java/org/apache/rocketmq/client/java/misc/UtilitiesTest.java +++ b/java/client/src/test/java/org/apache/rocketmq/client/java/misc/UtilitiesTest.java @@ -62,9 +62,27 @@ public void testStackTrace() { assertTrue(stackTrace.length() > 0); } + @Test + public void testGetJavaRuntimeName() { + final String javaRuntimeName = Utilities.getJavaRuntimeName(); + assertNotNull(javaRuntimeName); + } + + @Test + public void testGetJavaRuntimeVersion() { + final String javaRuntimeVersion = Utilities.getJavaRuntimeVersion(); + assertNotNull(javaRuntimeVersion); + } + @Test public void testGetJavaDescription() { final String javaDescription = Utilities.getJavaDescription(); assertNotNull(javaDescription); } + + @Test + public void testGetJavaEnvironmentSummary() { + final String javaSummary = Utilities.getJavaEnvironmentSummary(); + assertNotNull(javaSummary); + } } \ No newline at end of file diff --git a/java/client/src/test/java/org/apache/rocketmq/client/java/tool/TestBase.java b/java/client/src/test/java/org/apache/rocketmq/client/java/tool/TestBase.java index ef6723c99..ef9948b0d 100644 --- a/java/client/src/test/java/org/apache/rocketmq/client/java/tool/TestBase.java +++ b/java/client/src/test/java/org/apache/rocketmq/client/java/tool/TestBase.java @@ -31,11 +31,17 @@ import apache.rocketmq.v2.EndTransactionResponse; import apache.rocketmq.v2.ForwardMessageToDeadLetterQueueRequest; import apache.rocketmq.v2.ForwardMessageToDeadLetterQueueResponse; +import apache.rocketmq.v2.GetOffsetRequest; +import apache.rocketmq.v2.GetOffsetResponse; import apache.rocketmq.v2.MessageQueue; import apache.rocketmq.v2.MessageType; import apache.rocketmq.v2.Permission; +import apache.rocketmq.v2.PullMessageRequest; +import apache.rocketmq.v2.PullMessageResponse; import apache.rocketmq.v2.QueryAssignmentRequest; import apache.rocketmq.v2.QueryAssignmentResponse; +import apache.rocketmq.v2.QueryOffsetRequest; +import apache.rocketmq.v2.QueryOffsetResponse; import apache.rocketmq.v2.ReceiveMessageRequest; import apache.rocketmq.v2.ReceiveMessageResponse; import apache.rocketmq.v2.Resource; @@ -44,6 +50,8 @@ import apache.rocketmq.v2.SendResultEntry; import apache.rocketmq.v2.Status; import apache.rocketmq.v2.SystemProperties; +import apache.rocketmq.v2.UpdateOffsetRequest; +import apache.rocketmq.v2.UpdateOffsetResponse; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.SettableFuture; @@ -293,6 +301,11 @@ ForwardMessageToDeadLetterQueueResponse> okForwardMessageToDeadLetterQueueRespon return new RpcFuture<>(fakeRpcContext(), null, Futures.immediateFuture(response)); } + protected RpcFuture okUpdateOffsetResponseFuture() { + final Status status = Status.newBuilder().setCode(Code.OK).build(); + final UpdateOffsetResponse response = UpdateOffsetResponse.newBuilder().setStatus(status).build(); + return new RpcFuture<>(fakeRpcContext(), null, Futures.immediateFuture(response)); + } protected RpcFuture forwardMessageToDeadLetterQueueResponseFuture(Code code) { @@ -360,6 +373,32 @@ protected RpcFuture> okRecei return new RpcFuture<>(fakeRpcContext(), null, Futures.immediateFuture(responses)); } + protected RpcFuture> okPullMessageResponsesFuture(String topic, + int messageCount) { + final Status status = Status.newBuilder().setCode(Code.OK).build(); + final apache.rocketmq.v2.Message message = fakePbMessage(topic); + List responses = new ArrayList<>(); + PullMessageResponse statusResponse = PullMessageResponse.newBuilder().setStatus(status).build(); + responses.add(statusResponse); + for (int i = 0; i < messageCount; i++) { + PullMessageResponse messageResponse = PullMessageResponse.newBuilder().setMessage(message).build(); + responses.add(messageResponse); + } + return new RpcFuture<>(fakeRpcContext(), null, Futures.immediateFuture(responses)); + } + + protected RpcFuture okGetOffsetResponseFuture() { + final Status status = Status.newBuilder().setCode(Code.OK).build(); + GetOffsetResponse response = GetOffsetResponse.newBuilder().setOffset(0).setStatus(status).build(); + return new RpcFuture<>(fakeRpcContext(), null, Futures.immediateFuture(response)); + } + + protected RpcFuture okQueryOffsetResponseFuture() { + final Status status = Status.newBuilder().setCode(Code.OK).build(); + QueryOffsetResponse response = QueryOffsetResponse.newBuilder().setOffset(0).setStatus(status).build(); + return new RpcFuture<>(fakeRpcContext(), null, Futures.immediateFuture(response)); + } + protected ListenableFuture okEndTransactionResponseFuture() { SettableFuture future = SettableFuture.create(); final Status status = Status.newBuilder().setCode(Code.OK).build(); diff --git a/java/client/src/test/resources/logback-test.xml b/java/client/src/test/resources/logback-test.xml new file mode 100644 index 000000000..0063fd510 --- /dev/null +++ b/java/client/src/test/resources/logback-test.xml @@ -0,0 +1,55 @@ + + + + + + + + %yellow(%d{yyy-MM-dd HH:mm:ss.SSS,GMT+8}) %highlight(%-5p) %boldWhite([%pid]) %magenta([%t]) %boldGreen([%logger{12}#%M:%L]) - %m%n + + UTF-8 + + + + true + + ${rocketmq.log.root:-${user.home}${file.separator}logs${file.separator}rocketmq}${file.separator}rocketmq-client-test.log + + + + ${rocketmq.log.root:-${user.home}${file.separator}logs${file.separator}rocketmq}${file.separator}other_days${file.separator}rocketmq-client-test-%i.log.gz + + 1 + ${rocketmq.log.file.maxIndex:-10} + + + 64MB + + + %d{yyy-MM-dd HH:mm:ss.SSS,GMT+8} %-5p [%pid] [%t] [%logger{12}#%M:%L] - %m%n + UTF-8 + + + + + + + + + + + + \ No newline at end of file diff --git a/java/pom.xml b/java/pom.xml index d39b70947..ebf49f665 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -14,7 +14,8 @@ limitations under the License. --> - + org.apache apache @@ -356,7 +357,7 @@ 11 - + diff --git a/java/style/checkstyle-suppressions.xml b/java/style/checkstyle-suppressions.xml new file mode 100644 index 000000000..988aea214 --- /dev/null +++ b/java/style/checkstyle-suppressions.xml @@ -0,0 +1,25 @@ + + + + + + + + \ No newline at end of file diff --git a/java/style/checkstyle.xml b/java/style/checkstyle.xml index d9bfef468..0eb3fedfa 100644 --- a/java/style/checkstyle.xml +++ b/java/style/checkstyle.xml @@ -24,6 +24,7 @@ found at https://google.github.io/styleguide/javaguide.html. Deviations have been made where desired. --> + @@ -39,11 +40,9 @@ - - - - - + + + diff --git a/java/style/spotbugs-suppressions.xml b/java/style/spotbugs-suppressions.xml index b45993fd8..c01a1a73c 100644 --- a/java/style/spotbugs-suppressions.xml +++ b/java/style/spotbugs-suppressions.xml @@ -30,17 +30,17 @@ - + - + - + @@ -67,4 +67,14 @@ + + + + + + + + + +