From 43744e7d7127fde89ddfb03197b3814bcaf84c33 Mon Sep 17 00:00:00 2001 From: chenzlalvin Date: Wed, 23 Jun 2021 17:31:25 +0800 Subject: [PATCH 1/5] [RIP-21] submodule common & client & remoting --- client/pom.xml | 17 + .../client/exception/MQBrokerException.java | 6 + .../client/exception/MQRedirectException.java | 37 ++ .../rocketmq/client/impl/MQAdminImpl.java | 180 +++++++-- .../rocketmq/client/impl/MQClientAPIImpl.java | 208 ++++++++++- .../consumer/DefaultLitePullConsumerImpl.java | 5 +- .../consumer/DefaultMQPullConsumerImpl.java | 3 + .../consumer/DefaultMQPushConsumerImpl.java | 8 +- .../client/impl/consumer/PullAPIWrapper.java | 347 ++++++++++++++++-- .../consumer/PullResultWithLogicalQueues.java | 96 +++++ .../client/impl/factory/MQClientInstance.java | 102 ++++- .../impl/producer/DefaultMQProducerImpl.java | 231 +++++++++++- .../client/latency/MQFaultStrategy.java | 2 +- .../rocketmq/client/producer/SendResult.java | 9 + .../producer/SendResultForLogicalQueue.java | 46 +++ .../ConsumeMessageOpenTracingHookImpl.java | 2 +- .../consumer/DefaultLitePullConsumerTest.java | 4 +- ...DefaultMQPullConsumerLogicalQueueTest.java | 248 +++++++++++++ .../consumer/DefaultMQPushConsumerTest.java | 57 ++- ...ConsumeMessageConcurrentlyServiceTest.java | 2 +- .../DefaultMQProducerLogicalQueueTest.java | 311 ++++++++++++++++ .../producer/DefaultMQProducerTest.java | 23 +- .../DefaultMQConsumerWithOpenTracingTest.java | 13 +- .../trace/DefaultMQConsumerWithTraceTest.java | 6 +- .../DefaultMQProducerWithOpenTracingTest.java | 14 +- .../trace/DefaultMQProducerWithTraceTest.java | 26 +- ...nsactionMQProducerWithOpenTracingTest.java | 16 +- .../TransactionMQProducerWithTraceTest.java | 40 +- common/pom.xml | 17 + .../apache/rocketmq/common/BrokerConfig.java | 11 +- .../apache/rocketmq/common/ConfigManager.java | 11 + .../org/apache/rocketmq/common/MixAll.java | 13 + .../apache/rocketmq/common/TopicConfig.java | 10 + .../apache/rocketmq/common/TopicQueueId.java | 54 +++ .../rocketmq/common/constant/LoggerName.java | 1 + .../GenericMapSuperclassDeserializer.java | 58 +++ .../rocketmq/common/message/MessageConst.java | 2 + .../rocketmq/common/message/MessageQueue.java | 6 + .../rocketmq/common/protocol/RequestCode.java | 12 + .../common/protocol/ResponseCode.java | 1 + .../common/protocol/body/ClusterInfo.java | 14 + ...essageQueueForLogicalQueueRequestBody.java | 50 +++ .../body/MigrateLogicalQueueBody.java | 42 +++ .../ReuseTopicLogicalQueueRequestBody.java | 59 +++ .../SealTopicLogicalQueueRequestBody.java | 49 +++ .../body/TopicConfigSerializeWrapper.java | 11 + ...teTopicLogicalQueueMappingRequestBody.java | 49 +++ .../DeleteTopicLogicalQueueRequestHeader.java | 37 ++ .../header/GetMaxOffsetRequestHeader.java | 18 + .../header/GetTopicConfigRequestHeader.java | 45 +++ ...TopicLogicalQueueMappingRequestHeader.java | 37 ++ .../namesrv/GetRouteInfoRequestHeader.java | 20 + .../protocol/route/LogicalQueueRouteData.java | 309 ++++++++++++++++ .../protocol/route/LogicalQueuesInfo.java | 87 +++++ .../route/LogicalQueuesInfoUnordered.java | 108 ++++++ .../route/MessageQueueRouteState.java | 26 ++ .../common/protocol/route/TopicRouteData.java | 47 ++- .../protocol/route/TopicRouteDataNameSrv.java | 64 ++++ .../common/sysflag/MessageSysFlag.java | 1 + .../GenericMapSuperclassDeserializerTest.java | 57 +++ .../protocol/route/TopicRouteDataTest.java | 11 +- .../protocol/RemotingSerializable.java | 10 +- 62 files changed, 3229 insertions(+), 177 deletions(-) create mode 100644 client/src/main/java/org/apache/rocketmq/client/exception/MQRedirectException.java create mode 100644 client/src/main/java/org/apache/rocketmq/client/impl/consumer/PullResultWithLogicalQueues.java create mode 100644 client/src/main/java/org/apache/rocketmq/client/producer/SendResultForLogicalQueue.java create mode 100644 client/src/test/java/org/apache/rocketmq/client/consumer/DefaultMQPullConsumerLogicalQueueTest.java create mode 100644 client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerLogicalQueueTest.java create mode 100644 common/src/main/java/org/apache/rocketmq/common/TopicQueueId.java create mode 100644 common/src/main/java/org/apache/rocketmq/common/fastjson/GenericMapSuperclassDeserializer.java create mode 100644 common/src/main/java/org/apache/rocketmq/common/protocol/body/CreateMessageQueueForLogicalQueueRequestBody.java create mode 100644 common/src/main/java/org/apache/rocketmq/common/protocol/body/MigrateLogicalQueueBody.java create mode 100644 common/src/main/java/org/apache/rocketmq/common/protocol/body/ReuseTopicLogicalQueueRequestBody.java create mode 100644 common/src/main/java/org/apache/rocketmq/common/protocol/body/SealTopicLogicalQueueRequestBody.java create mode 100644 common/src/main/java/org/apache/rocketmq/common/protocol/body/UpdateTopicLogicalQueueMappingRequestBody.java create mode 100644 common/src/main/java/org/apache/rocketmq/common/protocol/header/DeleteTopicLogicalQueueRequestHeader.java create mode 100644 common/src/main/java/org/apache/rocketmq/common/protocol/header/GetTopicConfigRequestHeader.java create mode 100644 common/src/main/java/org/apache/rocketmq/common/protocol/header/QueryTopicLogicalQueueMappingRequestHeader.java create mode 100644 common/src/main/java/org/apache/rocketmq/common/protocol/route/LogicalQueueRouteData.java create mode 100644 common/src/main/java/org/apache/rocketmq/common/protocol/route/LogicalQueuesInfo.java create mode 100644 common/src/main/java/org/apache/rocketmq/common/protocol/route/LogicalQueuesInfoUnordered.java create mode 100644 common/src/main/java/org/apache/rocketmq/common/protocol/route/MessageQueueRouteState.java create mode 100644 common/src/main/java/org/apache/rocketmq/common/protocol/route/TopicRouteDataNameSrv.java create mode 100644 common/src/test/java/org/apache/rocketmq/common/fastjson/GenericMapSuperclassDeserializerTest.java diff --git a/client/pom.xml b/client/pom.xml index 95ef4617dd1..53277e0f381 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -27,6 +27,19 @@ rocketmq-client rocketmq-client ${project.version} + + + + org.apache.maven.plugins + maven-compiler-plugin + + 6 + 6 + + + + + ${project.groupId} @@ -73,5 +86,9 @@ log4j-slf4j-impl test + + com.google.guava + guava + diff --git a/client/src/main/java/org/apache/rocketmq/client/exception/MQBrokerException.java b/client/src/main/java/org/apache/rocketmq/client/exception/MQBrokerException.java index f07a38b81f0..7870ff1931b 100644 --- a/client/src/main/java/org/apache/rocketmq/client/exception/MQBrokerException.java +++ b/client/src/main/java/org/apache/rocketmq/client/exception/MQBrokerException.java @@ -25,6 +25,12 @@ public class MQBrokerException extends Exception { private final String errorMessage; private final String brokerAddr; + MQBrokerException() { + this.responseCode = 0; + this.errorMessage = null; + this.brokerAddr = null; + } + public MQBrokerException(int responseCode, String errorMessage) { super(FAQUrl.attachDefaultURL("CODE: " + UtilAll.responseCode2String(responseCode) + " DESC: " + errorMessage)); diff --git a/client/src/main/java/org/apache/rocketmq/client/exception/MQRedirectException.java b/client/src/main/java/org/apache/rocketmq/client/exception/MQRedirectException.java new file mode 100644 index 00000000000..036466767a4 --- /dev/null +++ b/client/src/main/java/org/apache/rocketmq/client/exception/MQRedirectException.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.client.exception; + +public class MQRedirectException extends MQBrokerException { + private static final StackTraceElement[] UNASSIGNED_STACK = new StackTraceElement[0]; + + private final byte[] body; + + public MQRedirectException(byte[] responseBody) { + this.body = responseBody; + } + + // This exception class is used as a flow control item, so stack trace is useless and performance killer. + @Override public synchronized Throwable fillInStackTrace() { + this.setStackTrace(UNASSIGNED_STACK); + return this; + } + + public byte[] getBody() { + return body; + } +} diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/MQAdminImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/MQAdminImpl.java index 8884e4adfef..dce830c7b98 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/MQAdminImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/MQAdminImpl.java @@ -16,40 +16,34 @@ */ package org.apache.rocketmq.client.impl; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.Collections; -import java.util.LinkedList; -import java.util.List; -import java.util.Set; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.locks.ReadWriteLock; -import java.util.concurrent.locks.ReentrantReadWriteLock; - +import com.google.common.base.Objects; +import com.google.common.collect.Lists; import org.apache.rocketmq.client.QueryResult; import org.apache.rocketmq.client.Validators; import org.apache.rocketmq.client.exception.MQBrokerException; import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.client.exception.MQRedirectException; import org.apache.rocketmq.client.impl.factory.MQClientInstance; import org.apache.rocketmq.client.impl.producer.TopicPublishInfo; import org.apache.rocketmq.client.log.ClientLogger; import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.TopicConfig; import org.apache.rocketmq.common.help.FAQUrl; -import org.apache.rocketmq.common.protocol.NamespaceUtil; -import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.common.message.MessageClientIDSetter; import org.apache.rocketmq.common.message.MessageConst; import org.apache.rocketmq.common.message.MessageDecoder; import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.message.MessageId; import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.common.protocol.NamespaceUtil; import org.apache.rocketmq.common.protocol.ResponseCode; import org.apache.rocketmq.common.protocol.header.QueryMessageRequestHeader; import org.apache.rocketmq.common.protocol.header.QueryMessageResponseHeader; import org.apache.rocketmq.common.protocol.route.BrokerData; +import org.apache.rocketmq.common.protocol.route.LogicalQueueRouteData; +import org.apache.rocketmq.common.protocol.route.LogicalQueuesInfo; import org.apache.rocketmq.common.protocol.route.TopicRouteData; +import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.remoting.InvokeCallback; import org.apache.rocketmq.remoting.common.RemotingUtil; import org.apache.rocketmq.remoting.exception.RemotingCommandException; @@ -57,6 +51,17 @@ import org.apache.rocketmq.remoting.netty.ResponseFuture; import org.apache.rocketmq.remoting.protocol.RemotingCommand; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; + public class MQAdminImpl { private final InternalLogger log = ClientLogger.getLog(); @@ -182,6 +187,10 @@ public Set fetchSubscribeMessageQueues(String topic) throws MQClie } public long searchOffset(MessageQueue mq, long timestamp) throws MQClientException { + LogicalQueueRouteData logicalQueueRouteData = searchLogicalQueueRouteByTimestamp(mq, timestamp); + if (logicalQueueRouteData != null) { + mq = logicalQueueRouteData.getMessageQueue(); + } String brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(mq.getBrokerName()); if (null == brokerAddr) { this.mQClientFactory.updateTopicRouteInfoFromNameServer(mq.getTopic()); @@ -190,8 +199,9 @@ public long searchOffset(MessageQueue mq, long timestamp) throws MQClientExcepti if (brokerAddr != null) { try { - return this.mQClientFactory.getMQClientAPIImpl().searchOffset(brokerAddr, mq.getTopic(), mq.getQueueId(), timestamp, + long offset = this.mQClientFactory.getMQClientAPIImpl().searchOffset(brokerAddr, mq.getTopic(), mq.getQueueId(), timestamp, timeoutMillis); + return correctLogicalQueueOffset(offset, logicalQueueRouteData); } catch (Exception e) { throw new MQClientException("Invoke Broker[" + brokerAddr + "] exception", e); } @@ -201,24 +211,50 @@ public long searchOffset(MessageQueue mq, long timestamp) throws MQClientExcepti } public long maxOffset(MessageQueue mq) throws MQClientException { - String brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(mq.getBrokerName()); - if (null == brokerAddr) { - this.mQClientFactory.updateTopicRouteInfoFromNameServer(mq.getTopic()); - brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(mq.getBrokerName()); - } + return this.maxOffset(mq, true); + } - if (brokerAddr != null) { - try { - return this.mQClientFactory.getMQClientAPIImpl().getMaxOffset(brokerAddr, mq.getTopic(), mq.getQueueId(), timeoutMillis); - } catch (Exception e) { - throw new MQClientException("Invoke Broker[" + brokerAddr + "] exception", e); + public long maxOffset(MessageQueue mq, boolean committed) throws MQClientException { + final MessageQueue origMq = mq; + String topic = mq.getTopic(); + LogicalQueueRouteData previousQueueRouteData = null; + for (int i = 0; i < 5; i++) { + LogicalQueueRouteData maxQueueRouteData = this.searchLogicalQueueRouteByOffset(origMq, Long.MAX_VALUE); + if (maxQueueRouteData != null) { + if (previousQueueRouteData != null && Objects.equal(previousQueueRouteData.getMessageQueue(), maxQueueRouteData.getMessageQueue())) { + throw new MQClientException("Topic route info not latest", null); + } + previousQueueRouteData = maxQueueRouteData; + mq = maxQueueRouteData.getMessageQueue(); + } + String brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(mq.getBrokerName()); + if (null == brokerAddr) { + this.mQClientFactory.updateTopicRouteInfoFromNameServer(topic); + brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(mq.getBrokerName()); } - } - throw new MQClientException("The broker[" + mq.getBrokerName() + "] not exist", null); + if (brokerAddr != null) { + try { + long offset = this.mQClientFactory.getMQClientAPIImpl().getMaxOffset(brokerAddr, topic, mq.getQueueId(), committed, maxQueueRouteData != null, timeoutMillis); + return correctLogicalQueueOffset(offset, maxQueueRouteData); + } catch (MQRedirectException e) { + this.mQClientFactory.updateTopicRouteInfoFromNameServer(topic, false, null, Collections.singleton(mq.getQueueId())); + continue; + } catch (Exception e) { + throw new MQClientException("Invoke Broker[" + brokerAddr + "] exception", e); + } + } + throw new MQClientException("The broker[" + mq.getBrokerName() + "] not exist", null); + } + throw new MQClientException("Redirect exceed max times", null); } public long minOffset(MessageQueue mq) throws MQClientException { + LogicalQueueRouteData minQueueRouteData = searchLogicalQueueRouteByOffset(mq, 0L); + if (minQueueRouteData != null) { + mq = minQueueRouteData.getMessageQueue(); + } + String brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(mq.getBrokerName()); if (null == brokerAddr) { this.mQClientFactory.updateTopicRouteInfoFromNameServer(mq.getTopic()); @@ -227,7 +263,8 @@ public long minOffset(MessageQueue mq) throws MQClientException { if (brokerAddr != null) { try { - return this.mQClientFactory.getMQClientAPIImpl().getMinOffset(brokerAddr, mq.getTopic(), mq.getQueueId(), timeoutMillis); + long offset = this.mQClientFactory.getMQClientAPIImpl().getMinOffset(brokerAddr, mq.getTopic(), mq.getQueueId(), timeoutMillis); + return correctLogicalQueueOffset(offset, minQueueRouteData); } catch (Exception e) { throw new MQClientException("Invoke Broker[" + brokerAddr + "] exception", e); } @@ -236,7 +273,29 @@ public long minOffset(MessageQueue mq) throws MQClientException { throw new MQClientException("The broker[" + mq.getBrokerName() + "] not exist", null); } + private List queryLogicalQueueRouteData(MessageQueue mq) { + if (MixAll.LOGICAL_QUEUE_MOCK_BROKER_NAME.equals(mq.getBrokerName())) { + TopicRouteData topicRouteData = this.mQClientFactory.queryTopicRouteData(mq.getTopic()); + if (topicRouteData == null) { + this.mQClientFactory.updateTopicRouteInfoFromNameServer(mq.getTopic()); + topicRouteData = this.mQClientFactory.queryTopicRouteData(mq.getTopic()); + } + if (topicRouteData != null) { + LogicalQueuesInfo logicalQueuesInfo = topicRouteData.getLogicalQueuesInfo(); + if (logicalQueuesInfo != null) { + return logicalQueuesInfo.get(mq.getQueueId()); + } + } + } + return null; + } + public long earliestMsgStoreTime(MessageQueue mq) throws MQClientException { + LogicalQueueRouteData minQueueRouteData = searchLogicalQueueRouteByOffset(mq, 0L); + if (minQueueRouteData != null) { + mq = minQueueRouteData.getMessageQueue(); + } + String brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(mq.getBrokerName()); if (null == brokerAddr) { this.mQClientFactory.updateTopicRouteInfoFromNameServer(mq.getTopic()); @@ -445,4 +504,71 @@ public void operationComplete(ResponseFuture responseFuture) { throw new MQClientException(ResponseCode.TOPIC_NOT_EXIST, "The topic[" + topic + "] not matched route info"); } + + private static long correctLogicalQueueOffset(long offset, LogicalQueueRouteData logicalQueueRouteData) { + if (logicalQueueRouteData == null) { + return offset; + } + return logicalQueueRouteData.toLogicalQueueOffset(offset); + } + + private LogicalQueueRouteData searchLogicalQueueRouteByTimestamp(MessageQueue mq, long timestamp) { + List queueRouteDataList = this.queryLogicalQueueRouteData(mq); + if (queueRouteDataList == null) { + return null; + } + LogicalQueueRouteData logicalQueueRouteData = null; + for (LogicalQueueRouteData el : queueRouteDataList) { + if (!el.isReadable()) { + continue; + } + if (logicalQueueRouteData == null && el.getFirstMsgTimeMillis() < 0) { + logicalQueueRouteData = el; + } else if (el.getFirstMsgTimeMillis() >= 0) { + if (el.getFirstMsgTimeMillis() <= timestamp && el.getLastMsgTimeMillis() >= timestamp) { + logicalQueueRouteData = el; + break; + } + } + } + if (logicalQueueRouteData == null) { + logicalQueueRouteData = queueRouteDataList.get(queueRouteDataList.size() - 1); + } + return logicalQueueRouteData; + } + + private LogicalQueueRouteData searchLogicalQueueRouteByOffset(MessageQueue mq, long offset) { + List queueRouteDataList = this.queryLogicalQueueRouteData(mq); + if (queueRouteDataList == null) { + return null; + } + { + List list = Lists.newArrayListWithCapacity(queueRouteDataList.size()); + for (LogicalQueueRouteData queueRouteData : queueRouteDataList) { + if (LogicalQueueRouteData.READABLE_PREDICT.apply(queueRouteData)) { + list.add(queueRouteData); + } + } + queueRouteDataList = list; + } + if (queueRouteDataList.isEmpty()) { + return null; + } + if (offset <= 0) { + // min + return Collections.min(queueRouteDataList); + } else if (offset == Long.MAX_VALUE) { + // max + return Collections.max(queueRouteDataList); + } + Collections.sort(queueRouteDataList); + LogicalQueueRouteData searchKey = new LogicalQueueRouteData(); + searchKey.setLogicalQueueDelta(offset); + int idx = Collections.binarySearch(queueRouteDataList, searchKey); + if (idx < 0) { + idx = -idx - 1; + idx -= 1; + } + return queueRouteDataList.get(idx); + } } diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java index ef57bde5e1e..114815eef8b 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java @@ -16,6 +16,7 @@ */ package org.apache.rocketmq.client.impl; +import com.google.common.base.Function; import java.io.UnsupportedEncodingException; import java.nio.ByteBuffer; import java.util.ArrayList; @@ -41,6 +42,7 @@ import org.apache.rocketmq.client.consumer.PullStatus; import org.apache.rocketmq.client.exception.MQBrokerException; import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.client.exception.MQRedirectException; import org.apache.rocketmq.client.hook.SendMessageContext; import org.apache.rocketmq.client.impl.consumer.PullResultExt; import org.apache.rocketmq.client.impl.factory.MQClientInstance; @@ -80,11 +82,13 @@ import org.apache.rocketmq.common.protocol.body.ConsumeStatsList; import org.apache.rocketmq.common.protocol.body.ConsumerConnection; import org.apache.rocketmq.common.protocol.body.ConsumerRunningInfo; +import org.apache.rocketmq.common.protocol.body.CreateMessageQueueForLogicalQueueRequestBody; import org.apache.rocketmq.common.protocol.body.GetConsumerStatusBody; import org.apache.rocketmq.common.protocol.body.GroupList; import org.apache.rocketmq.common.protocol.body.KVTable; import org.apache.rocketmq.common.protocol.body.LockBatchRequestBody; import org.apache.rocketmq.common.protocol.body.LockBatchResponseBody; +import org.apache.rocketmq.common.protocol.body.MigrateLogicalQueueBody; import org.apache.rocketmq.common.protocol.body.ProducerConnection; import org.apache.rocketmq.common.protocol.body.QueryAssignmentRequestBody; import org.apache.rocketmq.common.protocol.body.QueryAssignmentResponseBody; @@ -94,10 +98,13 @@ import org.apache.rocketmq.common.protocol.body.QueueTimeSpan; import org.apache.rocketmq.common.protocol.body.ResetOffsetBody; import org.apache.rocketmq.common.protocol.body.SetMessageRequestModeRequestBody; +import org.apache.rocketmq.common.protocol.body.ReuseTopicLogicalQueueRequestBody; +import org.apache.rocketmq.common.protocol.body.SealTopicLogicalQueueRequestBody; import org.apache.rocketmq.common.protocol.body.SubscriptionGroupWrapper; import org.apache.rocketmq.common.protocol.body.TopicConfigSerializeWrapper; import org.apache.rocketmq.common.protocol.body.TopicList; import org.apache.rocketmq.common.protocol.body.UnlockBatchRequestBody; +import org.apache.rocketmq.common.protocol.body.UpdateTopicLogicalQueueMappingRequestBody; import org.apache.rocketmq.common.protocol.header.AckMessageRequestHeader; import org.apache.rocketmq.common.protocol.header.ChangeInvisibleTimeRequestHeader; import org.apache.rocketmq.common.protocol.header.ChangeInvisibleTimeResponseHeader; @@ -108,6 +115,7 @@ import org.apache.rocketmq.common.protocol.header.CreateTopicRequestHeader; import org.apache.rocketmq.common.protocol.header.DeleteAccessConfigRequestHeader; import org.apache.rocketmq.common.protocol.header.DeleteSubscriptionGroupRequestHeader; +import org.apache.rocketmq.common.protocol.header.DeleteTopicLogicalQueueRequestHeader; import org.apache.rocketmq.common.protocol.header.DeleteTopicRequestHeader; import org.apache.rocketmq.common.protocol.header.EndTransactionRequestHeader; import org.apache.rocketmq.common.protocol.header.ExtraInfoUtil; @@ -127,6 +135,7 @@ import org.apache.rocketmq.common.protocol.header.GetMinOffsetRequestHeader; import org.apache.rocketmq.common.protocol.header.GetMinOffsetResponseHeader; import org.apache.rocketmq.common.protocol.header.GetProducerConnectionListRequestHeader; +import org.apache.rocketmq.common.protocol.header.GetTopicConfigRequestHeader; import org.apache.rocketmq.common.protocol.header.GetTopicStatsInfoRequestHeader; import org.apache.rocketmq.common.protocol.header.GetTopicsByClusterRequestHeader; import org.apache.rocketmq.common.protocol.header.PopMessageRequestHeader; @@ -140,6 +149,7 @@ import org.apache.rocketmq.common.protocol.header.QueryCorrectionOffsetHeader; import org.apache.rocketmq.common.protocol.header.QueryMessageRequestHeader; import org.apache.rocketmq.common.protocol.header.QueryTopicConsumeByWhoRequestHeader; +import org.apache.rocketmq.common.protocol.header.QueryTopicLogicalQueueMappingRequestHeader; import org.apache.rocketmq.common.protocol.header.ResetOffsetRequestHeader; import org.apache.rocketmq.common.protocol.header.ResumeCheckHalfMessageRequestHeader; import org.apache.rocketmq.common.protocol.header.SearchOffsetRequestHeader; @@ -164,8 +174,13 @@ import org.apache.rocketmq.common.protocol.heartbeat.HeartbeatData; import org.apache.rocketmq.common.protocol.heartbeat.MessageModel; import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData; +import org.apache.rocketmq.common.protocol.route.LogicalQueueRouteData; +import org.apache.rocketmq.common.protocol.route.LogicalQueuesInfo; +import org.apache.rocketmq.common.protocol.route.MessageQueueRouteState; import org.apache.rocketmq.common.protocol.route.TopicRouteData; +import org.apache.rocketmq.common.protocol.route.TopicRouteDataNameSrv; import org.apache.rocketmq.common.subscription.SubscriptionGroupConfig; +import org.apache.rocketmq.common.sysflag.MessageSysFlag; import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.remoting.CommandCustomHeader; import org.apache.rocketmq.remoting.InvokeCallback; @@ -184,6 +199,8 @@ import org.apache.rocketmq.remoting.protocol.RemotingCommand; import org.apache.rocketmq.remoting.protocol.RemotingSerializable; +import static com.google.common.base.Optional.fromNullable; + public class MQClientAPIImpl { private final static InternalLogger log = ClientLogger.getLog(); @@ -605,9 +622,13 @@ public void operationComplete(ResponseFuture responseFuture) { producer.updateFaultItem(brokerName, System.currentTimeMillis() - responseFuture.getBeginTimestamp(), false); } catch (Exception e) { - producer.updateFaultItem(brokerName, System.currentTimeMillis() - responseFuture.getBeginTimestamp(), true); - onExceptionImpl(brokerName, msg, timeoutMillis - cost, request, sendCallback, topicPublishInfo, instance, - retryTimesWhenSendFailed, times, e, context, false, producer); + if (e instanceof MQRedirectException) { + sendCallback.onException(e); + } else { + producer.updateFaultItem(brokerName, System.currentTimeMillis() - responseFuture.getBeginTimestamp(), true); + onExceptionImpl(brokerName, msg, timeoutMillis - cost, request, sendCallback, topicPublishInfo, instance, + retryTimesWhenSendFailed, times, e, context, false, producer); + } } } else { producer.updateFaultItem(brokerName, System.currentTimeMillis() - responseFuture.getBeginTimestamp(), true); @@ -693,6 +714,11 @@ private SendResult processSendResponse( final RemotingCommand response, final String addr ) throws MQBrokerException, RemotingCommandException { + HashMap extFields = response.getExtFields(); + if (extFields != null && extFields.containsKey(MessageConst.PROPERTY_REDIRECT)) { + throw new MQRedirectException(response.getBody()); + } + SendStatus sendStatus; switch (response.getCode()) { case ResponseCode.FLUSH_DISK_TIMEOUT: { @@ -941,6 +967,11 @@ private PullResult pullMessageSync( private PullResult processPullResponse( final RemotingCommand response, final String addr) throws MQBrokerException, RemotingCommandException { + HashMap extFields = response.getExtFields(); + if (extFields != null && extFields.containsKey(MessageConst.PROPERTY_REDIRECT)) { + throw new MQRedirectException(response.getBody()); + } + PullStatus pullStatus = PullStatus.NO_NEW_MSG; switch (response.getCode()) { case ResponseCode.SUCCESS: @@ -955,7 +986,6 @@ private PullResult processPullResponse( case ResponseCode.PULL_OFFSET_MOVED: pullStatus = PullStatus.OFFSET_ILLEGAL; break; - default: throw new MQBrokerException(response.getCode(), response.getRemark(), addr); } @@ -1107,15 +1137,28 @@ public long searchOffset(final String addr, final String topic, final int queueI } public long getMaxOffset(final String addr, final String topic, final int queueId, final long timeoutMillis) + throws RemotingException, MQBrokerException, InterruptedException { + return getMaxOffset(addr, topic, queueId, true, false, timeoutMillis); + } + + public long getMaxOffset(final String addr, final String topic, final int queueId, boolean committed, + boolean fromLogicalQueue, + final long timeoutMillis) throws RemotingException, MQBrokerException, InterruptedException { GetMaxOffsetRequestHeader requestHeader = new GetMaxOffsetRequestHeader(); requestHeader.setTopic(topic); requestHeader.setQueueId(queueId); + requestHeader.setCommitted(committed); + requestHeader.setLogicalQueue(fromLogicalQueue); RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_MAX_OFFSET, requestHeader); RemotingCommand response = this.remotingClient.invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr), request, timeoutMillis); assert response != null; + HashMap extFields = response.getExtFields(); + if (extFields != null && extFields.containsKey(MessageConst.PROPERTY_REDIRECT)) { + throw new MQRedirectException(response.getBody()); + } switch (response.getCode()) { case ResponseCode.SUCCESS: { GetMaxOffsetResponseHeader responseHeader = @@ -1611,8 +1654,15 @@ public TopicRouteData getTopicRouteInfoFromNameServer(final String topic, final public TopicRouteData getTopicRouteInfoFromNameServer(final String topic, final long timeoutMillis, boolean allowTopicNotExist) throws MQClientException, InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException { + return getTopicRouteInfoFromNameServer(topic, timeoutMillis, allowTopicNotExist, null); + } + + public TopicRouteData getTopicRouteInfoFromNameServer(final String topic, final long timeoutMillis, + boolean allowTopicNotExist, Set logicalQueueIdsFilter) throws MQClientException, InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException { GetRouteInfoRequestHeader requestHeader = new GetRouteInfoRequestHeader(); requestHeader.setTopic(topic); + requestHeader.setSysFlag(MessageSysFlag.LOGICAL_QUEUE_FLAG); + requestHeader.setLogicalQueueIdsFilter(logicalQueueIdsFilter); RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_ROUTEINFO_BY_TOPIC, requestHeader); @@ -1629,7 +1679,11 @@ public TopicRouteData getTopicRouteInfoFromNameServer(final String topic, final case ResponseCode.SUCCESS: { byte[] body = response.getBody(); if (body != null) { - return TopicRouteData.decode(body, TopicRouteData.class); + return fromNullable(RemotingSerializable.decode(body, TopicRouteDataNameSrv.class)).transform(new Function() { + @Override public TopicRouteData apply(TopicRouteDataNameSrv srv) { + return srv.toTopicRouteData(); + } + }).orNull(); } } default: @@ -2537,4 +2591,148 @@ public void setMessageRequestMode(final String brokerAddr, final String topic, f throw new MQClientException(response.getCode(), response.getRemark()); } } + + public LogicalQueuesInfo queryTopicLogicalQueue(String brokerAddr, String topic, + long timeoutMillis) throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException, MQBrokerException { + QueryTopicLogicalQueueMappingRequestHeader requestHeader = new QueryTopicLogicalQueueMappingRequestHeader(); + requestHeader.setTopic(topic); + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.QUERY_TOPIC_LOGICAL_QUEUE_MAPPING, requestHeader); + RemotingCommand response = this.remotingClient.invokeSync(brokerAddr, request, timeoutMillis); + assert response != null; + if (response.getCode() != ResponseCode.SUCCESS) { + throw new MQBrokerException(response.getCode(), response.getRemark()); + } + return RemotingSerializable.decode(response.getBody(), LogicalQueuesInfo.class); + } + + public void updateTopicLogicalQueue(String brokerAddr, String topic, int queueId, int logicalQueueIndex, + long timeoutMillis) throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException, MQBrokerException { + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.UPDATE_TOPIC_LOGICAL_QUEUE_MAPPING, null); + UpdateTopicLogicalQueueMappingRequestBody requestBody = new UpdateTopicLogicalQueueMappingRequestBody(); + requestBody.setTopic(topic); + requestBody.setQueueId(queueId); + requestBody.setLogicalQueueIdx(logicalQueueIndex); + request.setBody(requestBody.encode()); + RemotingCommand response = this.remotingClient.invokeSync(brokerAddr, request, timeoutMillis); + assert response != null; + if (response.getCode() != ResponseCode.SUCCESS) { + throw new MQBrokerException(response.getCode(), response.getRemark()); + } + } + + public void deleteTopicLogicalQueueMapping(String brokerAddr, String topic, long timeoutMillis) throws MQBrokerException, InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException { + DeleteTopicLogicalQueueRequestHeader requestHeader = new DeleteTopicLogicalQueueRequestHeader(); + requestHeader.setTopic(topic); + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.DELETE_TOPIC_LOGICAL_QUEUE_MAPPING, requestHeader); + RemotingCommand response = this.remotingClient.invokeSync(brokerAddr, request, timeoutMillis); + assert response != null; + if (response.getCode() != ResponseCode.SUCCESS) { + throw new MQBrokerException(response.getCode(), response.getRemark()); + } + } + + public LogicalQueueRouteData sealTopicLogicalQueue(String brokerAddr, LogicalQueueRouteData queueRouteData, long timeoutMillis) throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException, MQBrokerException { + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.SEAL_TOPIC_LOGICAL_QUEUE, null); + SealTopicLogicalQueueRequestBody requestBody = new SealTopicLogicalQueueRequestBody(); + MessageQueue messageQueue = queueRouteData.getMessageQueue(); + requestBody.setTopic(messageQueue.getTopic()); + requestBody.setQueueId(messageQueue.getQueueId()); + requestBody.setLogicalQueueIndex(queueRouteData.getLogicalQueueIndex()); + request.setBody(requestBody.encode()); + RemotingCommand response = this.remotingClient.invokeSync(brokerAddr, request, timeoutMillis); + assert response != null; + if (response.getCode() != ResponseCode.SUCCESS) { + throw new MQBrokerException(response.getCode(), response.getRemark()); + } + return RemotingSerializable.decode(response.getBody(), LogicalQueueRouteData.class); + } + + public LogicalQueueRouteData reuseTopicLogicalQueue(String brokerAddr, String topic, int queueId, + int logicalQueueIdx, + MessageQueueRouteState messageQueueRouteState, long timeoutMillis) throws InterruptedException, MQBrokerException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException { + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.REUSE_TOPIC_LOGICAL_QUEUE, null); + ReuseTopicLogicalQueueRequestBody requestBody = new ReuseTopicLogicalQueueRequestBody(); + requestBody.setTopic(topic); + requestBody.setQueueId(queueId); + requestBody.setLogicalQueueIndex(logicalQueueIdx); + requestBody.setMessageQueueRouteState(messageQueueRouteState); + request.setBody(requestBody.encode()); + RemotingCommand response = this.remotingClient.invokeSync(brokerAddr, request, timeoutMillis); + assert response != null; + if (response.getCode() != ResponseCode.SUCCESS) { + throw new MQBrokerException(response.getCode(), response.getRemark()); + } + return RemotingSerializable.decode(response.getBody(), LogicalQueueRouteData.class); + } + + public LogicalQueueRouteData createMessageQueueForLogicalQueue(String brokerAddr, String topic, int logicalQueueIdx, + MessageQueueRouteState messageQueueStatus, + long timeoutMillis) throws InterruptedException, MQBrokerException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException { + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CREATE_MESSAGE_QUEUE_FOR_LOGICAL_QUEUE, null); + CreateMessageQueueForLogicalQueueRequestBody requestBody = new CreateMessageQueueForLogicalQueueRequestBody(); + requestBody.setTopic(topic); + requestBody.setLogicalQueueIndex(logicalQueueIdx); + requestBody.setMessageQueueStatus(messageQueueStatus); + request.setBody(requestBody.encode()); + RemotingCommand response = this.remotingClient.invokeSync(brokerAddr, request, timeoutMillis); + assert response != null; + if (response.getCode() != ResponseCode.SUCCESS) { + throw new MQBrokerException(response.getCode(), response.getRemark()); + } + return RemotingSerializable.decode(response.getBody(), LogicalQueueRouteData.class); + } + + private MigrateLogicalQueueBody migrateTopicLogicalQueue(int requestCode, String brokerAddr, + LogicalQueueRouteData fromQueueRouteData, LogicalQueueRouteData toQueueRouteData, + long timeoutMillis) throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException, MQBrokerException { + RemotingCommand request = RemotingCommand.createRequestCommand(requestCode, null); + MigrateLogicalQueueBody requestBody = new MigrateLogicalQueueBody(); + requestBody.setFromQueueRouteData(fromQueueRouteData); + requestBody.setToQueueRouteData(toQueueRouteData); + request.setBody(requestBody.encode()); + RemotingCommand response = this.remotingClient.invokeSync(brokerAddr, request, timeoutMillis); + assert response != null; + if (response.getCode() != ResponseCode.SUCCESS) { + throw new MQBrokerException(response.getCode(), response.getRemark()); + } + return response.getBody() != null ? RemotingSerializable.decode(response.getBody(), MigrateLogicalQueueBody.class) : null; + } + + public MigrateLogicalQueueBody migrateTopicLogicalQueuePrepare(String brokerAddr, + LogicalQueueRouteData fromQueueRouteData, LogicalQueueRouteData toQueueRouteData, + long timeoutMillis) throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException, MQBrokerException { + return migrateTopicLogicalQueue(RequestCode.MIGRATE_TOPIC_LOGICAL_QUEUE_PREPARE, brokerAddr, fromQueueRouteData, toQueueRouteData, timeoutMillis); + } + + public MigrateLogicalQueueBody migrateTopicLogicalQueueCommit(String brokerAddr, + LogicalQueueRouteData fromQueueRouteData, LogicalQueueRouteData toQueueRouteData, + long timeoutMillis) throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException, MQBrokerException { + return migrateTopicLogicalQueue(RequestCode.MIGRATE_TOPIC_LOGICAL_QUEUE_COMMIT, brokerAddr, fromQueueRouteData, toQueueRouteData, timeoutMillis); + } + + public void migrateTopicLogicalQueueNotify(String brokerAddr, + LogicalQueueRouteData fromQueueRouteData, + LogicalQueueRouteData toQueueRouteData, + long timeoutMillis) throws InterruptedException, MQBrokerException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException { + migrateTopicLogicalQueue(RequestCode.MIGRATE_TOPIC_LOGICAL_QUEUE_NOTIFY, brokerAddr, fromQueueRouteData, toQueueRouteData, timeoutMillis); + } + + public TopicConfig getTopicConfig(final String brokerAddr, String topic, + long timeoutMillis) throws InterruptedException, + RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException, MQBrokerException { + GetTopicConfigRequestHeader header = new GetTopicConfigRequestHeader(); + header.setTopic(topic); + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_TOPIC_CONFIG, header); + RemotingCommand response = this.remotingClient + .invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), brokerAddr), request, timeoutMillis); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + return RemotingSerializable.decode(response.getBody(), TopicConfig.class); + } + default: + break; + } + throw new MQBrokerException(response.getCode(), response.getRemark()); + } } diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java index 2e73f1a5170..3be0658c070 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java @@ -147,7 +147,7 @@ private enum SubscriptionType { private final MessageQueueLock messageQueueLock = new MessageQueueLock(); - private final ArrayList consumeMessageHookList = new ArrayList<>(); + private final ArrayList consumeMessageHookList = new ArrayList(); // only for test purpose, will be modified by reflection in unit test. @SuppressWarnings("FieldMayBeFinal") private static boolean doNotUpdateTopicSubscribeInfoWhenSubscriptionChanged = false; @@ -898,6 +898,9 @@ private PullResult pullSyncImpl(MessageQueue mq, SubscriptionData subscriptionDa null ); this.pullAPIWrapper.processPullResult(mq, pullResult, subscriptionData); + if (pullResult instanceof PullResultWithLogicalQueues) { + pullResult = ((PullResultWithLogicalQueues) pullResult).getOrigPullResultExt(); + } if (!this.consumeMessageHookList.isEmpty()) { ConsumeMessageContext consumeMessageContext = new ConsumeMessageContext(); consumeMessageContext.setNamespace(defaultLitePullConsumer.getNamespace()); diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPullConsumerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPullConsumerImpl.java index eed5fa43f04..e54d9b6a341 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPullConsumerImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPullConsumerImpl.java @@ -265,6 +265,9 @@ private PullResult pullSyncImpl(MessageQueue mq, SubscriptionData subscriptionDa null ); this.pullAPIWrapper.processPullResult(mq, pullResult, subscriptionData); + if (pullResult instanceof PullResultWithLogicalQueues) { + pullResult = ((PullResultWithLogicalQueues) pullResult).getOrigPullResultExt(); + } //If namespace is not null , reset Topic without namespace. this.resetTopic(pullResult.getMsgFoundList()); if (!this.consumeMessageHookList.isEmpty()) { diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java index b478cb10932..34a5043ed1c 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java @@ -859,9 +859,11 @@ public synchronized void start() throws MQClientException { this.rebalanceImpl.setAllocateMessageQueueStrategy(this.defaultMQPushConsumer.getAllocateMessageQueueStrategy()); this.rebalanceImpl.setmQClientFactory(this.mQClientFactory); - this.pullAPIWrapper = new PullAPIWrapper( - mQClientFactory, - this.defaultMQPushConsumer.getConsumerGroup(), isUnitMode()); + if (this.pullAPIWrapper == null) { + this.pullAPIWrapper = new PullAPIWrapper( + mQClientFactory, + this.defaultMQPushConsumer.getConsumerGroup(), isUnitMode()); + } this.pullAPIWrapper.registerFilterMessageHook(filterMessageHookList); if (this.defaultMQPushConsumer.getOffsetStore() != null) { diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/PullAPIWrapper.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/PullAPIWrapper.java index 95b609e439a..5868e30f2bb 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/PullAPIWrapper.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/PullAPIWrapper.java @@ -16,19 +16,15 @@ */ package org.apache.rocketmq.client.impl.consumer; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.List; -import java.util.Random; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.atomic.AtomicLong; +import com.alibaba.fastjson.JSON; +import com.google.common.base.Objects; import org.apache.rocketmq.client.consumer.PopCallback; import org.apache.rocketmq.client.consumer.PullCallback; import org.apache.rocketmq.client.consumer.PullResult; import org.apache.rocketmq.client.consumer.PullStatus; import org.apache.rocketmq.client.exception.MQBrokerException; import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.client.exception.MQRedirectException; import org.apache.rocketmq.client.hook.FilterMessageContext; import org.apache.rocketmq.client.hook.FilterMessageHook; import org.apache.rocketmq.client.impl.CommunicationMode; @@ -44,13 +40,29 @@ import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.common.protocol.header.PopMessageRequestHeader; +import org.apache.rocketmq.common.protocol.ResponseCode; import org.apache.rocketmq.common.protocol.header.PullMessageRequestHeader; import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData; +import org.apache.rocketmq.common.protocol.route.LogicalQueueRouteData; +import org.apache.rocketmq.common.protocol.route.LogicalQueuesInfo; +import org.apache.rocketmq.common.protocol.route.MessageQueueRouteState; import org.apache.rocketmq.common.protocol.route.TopicRouteData; import org.apache.rocketmq.common.sysflag.PullSysFlag; import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.remoting.exception.RemotingException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Random; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +import static com.google.common.base.Optional.fromNullable; + public class PullAPIWrapper { private final InternalLogger log = ClientLogger.getLog(); private final MQClientInstance mQClientFactory; @@ -60,7 +72,7 @@ public class PullAPIWrapper { new ConcurrentHashMap(32); private volatile boolean connectBrokerByUser = false; private volatile long defaultBrokerId = MixAll.MASTER_ID; - private Random random = new Random(System.currentTimeMillis()); + private Random random = new Random(System.nanoTime()); private ArrayList filterMessageHookList = new ArrayList(); public PullAPIWrapper(MQClientInstance mQClientFactory, String consumerGroup, boolean unitMode) { @@ -71,13 +83,36 @@ public PullAPIWrapper(MQClientInstance mQClientFactory, String consumerGroup, bo public PullResult processPullResult(final MessageQueue mq, final PullResult pullResult, final SubscriptionData subscriptionData) { - PullResultExt pullResultExt = (PullResultExt) pullResult; + final PullResultExt pullResultExt = (PullResultExt) pullResult; + + LogicalQueueRouteData queueRouteData = null; + PullResultWithLogicalQueues pullResultWithLogicalQueues = null; + if (pullResultExt instanceof PullResultWithLogicalQueues) { + pullResultWithLogicalQueues = (PullResultWithLogicalQueues) pullResultExt; + queueRouteData = pullResultWithLogicalQueues.getQueueRouteData(); + } + + if (queueRouteData != null) { + pullResultWithLogicalQueues.setOrigPullResultExt(new PullResultExt(pullResultExt.getPullStatus(), + queueRouteData.toLogicalQueueOffset(pullResultExt.getNextBeginOffset()), + queueRouteData.toLogicalQueueOffset(pullResultExt.getMinOffset()), + // although this maxOffset may not belong to this queue route, but the actual value must be a larger one, and since maxOffset here is not an accurate value, we just do it to make things simple. + queueRouteData.toLogicalQueueOffset(pullResultExt.getMaxOffset()), + pullResultExt.getMsgFoundList(), + pullResultExt.getSuggestWhichBrokerId(), + pullResultExt.getMessageBinary())); + } this.updatePullFromWhichNode(mq, pullResultExt.getSuggestWhichBrokerId()); if (PullStatus.FOUND == pullResult.getPullStatus()) { ByteBuffer byteBuffer = ByteBuffer.wrap(pullResultExt.getMessageBinary()); List msgList = MessageDecoder.decodes(byteBuffer); + if (queueRouteData != null) { + // prevent pulled data is out of current queue route, this happens when some commit log data is cleaned in the broker but still pull from it. + msgList = queueRouteData.filterMessages(msgList); + } + List msgListFilterAgain = msgList; if (!subscriptionData.getTagsSet().isEmpty() && !subscriptionData.isClassFilterMode()) { msgListFilterAgain = new ArrayList(msgList.size()); @@ -107,6 +142,10 @@ public PullResult processPullResult(final MessageQueue mq, final PullResult pull MessageAccessor.putProperty(msg, MessageConst.PROPERTY_MAX_OFFSET, Long.toString(pullResult.getMaxOffset())); msg.setBrokerName(mq.getBrokerName()); + msg.setQueueId(mq.getQueueId()); + if (queueRouteData != null) { + msg.setQueueOffset(queueRouteData.toLogicalQueueOffset(msg.getQueueOffset())); + } } pullResultExt.setMsgFoundList(msgListFilterAgain); @@ -114,7 +153,7 @@ public PullResult processPullResult(final MessageQueue mq, final PullResult pull pullResultExt.setMessageBinary(null); - return pullResult; + return pullResultExt; } public void updatePullFromWhichNode(final MessageQueue mq, final long brokerId) { @@ -143,24 +182,67 @@ public void executeHook(final FilterMessageContext context) { } public PullResult pullKernelImpl( - final MessageQueue mq, + MessageQueue mq, final String subExpression, final String expressionType, final long subVersion, - final long offset, + long offset, final int maxNums, final int sysFlag, - final long commitOffset, + long commitOffset, final long brokerSuspendMaxTimeMillis, final long timeoutMillis, final CommunicationMode communicationMode, - final PullCallback pullCallback + PullCallback pullCallback ) throws MQClientException, RemotingException, MQBrokerException, InterruptedException { + if (MixAll.LOGICAL_QUEUE_MOCK_BROKER_NAME.equals(mq.getBrokerName())) { + LogicalQueueContext logicalQueueContext = new LogicalQueueContext(mq, subExpression, expressionType, subVersion, offset, maxNums, sysFlag, commitOffset, brokerSuspendMaxTimeMillis, timeoutMillis, communicationMode, pullCallback); + while (true) { + try { + MessageQueue messageQueue = logicalQueueContext.getModifiedMessageQueue(); + if (messageQueue == null) { + if (pullCallback != null) { + pullCallback.onSuccess(logicalQueueContext.getPullResult()); + return null; + } else { + return logicalQueueContext.getPullResult(); + } + } + PullResult pullResult = this.pullKernelImplWithoutRetry(messageQueue, subExpression, expressionType, subVersion, logicalQueueContext.getModifiedOffset(), maxNums, sysFlag, logicalQueueContext.getModifiedCommitOffset(), brokerSuspendMaxTimeMillis, timeoutMillis, communicationMode, logicalQueueContext.wrapPullCallback()); + return logicalQueueContext.wrapPullResult(pullResult); + } catch (MQRedirectException e) { + if (!logicalQueueContext.shouldRetry(e)) { + throw new MQBrokerException(ResponseCode.SYSTEM_ERROR, "redirect"); + } + } + } + } else { + return this.pullKernelImplWithoutRetry(mq, subExpression, expressionType, subVersion, offset, maxNums, sysFlag, commitOffset, brokerSuspendMaxTimeMillis, timeoutMillis, communicationMode, pullCallback); + } + } + + public PullResult pullKernelImplWithoutRetry( + MessageQueue mq, + final String subExpression, + final String expressionType, + final long subVersion, + long offset, + final int maxNums, + final int sysFlag, + long commitOffset, + final long brokerSuspendMaxTimeMillis, + final long timeoutMillis, + final CommunicationMode communicationMode, + PullCallback pullCallback + ) throws MQClientException, RemotingException, MQBrokerException, InterruptedException { + String topic = mq.getTopic(); + int queueId = mq.getQueueId(); + FindBrokerResult findBrokerResult = this.mQClientFactory.findBrokerAddressInSubscribe(mq.getBrokerName(), this.recalculatePullFromWhichNode(mq), false); if (null == findBrokerResult) { - this.mQClientFactory.updateTopicRouteInfoFromNameServer(mq.getTopic()); + this.mQClientFactory.updateTopicRouteInfoFromNameServer(topic); findBrokerResult = this.mQClientFactory.findBrokerAddressInSubscribe(mq.getBrokerName(), this.recalculatePullFromWhichNode(mq), false); @@ -183,8 +265,8 @@ public PullResult pullKernelImpl( PullMessageRequestHeader requestHeader = new PullMessageRequestHeader(); requestHeader.setConsumerGroup(this.consumerGroup); - requestHeader.setTopic(mq.getTopic()); - requestHeader.setQueueId(mq.getQueueId()); + requestHeader.setTopic(topic); + requestHeader.setQueueId(queueId); requestHeader.setQueueOffset(offset); requestHeader.setMaxMsgNums(maxNums); requestHeader.setSysFlag(sysFlagInner); @@ -196,17 +278,15 @@ public PullResult pullKernelImpl( String brokerAddr = findBrokerResult.getBrokerAddr(); if (PullSysFlag.hasClassFilterFlag(sysFlagInner)) { - brokerAddr = computePullFromWhichFilterServer(mq.getTopic(), brokerAddr); + brokerAddr = computePullFromWhichFilterServer(topic, brokerAddr); } - PullResult pullResult = this.mQClientFactory.getMQClientAPIImpl().pullMessage( + return this.mQClientFactory.getMQClientAPIImpl().pullMessage( brokerAddr, requestHeader, timeoutMillis, communicationMode, pullCallback); - - return pullResult; } throw new MQClientException("The broker[" + mq.getBrokerName() + "] not exist", null); @@ -322,4 +402,229 @@ public void popAsync(MessageQueue mq, long invisibleTime, int maxNums, String co } throw new MQClientException("The broker[" + mq.getBrokerName() + "] not exist", null); } + + private class LogicalQueueContext implements PullCallback { + private final MessageQueue mq; + private final String subExpression; + private final String expressionType; + private final long subVersion; + private final long offset; + private final int maxNums; + private final int sysFlag; + private final long commitOffset; + private final long brokerSuspendMaxTimeMillis; + private final long timeoutMillis; + private final CommunicationMode communicationMode; + private final PullCallback pullCallback; + + private volatile LogicalQueuesInfo logicalQueuesInfo; + private volatile LogicalQueueRouteData logicalQueueRouteData; + private volatile boolean isMaxReadableQueueRoute; + + private volatile PullResultExt pullResult = null; + + private final AtomicInteger retry = new AtomicInteger(); + + public LogicalQueueContext(MessageQueue mq, String subExpression, String expressionType, long subVersion, + long offset, int maxNums, int sysFlag, long commitOffset, long brokerSuspendMaxTimeMillis, + long timeoutMillis, CommunicationMode communicationMode, + PullCallback pullCallback) { + this.mq = mq; + this.subExpression = subExpression; + this.expressionType = expressionType; + this.subVersion = subVersion; + this.offset = offset; + this.maxNums = maxNums; + this.sysFlag = sysFlag; + this.commitOffset = commitOffset; + this.brokerSuspendMaxTimeMillis = brokerSuspendMaxTimeMillis; + this.timeoutMillis = timeoutMillis; + this.communicationMode = communicationMode; + this.pullCallback = pullCallback; + + this.buildLogicalQueuesInfo(); + } + + private boolean notUsingLogicalQueue() { + return !Objects.equal(mq.getBrokerName(), MixAll.LOGICAL_QUEUE_MOCK_BROKER_NAME) || this.logicalQueuesInfo == null; + } + + private void buildLogicalQueuesInfo() { + TopicRouteData topicRouteData = PullAPIWrapper.this.mQClientFactory.queryTopicRouteData(mq.getTopic()); + if (topicRouteData != null) { + this.logicalQueuesInfo = topicRouteData.getLogicalQueuesInfo(); + } + } + + @Override public void onSuccess(PullResult pullResult) { + this.pullCallback.onSuccess(this.wrapPullResult(pullResult)); + } + + @Override public void onException(Throwable t) { + if (!this.shouldRetry(t)) { + this.pullCallback.onException(t); + return; + } + MessageQueue messageQueue = this.getModifiedMessageQueue(); + if (messageQueue == null) { + this.pullCallback.onSuccess(this.getPullResult()); + return; + } + try { + PullAPIWrapper.this.pullKernelImplWithoutRetry(messageQueue, subExpression, expressionType, subVersion, this.getModifiedOffset(), maxNums, sysFlag, this.getModifiedCommitOffset(), brokerSuspendMaxTimeMillis, timeoutMillis, communicationMode, this); + } catch (Exception e) { + this.pullCallback.onException(e); + } + } + + public MessageQueue getModifiedMessageQueue() { + if (this.notUsingLogicalQueue()) { + return this.mq; + } + this.logicalQueuesInfo.readLock().lock(); + try { + List queueRouteDataList = fromNullable(this.logicalQueuesInfo.get(this.mq.getQueueId())).or(Collections.emptyList()); + LogicalQueueRouteData searchKey = new LogicalQueueRouteData(); + searchKey.setState(MessageQueueRouteState.Normal); + searchKey.setLogicalQueueDelta(offset); + // it's sorted after getTopicRouteInfoFromNameServer + int startIdx = Collections.binarySearch(queueRouteDataList, searchKey); + if (startIdx < 0) { + startIdx = -startIdx - 1; + // lower entry + startIdx -= 1; + } + this.logicalQueueRouteData = null; + this.pullResult = null; + LogicalQueueRouteData lastReadableLogicalQueueRouteData = null; // first item which delta > offset + LogicalQueueRouteData minReadableLogicalQueueRouteData = null; + LogicalQueueRouteData maxReadableLogicalQueueRouteData = null; + for (int i = 0, size = queueRouteDataList.size(); i < size; i++) { + LogicalQueueRouteData queueRouteData = queueRouteDataList.get(i); + if (!queueRouteData.isReadable()) { + continue; + } + maxReadableLogicalQueueRouteData = queueRouteData; + if (minReadableLogicalQueueRouteData == null) { + minReadableLogicalQueueRouteData = queueRouteData; + if (i < startIdx) { + // must consider following `i++` operation when invoke `continue`, so decrease first + i = startIdx - 1; + continue; + } + } + if (queueRouteData.getLogicalQueueDelta() > offset) { + if (this.logicalQueueRouteData != null) { + if (this.logicalQueueRouteData.toLogicalQueueOffset(this.logicalQueueRouteData.getOffsetMax()) <= offset) { + this.logicalQueueRouteData = queueRouteData; + } + break; + } else { + if (lastReadableLogicalQueueRouteData == null) { + lastReadableLogicalQueueRouteData = queueRouteData; + } + } + } else { + this.logicalQueueRouteData = queueRouteData; + } + } + if (this.logicalQueueRouteData == null) { + if (lastReadableLogicalQueueRouteData != null) { + this.pullResult = new PullResultExt(PullStatus.OFFSET_ILLEGAL, lastReadableLogicalQueueRouteData.getLogicalQueueDelta(), minReadableLogicalQueueRouteData.getLogicalQueueDelta(), maxReadableLogicalQueueRouteData.getLogicalQueueDelta(), null, 0, null); + return null; + } else { + if (maxReadableLogicalQueueRouteData != null) { + this.logicalQueueRouteData = maxReadableLogicalQueueRouteData; + } else { + if (!queueRouteDataList.isEmpty()) { + this.logicalQueueRouteData = queueRouteDataList.get(queueRouteDataList.size() - 1); + } else { + pullResult = new PullResultExt(PullStatus.NO_NEW_MSG, 0, 0, 0, null, 0, null); + return null; + } + } + } + } + this.isMaxReadableQueueRoute = this.logicalQueueRouteData.isSameTo(maxReadableLogicalQueueRouteData); + return this.logicalQueueRouteData.getMessageQueue(); + } finally { + this.logicalQueuesInfo.readLock().unlock(); + } + } + + public PullResultExt getPullResult() { + return pullResult; + } + + public PullCallback wrapPullCallback() { + if (this.notUsingLogicalQueue()) { + return this.pullCallback; + } + if (!CommunicationMode.ASYNC.equals(this.communicationMode)) { + return this.pullCallback; + } + return this; + } + + public long getModifiedOffset() { + return this.logicalQueueRouteData.toMessageQueueOffset(this.offset); + } + + public long getModifiedCommitOffset() { + // TODO should this be modified too? If offset is not in current broker's range, how do we handle it? + return this.commitOffset; + } + + public void incrRetry() { + this.retry.incrementAndGet(); + } + + public boolean shouldRetry(Throwable t) { + this.incrRetry(); + if (this.retry.get() >= 3) { + return false; + } + if (t instanceof MQRedirectException) { + MQRedirectException e = (MQRedirectException) t; + this.processResponseBody(e.getBody()); + return true; + } + return false; + } + + public PullResult wrapPullResult(PullResult pullResult) { + if (pullResult == null) { + return null; + } + if (this.logicalQueueRouteData == null) { + return pullResult; + } + if (!this.isMaxReadableQueueRoute && PullStatus.NO_MATCHED_MSG.equals(pullResult.getPullStatus())) { + PullStatus status = PullStatus.OFFSET_ILLEGAL; + if (pullResult instanceof PullResultExt) { + PullResultExt pullResultExt = (PullResultExt) pullResult; + pullResult = new PullResultExt(status, pullResultExt.getNextBeginOffset(), pullResultExt.getMinOffset(), pullResultExt.getMaxOffset(), pullResultExt.getMsgFoundList(), pullResultExt.getSuggestWhichBrokerId(), pullResultExt.getMessageBinary()); + } else { + pullResult = new PullResult(status, pullResult.getNextBeginOffset(), pullResult.getMinOffset(), pullResult.getMaxOffset(), pullResult.getMsgFoundList()); + } + } + // method PullAPIWrapper#processPullResult will modify queueOffset/nextBeginOffset/minOffset/maxOffset + return new PullResultWithLogicalQueues(pullResult, this.logicalQueueRouteData); + } + + public void processResponseBody(byte[] responseBody) { + log.info("LogicalQueueContext.processResponseBody got redirect {}: {}", this.logicalQueueRouteData, responseBody != null ? new String(responseBody, MessageDecoder.CHARSET_UTF8) : null); + if (responseBody != null) { + try { + List queueRouteDataList = JSON.parseObject(responseBody, MixAll.TYPE_LIST_LOGICAL_QUEUE_ROUTE_DATA); + this.logicalQueuesInfo.updateLogicalQueueRouteDataList(this.mq.getQueueId(), queueRouteDataList); + return; + } catch (Exception e) { + log.warn("LogicalQueueContext.processResponseBody {} update exception, fallback to updateTopicRouteInfoFromNameServer", this.logicalQueueRouteData, e); + } + } + PullAPIWrapper.this.mQClientFactory.updateTopicRouteInfoFromNameServer(mq.getTopic(), false, null, Collections.singleton(this.mq.getQueueId())); + this.buildLogicalQueuesInfo(); + } + } } diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/PullResultWithLogicalQueues.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/PullResultWithLogicalQueues.java new file mode 100644 index 00000000000..8b85e548ca7 --- /dev/null +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/PullResultWithLogicalQueues.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.client.impl.consumer; + +import java.util.List; +import org.apache.rocketmq.client.consumer.PullResult; +import org.apache.rocketmq.client.consumer.PullStatus; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.protocol.route.LogicalQueueRouteData; + +public class PullResultWithLogicalQueues extends PullResultExt { + private PullResultExt origPullResultExt; + private final LogicalQueueRouteData queueRouteData; + + public PullResultWithLogicalQueues(PullResult pullResult, LogicalQueueRouteData floorQueueRouteData) { + super(pullResult.getPullStatus(), pullResult.getNextBeginOffset(), pullResult.getMinOffset(), pullResult.getMaxOffset(), pullResult.getMsgFoundList(), + pullResult instanceof PullResultExt ? ((PullResultExt) pullResult).getSuggestWhichBrokerId() : MixAll.MASTER_ID, + pullResult instanceof PullResultExt ? ((PullResultExt) pullResult).getMessageBinary() : null); + if (pullResult instanceof PullResultExt) { + this.origPullResultExt = (PullResultExt) pullResult; + } else { + this.origPullResultExt = new PullResultExt(pullResult.getPullStatus(), pullResult.getNextBeginOffset(), pullResult.getMinOffset(), pullResult.getMaxOffset(), pullResult.getMsgFoundList(), MixAll.MASTER_ID, null); + } + this.queueRouteData = floorQueueRouteData; + } + + public PullResult getOrigPullResultExt() { + return origPullResultExt; + } + + public LogicalQueueRouteData getQueueRouteData() { + return queueRouteData; + } + + public void setOrigPullResultExt(PullResultExt pullResultExt) { + this.origPullResultExt = pullResultExt; + } + + @Override public PullStatus getPullStatus() { + return origPullResultExt.getPullStatus(); + } + + @Override public long getNextBeginOffset() { + return origPullResultExt.getNextBeginOffset(); + } + + @Override public long getMinOffset() { + return origPullResultExt.getMinOffset(); + } + + @Override public long getMaxOffset() { + return origPullResultExt.getMaxOffset(); + } + + @Override public List getMsgFoundList() { + return origPullResultExt.getMsgFoundList(); + } + + @Override public void setMsgFoundList(List msgFoundList) { + origPullResultExt.setMsgFoundList(msgFoundList); + } + + @Override public byte[] getMessageBinary() { + return origPullResultExt.getMessageBinary(); + } + + @Override public void setMessageBinary(byte[] messageBinary) { + origPullResultExt.setMessageBinary(messageBinary); + } + + @Override public long getSuggestWhichBrokerId() { + return origPullResultExt.getSuggestWhichBrokerId(); + } + + @Override public String toString() { + return "PullResultWithLogicalQueues{" + + "origPullResultExt=" + origPullResultExt + + ", queueRouteData=" + queueRouteData + + '}'; + } +} diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java b/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java index 619cfc204ef..c1a50dd53f8 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java @@ -18,6 +18,7 @@ import java.io.UnsupportedEncodingException; import java.util.Collections; +import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -76,6 +77,8 @@ import org.apache.rocketmq.common.protocol.heartbeat.ProducerData; import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData; import org.apache.rocketmq.common.protocol.route.BrokerData; +import org.apache.rocketmq.common.protocol.route.LogicalQueueRouteData; +import org.apache.rocketmq.common.protocol.route.LogicalQueuesInfo; import org.apache.rocketmq.common.protocol.route.QueueData; import org.apache.rocketmq.common.protocol.route.TopicRouteData; import org.apache.rocketmq.logging.InternalLogger; @@ -174,6 +177,32 @@ public static TopicPublishInfo topicRouteData2TopicPublishInfo(final String topi } info.setOrderTopic(true); + } else if (route.getOrderTopicConf() == null && route.getLogicalQueuesInfo() != null) { + info.setOrderTopic(false); + List messageQueueList = info.getMessageQueueList(); + LogicalQueuesInfo logicalQueueInfo = route.getLogicalQueuesInfo(); + for (Map.Entry> entry : logicalQueueInfo.entrySet()) { + boolean someWritable = false; + for (LogicalQueueRouteData logicalQueueRouteData : entry.getValue()) { + if (logicalQueueRouteData.isWritable()) { + someWritable = true; + break; + } + } + if (!someWritable) { + continue; + } + MessageQueue mq = new MessageQueue(); + mq.setQueueId(entry.getKey()); + mq.setBrokerName(MixAll.LOGICAL_QUEUE_MOCK_BROKER_NAME); + mq.setTopic(topic); + messageQueueList.add(mq); + } + Collections.sort(messageQueueList, new Comparator() { + @Override public int compare(MessageQueue o1, MessageQueue o2) { + return MixAll.compareInteger(o1.getQueueId(), o2.getQueueId()); + } + }); } else { List qds = route.getQueueDatas(); Collections.sort(qds); @@ -210,6 +239,27 @@ public static TopicPublishInfo topicRouteData2TopicPublishInfo(final String topi public static Set topicRouteData2TopicSubscribeInfo(final String topic, final TopicRouteData route) { Set mqList = new HashSet(); + if (route.getLogicalQueuesInfo() != null) { + LogicalQueuesInfo logicalQueueInfo = route.getLogicalQueuesInfo(); + for (Map.Entry> entry : logicalQueueInfo.entrySet()) { + boolean someReadable = false; + for (LogicalQueueRouteData logicalQueueRouteData : entry.getValue()) { + if (logicalQueueRouteData.isReadable()) { + someReadable = true; + break; + } + } + if (!someReadable) { + continue; + } + MessageQueue mq = new MessageQueue(); + mq.setQueueId(entry.getKey()); + mq.setBrokerName(MixAll.LOGICAL_QUEUE_MOCK_BROKER_NAME); + mq.setTopic(topic); + mqList.add(mq); + } + return mqList; + } List qds = route.getQueueDatas(); for (QueueData qd : qds) { if (PermName.isReadable(qd.getPerm())) { @@ -606,6 +656,11 @@ private void uploadFilterClassSource() { public boolean updateTopicRouteInfoFromNameServer(final String topic, boolean isDefault, DefaultMQProducer defaultMQProducer) { + return this.updateTopicRouteInfoFromNameServer(topic, isDefault, defaultMQProducer, null); + } + + public boolean updateTopicRouteInfoFromNameServer(final String topic, boolean isDefault, + DefaultMQProducer defaultMQProducer, Set logicalQueueIdsFilter) { try { if (this.lockNamesrv.tryLock(LOCK_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS)) { try { @@ -621,7 +676,7 @@ public boolean updateTopicRouteInfoFromNameServer(final String topic, boolean is } } } else { - topicRouteData = this.mQClientAPIImpl.getTopicRouteInfoFromNameServer(topic, 1000 * 3); + topicRouteData = this.mQClientAPIImpl.getTopicRouteInfoFromNameServer(topic, 1000 * 3, true, logicalQueueIdsFilter); } if (topicRouteData != null) { TopicRouteData old = this.topicRouteTable.get(topic); @@ -633,7 +688,26 @@ public boolean updateTopicRouteInfoFromNameServer(final String topic, boolean is } if (changed) { - TopicRouteData cloneTopicRouteData = topicRouteData.cloneTopicRouteData(); + TopicRouteData cloneTopicRouteData = new TopicRouteData(topicRouteData); + if (logicalQueueIdsFilter != null && cloneTopicRouteData.getLogicalQueuesInfo() != null) { + TopicRouteData curTopicRouteData = this.topicRouteTable.get(topic); + if (curTopicRouteData != null) { + LogicalQueuesInfo curLogicalQueuesInfo = curTopicRouteData.getLogicalQueuesInfo(); + if (curLogicalQueuesInfo != null) { + LogicalQueuesInfo cloneLogicalQueuesInfo = cloneTopicRouteData.getLogicalQueuesInfo(); + curLogicalQueuesInfo.readLock().lock(); + try { + for (Entry> entry : curLogicalQueuesInfo.entrySet()) { + if (!cloneLogicalQueuesInfo.containsKey(entry.getKey())) { + cloneLogicalQueuesInfo.put(entry.getKey(), entry.getValue()); + } + } + } finally { + curLogicalQueuesInfo.readLock().unlock(); + } + } + } + } for (BrokerData bd : topicRouteData.getBrokerDatas()) { this.brokerAddrTable.put(bd.getBrokerName(), bd.getBrokerAddrs()); @@ -791,8 +865,15 @@ private void uploadFilterClassToAllFilterServer(final String consumerGroup, fina private boolean topicRouteDataIsChange(TopicRouteData olddata, TopicRouteData nowdata) { if (olddata == null || nowdata == null) return true; - TopicRouteData old = olddata.cloneTopicRouteData(); - TopicRouteData now = nowdata.cloneTopicRouteData(); + LogicalQueuesInfo oldLogicalQueuesInfo = olddata.getLogicalQueuesInfo(); + LogicalQueuesInfo newLogicalQueuesInfo = nowdata.getLogicalQueuesInfo(); + if (oldLogicalQueuesInfo != null && newLogicalQueuesInfo != null) { + return oldLogicalQueuesInfo.keySet().equals(newLogicalQueuesInfo.keySet()); + } else if (oldLogicalQueuesInfo != null || newLogicalQueuesInfo != null) { + return true; + } + TopicRouteData old = new TopicRouteData(olddata); + TopicRouteData now = new TopicRouteData(nowdata); Collections.sort(old.getQueueDatas()); Collections.sort(old.getBrokerDatas()); Collections.sort(now.getQueueDatas()); @@ -814,6 +895,10 @@ private boolean isNeedUpdateTopicRouteInfo(final String topic) { } } + if (result) { + return true; + } + { Iterator> it = this.consumerTable.entrySet().iterator(); while (it.hasNext() && !result) { @@ -1280,4 +1365,13 @@ public NettyClientConfig getNettyClientConfig() { public ClientConfig getClientConfig() { return clientConfig; } + + public TopicRouteData queryTopicRouteData(String topic) { + TopicRouteData data = this.getAnExistTopicRouteData(topic); + if (data == null) { + this.updateTopicRouteInfoFromNameServer(topic); + data = this.getAnExistTopicRouteData(topic); + } + return data; + } } diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java index fac3ed3561f..8802a9cfd02 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java @@ -16,12 +16,16 @@ */ package org.apache.rocketmq.client.impl.producer; +import com.alibaba.fastjson.JSON; +import com.google.common.base.Objects; import java.io.IOException; import java.net.UnknownHostException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Random; import java.util.Set; import java.util.Timer; @@ -41,6 +45,7 @@ import org.apache.rocketmq.client.common.ClientErrorCode; import org.apache.rocketmq.client.exception.MQBrokerException; import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.client.exception.MQRedirectException; import org.apache.rocketmq.client.exception.RequestTimeoutException; import org.apache.rocketmq.client.hook.CheckForbiddenContext; import org.apache.rocketmq.client.hook.CheckForbiddenHook; @@ -62,6 +67,7 @@ import org.apache.rocketmq.client.producer.RequestResponseFuture; import org.apache.rocketmq.client.producer.SendCallback; import org.apache.rocketmq.client.producer.SendResult; +import org.apache.rocketmq.client.producer.SendResultForLogicalQueue; import org.apache.rocketmq.client.producer.SendStatus; import org.apache.rocketmq.client.producer.TransactionCheckListener; import org.apache.rocketmq.client.producer.TransactionListener; @@ -86,6 +92,9 @@ import org.apache.rocketmq.common.protocol.header.CheckTransactionStateRequestHeader; import org.apache.rocketmq.common.protocol.header.EndTransactionRequestHeader; import org.apache.rocketmq.common.protocol.header.SendMessageRequestHeader; +import org.apache.rocketmq.common.protocol.route.LogicalQueueRouteData; +import org.apache.rocketmq.common.protocol.route.LogicalQueuesInfo; +import org.apache.rocketmq.common.protocol.route.TopicRouteData; import org.apache.rocketmq.common.sysflag.MessageSysFlag; import org.apache.rocketmq.common.utils.CorrelationIdUtil; import org.apache.rocketmq.logging.InternalLogger; @@ -96,6 +105,8 @@ import org.apache.rocketmq.remoting.exception.RemotingTimeoutException; import org.apache.rocketmq.remoting.exception.RemotingTooMuchRequestException; +; + public class DefaultMQProducerImpl implements MQProducerInner { private final InternalLogger log = ClientLogger.getLog(); private final Random random = new Random(); @@ -502,7 +513,7 @@ public void send(Message msg, * * @param msg * @param sendCallback - * @param timeout the sendCallback will be invoked at most time + * @param timeout the sendCallback will be invoked at most time * @throws RejectedExecutionException */ @Deprecated @@ -718,6 +729,38 @@ private SendResult sendKernelImpl(final Message msg, final SendCallback sendCallback, final TopicPublishInfo topicPublishInfo, final long timeout) throws MQClientException, RemotingException, MQBrokerException, InterruptedException { + if (MixAll.LOGICAL_QUEUE_MOCK_BROKER_NAME.equals(mq.getBrokerName())) { + LogicalQueueSendContext logicalQueueContext = new LogicalQueueSendContext(msg, mq, communicationMode, sendCallback, topicPublishInfo, timeout); + while (true) { + try { + SendResult sendResult = this.sendKernelImplWithoutRetry(msg, + logicalQueueContext.getModifiedMessageQueue(), + communicationMode, + logicalQueueContext.wrapSendCallback(), + topicPublishInfo, + timeout); + return logicalQueueContext.wrapSendResult(sendResult); + } catch (MQRedirectException e) { + if (!logicalQueueContext.shouldRetry(e)) { + throw new MQBrokerException(ResponseCode.SYSTEM_ERROR, "redirect"); + } + } catch (RemotingException e) { + if (!logicalQueueContext.shouldRetry(e)) { + throw e; + } + } + } + } else { + return sendKernelImplWithoutRetry(msg, mq, communicationMode, sendCallback, topicPublishInfo, timeout); + } + } + + private SendResult sendKernelImplWithoutRetry(final Message msg, + final MessageQueue mq, + final CommunicationMode communicationMode, + SendCallback sendCallback, + final TopicPublishInfo topicPublishInfo, + final long timeout) throws MQClientException, RemotingException, MQBrokerException, InterruptedException { long beginStartTime = System.currentTimeMillis(); String brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(mq.getBrokerName()); if (null == brokerAddr) { @@ -754,6 +797,10 @@ private SendResult sendKernelImpl(final Message msg, sysFlag |= MessageSysFlag.TRANSACTION_PREPARED_TYPE; } + if (!CommunicationMode.ONEWAY.equals(communicationMode)) { + sysFlag |= MessageSysFlag.LOGICAL_QUEUE_FLAG; + } + if (hasCheckForbiddenHook()) { CheckForbiddenContext checkForbiddenContext = new CheckForbiddenContext(); checkForbiddenContext.setNameSrvAddr(this.defaultMQProducer.getNamesrvAddr()); @@ -1006,6 +1053,7 @@ public void doExecuteEndTransactionHook(Message msg, String msgId, String broker executeEndTransactionHook(context); } } + /** * DEFAULT ONEWAY ------------------------------------------------------- */ @@ -1058,7 +1106,7 @@ public void send(Message msg, MessageQueue mq, SendCallback sendCallback) * @param msg * @param mq * @param sendCallback - * @param timeout the sendCallback will be invoked at most time + * @param timeout the sendCallback will be invoked at most time * @throws MQClientException * @throws RemotingException * @throws InterruptedException @@ -1188,7 +1236,7 @@ public void send(Message msg, MessageQueueSelector selector, Object arg, SendCal * @param selector * @param arg * @param sendCallback - * @param timeout the sendCallback will be invoked at most time + * @param timeout the sendCallback will be invoked at most time * @throws MQClientException * @throws RemotingException * @throws InterruptedException @@ -1528,7 +1576,8 @@ public void onException(Throwable e) { } } - private Message waitResponse(Message msg, long timeout, RequestResponseFuture requestResponseFuture, long cost) throws InterruptedException, RequestTimeoutException, MQClientException { + private Message waitResponse(Message msg, long timeout, RequestResponseFuture requestResponseFuture, + long cost) throws InterruptedException, RequestTimeoutException, MQClientException { Message responseMessage = requestResponseFuture.waitResponseMessage(timeout - cost); if (responseMessage == null) { if (requestResponseFuture.isSendRequestOk()) { @@ -1644,4 +1693,178 @@ public void setSendLatencyFaultEnable(final boolean sendLatencyFaultEnable) { public DefaultMQProducer getDefaultMQProducer() { return defaultMQProducer; } + + private class LogicalQueueSendContext implements SendCallback { + private final Message msg; + private final MessageQueue mq; + private final CommunicationMode communicationMode; + private final SendCallback sendCallback; + private final TopicPublishInfo topicPublishInfo; + private final long timeout; + + private volatile LogicalQueuesInfo logicalQueuesInfo; + private volatile LogicalQueueRouteData writableQueueRouteData; + + private final AtomicInteger retry = new AtomicInteger(); + + public LogicalQueueSendContext(Message msg, MessageQueue mq, + CommunicationMode communicationMode, SendCallback sendCallback, + TopicPublishInfo topicPublishInfo, long timeout) { + this.msg = msg; + this.mq = mq; + this.communicationMode = communicationMode; + this.sendCallback = sendCallback; + this.topicPublishInfo = topicPublishInfo; + this.timeout = timeout; + + if (topicPublishInfo == null) { + topicPublishInfo = DefaultMQProducerImpl.this.tryToFindTopicPublishInfo(mq.getTopic()); + } + if (topicPublishInfo != null) { + this.logicalQueuesInfo = topicPublishInfo.getTopicRouteData().getLogicalQueuesInfo(); + } else { + this.logicalQueuesInfo = null; + } + } + + private boolean notUsingLogicalQueue() { + return !Objects.equal(mq.getBrokerName(), MixAll.LOGICAL_QUEUE_MOCK_BROKER_NAME) || this.logicalQueuesInfo == null; + } + + public MessageQueue getModifiedMessageQueue() throws MQClientException { + if (this.notUsingLogicalQueue()) { + return this.mq; + } + this.writableQueueRouteData = getWritableQueueRouteData(); + MessageQueue mq = new MessageQueue(this.mq); + mq.setBrokerName(writableQueueRouteData.getBrokerName()); + mq.setQueueId(writableQueueRouteData.getQueueId()); + return mq; + } + + private LogicalQueueRouteData getWritableQueueRouteData() throws MQClientException { + this.logicalQueuesInfo.readLock().lock(); + try { + List queueRouteDataList = logicalQueuesInfo.get(mq.getQueueId()); + if (queueRouteDataList == null || queueRouteDataList.size() == 0) { + throw new MQClientException(String.format(Locale.ENGLISH, "send to a logical queue %d but no queue route data found", mq.getQueueId()), null); + } + // usually writable queue is placed in the last position, or second last when queue migrating + for (int i = queueRouteDataList.size() - 1; i >= 0; i--) { + LogicalQueueRouteData queueRouteData = queueRouteDataList.get(i); + if (queueRouteData.isWritable()) { + return queueRouteData; + } + } + throw new MQClientException(String.format(Locale.ENGLISH, "send to a logical queue %d but no writable queue route data found", mq.getQueueId()), null); + } finally { + this.logicalQueuesInfo.readLock().unlock(); + } + } + + @Override public void onSuccess(SendResult sendResult) { + this.sendCallback.onSuccess(this.wrapSendResult(sendResult)); + } + + @Override public void onException(Throwable t) { + if (this.shouldRetry(t)) { + try { + DefaultMQProducerImpl.this.sendKernelImplWithoutRetry(msg, this.getModifiedMessageQueue(), communicationMode, this, topicPublishInfo, timeout); + return; + } catch (Exception e) { + t = e; + } + } + if (t instanceof MQRedirectException) { + t = new MQBrokerException(ResponseCode.SYSTEM_ERROR, "redirect"); + } + this.sendCallback.onException(t); + } + + private void handleRedirectException(MQRedirectException re) { + byte[] responseBody = re.getBody(); + log.info("LogicalQueueContext.processResponseBody got redirect {}: {}", this.writableQueueRouteData, responseBody != null ? new String(responseBody, MessageDecoder.CHARSET_UTF8) : null); + + try { + List newQueueRouteDataList = JSON.parseObject(responseBody, MixAll.TYPE_LIST_LOGICAL_QUEUE_ROUTE_DATA); + this.logicalQueuesInfo.updateLogicalQueueRouteDataList(this.mq.getQueueId(), newQueueRouteDataList); + } catch (Exception e) { + log.warn("LogicalQueueContext.processResponseBody {} update exception, fallback to updateTopicRouteInfoFromNameServer", this.writableQueueRouteData, e); + DefaultMQProducerImpl.this.mQClientFactory.updateTopicRouteInfoFromNameServer(this.mq.getTopic(), false, null, Collections.singleton(mq.getQueueId())); + TopicRouteData topicRouteData = DefaultMQProducerImpl.this.mQClientFactory.getAnExistTopicRouteData(mq.getTopic()); + if (topicRouteData != null) { + this.logicalQueuesInfo = topicRouteData.getLogicalQueuesInfo(); + } else { + this.logicalQueuesInfo = null; + } + } + } + + public SendCallback wrapSendCallback() { + if (this.notUsingLogicalQueue()) { + return this.sendCallback; + } + if (!CommunicationMode.ASYNC.equals(this.communicationMode)) { + return this.sendCallback; + } + return this; + } + + public boolean shouldRetry(Throwable t) { + this.incrRetry(); + if (this.exceedMaxRetry()) { + log.warn("retry {} too many times: {}", this.retry.get(), this.writableQueueRouteData); + return false; + } + if (!this.writableQueueRouteData.isWritable()) { + log.warn("no writable queue: {}", this.writableQueueRouteData); + return false; + } + if (t instanceof MQRedirectException) { + this.handleRedirectException((MQRedirectException) t); + return true; + } + return !(t instanceof RemotingException) || this.handleRemotingException((RemotingException) t); + } + + public boolean exceedMaxRetry() { + return this.retry.get() >= 3; + } + + public void incrRetry() { + this.retry.incrementAndGet(); + } + + public SendResult wrapSendResult(SendResult sendResult) { + if (sendResult == null) { + return null; + } + SendResultForLogicalQueue newSendResult = new SendResultForLogicalQueue(sendResult, this.writableQueueRouteData.getLogicalQueueIndex()); + long queueOffset = newSendResult.getQueueOffset(); + if (queueOffset >= 0) { + newSendResult.setQueueOffset(LogicalQueueSendContext.this.writableQueueRouteData.toLogicalQueueOffset(queueOffset)); + } + return newSendResult; + } + + public boolean handleRemotingException(RemotingException e) { + if (e instanceof RemotingTooMuchRequestException) { + return false; + } + DefaultMQProducerImpl.this.mQClientFactory.updateTopicRouteInfoFromNameServer(this.mq.getTopic(), false, null, Collections.singleton(mq.getQueueId())); + this.logicalQueuesInfo = DefaultMQProducerImpl.this.getTopicPublishInfoTable().get(mq.getTopic()).getTopicRouteData().getLogicalQueuesInfo(); + LogicalQueueRouteData writableQueueRouteData; + try { + writableQueueRouteData = this.getWritableQueueRouteData(); + } catch (MQClientException ce) { + log.warn("getWritableQueueRouteData exception: {}", this.logicalQueuesInfo.get(mq.getQueueId()), ce); + return false; + } + if (Objects.equal(this.writableQueueRouteData.getMessageQueue(), writableQueueRouteData.getMessageQueue()) && writableQueueRouteData.isWritable()) { + // still same MessageQueue and still writable, no need to retry + return false; + } + return true; + } + } } diff --git a/client/src/main/java/org/apache/rocketmq/client/latency/MQFaultStrategy.java b/client/src/main/java/org/apache/rocketmq/client/latency/MQFaultStrategy.java index ea3d07e6d0f..c74b3cd73df 100644 --- a/client/src/main/java/org/apache/rocketmq/client/latency/MQFaultStrategy.java +++ b/client/src/main/java/org/apache/rocketmq/client/latency/MQFaultStrategy.java @@ -19,8 +19,8 @@ import org.apache.rocketmq.client.impl.producer.TopicPublishInfo; import org.apache.rocketmq.client.log.ClientLogger; -import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.logging.InternalLogger; public class MQFaultStrategy { private final static InternalLogger log = ClientLogger.getLog(); diff --git a/client/src/main/java/org/apache/rocketmq/client/producer/SendResult.java b/client/src/main/java/org/apache/rocketmq/client/producer/SendResult.java index 80948830fba..dd7ea1cdc5f 100644 --- a/client/src/main/java/org/apache/rocketmq/client/producer/SendResult.java +++ b/client/src/main/java/org/apache/rocketmq/client/producer/SendResult.java @@ -28,6 +28,7 @@ public class SendResult { private String offsetMsgId; private String regionId; private boolean traceOn = true; + private byte[] rawRespBody; public SendResult() { } @@ -130,4 +131,12 @@ public String toString() { return "SendResult [sendStatus=" + sendStatus + ", msgId=" + msgId + ", offsetMsgId=" + offsetMsgId + ", messageQueue=" + messageQueue + ", queueOffset=" + queueOffset + "]"; } + + public void setRawRespBody(byte[] body) { + this.rawRespBody = body; + } + + public byte[] getRawRespBody() { + return rawRespBody; + } } diff --git a/client/src/main/java/org/apache/rocketmq/client/producer/SendResultForLogicalQueue.java b/client/src/main/java/org/apache/rocketmq/client/producer/SendResultForLogicalQueue.java new file mode 100644 index 00000000000..09cd469d916 --- /dev/null +++ b/client/src/main/java/org/apache/rocketmq/client/producer/SendResultForLogicalQueue.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.client.producer; + +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.message.MessageQueue; + +public class SendResultForLogicalQueue extends SendResult { + private final String origBrokerName; + private final int origQueueId; + + public SendResultForLogicalQueue(SendResult sendResult, int logicalQueueIdx) { + super(sendResult.getSendStatus(), sendResult.getMsgId(), sendResult.getOffsetMsgId(), new MessageQueue(sendResult.getMessageQueue().getTopic(), MixAll.LOGICAL_QUEUE_MOCK_BROKER_NAME, logicalQueueIdx), sendResult.getQueueOffset()); + this.origBrokerName = sendResult.getMessageQueue().getBrokerName(); + this.origQueueId = sendResult.getMessageQueue().getQueueId(); + } + + public String getOrigBrokerName() { + return origBrokerName; + } + + public int getOrigQueueId() { + return origQueueId; + } + + @Override public String toString() { + return "SendResultForLogicalQueue{" + + "origBrokerName='" + origBrokerName + '\'' + + ", origQueueId=" + origQueueId + + "} " + super.toString(); + } +} diff --git a/client/src/main/java/org/apache/rocketmq/client/trace/hook/ConsumeMessageOpenTracingHookImpl.java b/client/src/main/java/org/apache/rocketmq/client/trace/hook/ConsumeMessageOpenTracingHookImpl.java index 28fccae06f8..fe97c773949 100644 --- a/client/src/main/java/org/apache/rocketmq/client/trace/hook/ConsumeMessageOpenTracingHookImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/trace/hook/ConsumeMessageOpenTracingHookImpl.java @@ -51,7 +51,7 @@ public void consumeMessageBefore(ConsumeMessageContext context) { if (context == null || context.getMsgList() == null || context.getMsgList().isEmpty()) { return; } - List spanList = new ArrayList<>(); + List spanList = new ArrayList(); for (MessageExt msg : context.getMsgList()) { if (msg == null) { continue; diff --git a/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumerTest.java b/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumerTest.java index ba2a6a27d45..9b9b7e4e381 100644 --- a/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumerTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumerTest.java @@ -95,7 +95,9 @@ public class DefaultLitePullConsumerTest { @Before public void init() throws Exception { ConcurrentMap factoryTable = (ConcurrentMap) FieldUtils.readDeclaredField(MQClientManager.getInstance(), "factoryTable", true); - factoryTable.forEach((s, instance) -> instance.shutdown()); + for (MQClientInstance instance : factoryTable.values()) { + instance.shutdown(); + } factoryTable.clear(); Field field = MQClientInstance.class.getDeclaredField("rebalanceService"); diff --git a/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultMQPullConsumerLogicalQueueTest.java b/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultMQPullConsumerLogicalQueueTest.java new file mode 100644 index 00000000000..15ec5642662 --- /dev/null +++ b/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultMQPullConsumerLogicalQueueTest.java @@ -0,0 +1,248 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.client.consumer; + +import com.alibaba.fastjson.JSON; +import com.google.common.collect.ImmutableList; +import com.google.common.util.concurrent.SettableFuture; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import org.apache.commons.lang3.reflect.FieldUtils; +import org.apache.rocketmq.client.ClientConfig; +import org.apache.rocketmq.client.exception.MQRedirectException; +import org.apache.rocketmq.client.impl.CommunicationMode; +import org.apache.rocketmq.client.impl.FindBrokerResult; +import org.apache.rocketmq.client.impl.MQClientAPIImpl; +import org.apache.rocketmq.client.impl.MQClientManager; +import org.apache.rocketmq.client.impl.consumer.PullAPIWrapper; +import org.apache.rocketmq.client.impl.consumer.PullResultExt; +import org.apache.rocketmq.client.impl.factory.MQClientInstance; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.common.protocol.header.PullMessageRequestHeader; +import org.apache.rocketmq.common.protocol.route.BrokerData; +import org.apache.rocketmq.common.protocol.route.LogicalQueueRouteData; +import org.apache.rocketmq.common.protocol.route.LogicalQueuesInfo; +import org.apache.rocketmq.common.protocol.route.MessageQueueRouteState; +import org.apache.rocketmq.common.protocol.route.QueueData; +import org.apache.rocketmq.common.protocol.route.TopicRouteData; +import org.assertj.core.util.Lists; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.ArgumentMatchers; +import org.mockito.Mock; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.junit.MockitoJUnitRunner; +import org.mockito.stubbing.Answer; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.isNull; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class DefaultMQPullConsumerLogicalQueueTest { + private MQClientInstance mQClientFactory; + @Mock + private MQClientAPIImpl mQClientAPIImpl; + private DefaultMQPullConsumer pullConsumer; + private String topic; + private static final String cluster = "DefaultCluster"; + private static final String broker1Name = "BrokerA"; + private static final String broker1Addr = "127.0.0.2:10911"; + private static final String broker2Name = "BrokerB"; + private static final String broker2Addr = "127.0.0.3:10911"; + + @Before + public void init() throws Exception { + topic = "FooBar" + System.nanoTime(); + + mQClientFactory = spy(MQClientManager.getInstance().getOrCreateMQClientInstance(new ClientConfig())); + + FieldUtils.writeField(mQClientFactory, "mQClientAPIImpl", mQClientAPIImpl, true); + + pullConsumer = new DefaultMQPullConsumer("FooBarGroup" + System.nanoTime()); + pullConsumer.setNamesrvAddr("127.0.0.1:9876"); + pullConsumer.start(); + + PullAPIWrapper pullAPIWrapper = pullConsumer.getDefaultMQPullConsumerImpl().getPullAPIWrapper(); + FieldUtils.writeDeclaredField(pullAPIWrapper, "mQClientFactory", mQClientFactory, true); + + when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong(), anyBoolean(), ArgumentMatchers.>any())).thenReturn(createTopicRouteData()); + + doReturn(new FindBrokerResult(broker1Addr, false)).when(mQClientFactory).findBrokerAddressInSubscribe(eq(broker1Name), anyLong(), anyBoolean()); + doReturn(new FindBrokerResult(broker2Addr, false)).when(mQClientFactory).findBrokerAddressInSubscribe(eq(broker2Name), anyLong(), anyBoolean()); + } + + @After + public void terminate() { + pullConsumer.shutdown(); + } + + @Test + public void testStart_OffsetShouldNotNUllAfterStart() { + Assert.assertNotNull(pullConsumer.getOffsetStore()); + } + + @Test + public void testPullMessage_Success() throws Exception { + doAnswer(new Answer() { + @Override public PullResultExt answer(InvocationOnMock mock) throws Throwable { + PullMessageRequestHeader requestHeader = mock.getArgument(1); + return DefaultMQPullConsumerLogicalQueueTest.this.createPullResult(requestHeader, PullStatus.FOUND, Collections.singletonList(new MessageExt())); + } + }).when(mQClientAPIImpl).pullMessage(eq(broker1Addr), any(PullMessageRequestHeader.class), anyLong(), eq(CommunicationMode.SYNC), (PullCallback) isNull()); + + MessageQueue messageQueue = new MessageQueue(topic, MixAll.LOGICAL_QUEUE_MOCK_BROKER_NAME, 0); + PullResult pullResult = pullConsumer.pull(messageQueue, "*", 1024, 3); + assertThat(pullResult).isNotNull(); + assertThat(pullResult.getPullStatus()).isEqualTo(PullStatus.FOUND); + assertThat(pullResult.getNextBeginOffset()).isEqualTo(1024 + 1); + assertThat(pullResult.getMinOffset()).isEqualTo(123); + assertThat(pullResult.getMaxOffset()).isEqualTo(2048); + assertThat(pullResult.getMsgFoundList()).isEqualTo(Collections.emptyList()); + } + + @Test + public void testPullMessage_NotFound() throws Exception { + doAnswer(new Answer() { + @Override public PullResult answer(InvocationOnMock mock) throws Throwable { + PullMessageRequestHeader requestHeader = mock.getArgument(1); + return DefaultMQPullConsumerLogicalQueueTest.this.createPullResult(requestHeader, PullStatus.NO_NEW_MSG, new ArrayList()); + } + }).when(mQClientAPIImpl).pullMessage(eq(broker1Addr), any(PullMessageRequestHeader.class), anyLong(), eq(CommunicationMode.SYNC), (PullCallback) isNull()); + + MessageQueue messageQueue = new MessageQueue(topic, MixAll.LOGICAL_QUEUE_MOCK_BROKER_NAME, 0); + PullResult pullResult = pullConsumer.pull(messageQueue, "*", 1024, 3); + assertThat(pullResult.getPullStatus()).isEqualTo(PullStatus.NO_NEW_MSG); + } + + @Test + public void testPullMessageAsync_Success() throws Exception { + doAnswer(new Answer() { + @Override public PullResult answer(InvocationOnMock mock) throws Throwable { + PullMessageRequestHeader requestHeader = mock.getArgument(1); + PullResult pullResult = DefaultMQPullConsumerLogicalQueueTest.this.createPullResult(requestHeader, PullStatus.FOUND, Collections.singletonList(new MessageExt())); + + PullCallback pullCallback = mock.getArgument(4); + pullCallback.onSuccess(pullResult); + return null; + } + }).when(mQClientAPIImpl).pullMessage(eq(broker1Addr), any(PullMessageRequestHeader.class), anyLong(), eq(CommunicationMode.ASYNC), any(PullCallback.class)); + + final SettableFuture future = SettableFuture.create(); + MessageQueue messageQueue = new MessageQueue(topic, broker1Name, 0); + pullConsumer.pull(messageQueue, "*", 1024, 3, new PullCallback() { + @Override + public void onSuccess(PullResult pullResult) { + future.set(pullResult); + } + + @Override + public void onException(Throwable e) { + future.setException(e); + } + }); + PullResult pullResult = future.get(3, TimeUnit.SECONDS); + assertThat(pullResult).isNotNull(); + assertThat(pullResult.getPullStatus()).isEqualTo(PullStatus.FOUND); + assertThat(pullResult.getNextBeginOffset()).isEqualTo(1024 + 1); + assertThat(pullResult.getMinOffset()).isEqualTo(123); + assertThat(pullResult.getMaxOffset()).isEqualTo(2048); + assertThat(pullResult.getMsgFoundList()).isEqualTo(Collections.emptyList()); + } + + @Test + public void testPullMessageSync_Redirect() throws Exception { + doAnswer(new Answer() { + @Override public PullResult answer(InvocationOnMock mock) throws Throwable { + throw new MQRedirectException(JSON.toJSONBytes(ImmutableList.of( + new LogicalQueueRouteData(0, 0, new MessageQueue(topic, broker1Name, 0), MessageQueueRouteState.Expired, 0, 0, 0, 0, broker1Addr), + new LogicalQueueRouteData(0, 10, new MessageQueue(topic, broker2Name, 0), MessageQueueRouteState.Normal, 0, -1, -1, -1, broker2Addr) + ))); + } + }).when(mQClientAPIImpl).pullMessage(eq(broker1Addr), any(PullMessageRequestHeader.class), anyLong(), eq(CommunicationMode.SYNC), (PullCallback) isNull()); + doAnswer(new Answer() { + @Override public PullResult answer(InvocationOnMock mock) throws Throwable { + PullMessageRequestHeader requestHeader = mock.getArgument(1); + return DefaultMQPullConsumerLogicalQueueTest.this.createPullResult(requestHeader, PullStatus.FOUND, Collections.singletonList(new MessageExt())); + } + }).when(mQClientAPIImpl).pullMessage(eq(broker2Addr), any(PullMessageRequestHeader.class), anyLong(), eq(CommunicationMode.SYNC), (PullCallback) isNull()); + + MessageQueue messageQueue = new MessageQueue(topic, MixAll.LOGICAL_QUEUE_MOCK_BROKER_NAME, 0); + PullResult pullResult = pullConsumer.pull(messageQueue, "*", 1024, 3); + assertThat(pullResult).isNotNull(); + assertThat(pullResult.getPullStatus()).isEqualTo(PullStatus.FOUND); + assertThat(pullResult.getNextBeginOffset()).isEqualTo(1024 + 1); + assertThat(pullResult.getMinOffset()).isEqualTo(123 + 10); + assertThat(pullResult.getMaxOffset()).isEqualTo(2048 + 10); + assertThat(pullResult.getMsgFoundList()).isEqualTo(Collections.emptyList()); + } + + private TopicRouteData createTopicRouteData() { + TopicRouteData topicRouteData = new TopicRouteData(); + + topicRouteData.setFilterServerTable(new HashMap>()); + topicRouteData.setBrokerDatas(ImmutableList.of( + new BrokerData(cluster, broker1Name, new HashMap(Collections.singletonMap(MixAll.MASTER_ID, broker1Addr))), + new BrokerData(cluster, broker2Name, new HashMap(Collections.singletonMap(MixAll.MASTER_ID, broker2Addr))) + )); + + List queueDataList = new ArrayList(); + QueueData queueData; + queueData = new QueueData(); + queueData.setBrokerName(broker1Name); + queueData.setPerm(6); + queueData.setReadQueueNums(3); + queueData.setWriteQueueNums(4); + queueData.setTopicSysFlag(0); + queueDataList.add(queueData); + queueData = new QueueData(); + queueData.setBrokerName(broker2Name); + queueData.setPerm(6); + queueData.setReadQueueNums(3); + queueData.setWriteQueueNums(4); + queueData.setTopicSysFlag(0); + queueDataList.add(queueData); + topicRouteData.setQueueDatas(queueDataList); + + LogicalQueuesInfo info = new LogicalQueuesInfo(); + info.put(0, Lists.newArrayList(new LogicalQueueRouteData(0, 0, new MessageQueue(topic, broker1Name, 0), MessageQueueRouteState.Normal, 0, 0, 0, 0, broker1Addr))); + topicRouteData.setLogicalQueuesInfo(info); + return topicRouteData; + } + + private PullResultExt createPullResult(PullMessageRequestHeader requestHeader, PullStatus pullStatus, + List messageExtList) throws Exception { + return new PullResultExt(pullStatus, requestHeader.getQueueOffset() + messageExtList.size(), 123, 2048, messageExtList, 0, new byte[] {}); + } +} \ No newline at end of file diff --git a/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumerTest.java b/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumerTest.java index 7c3c501e054..abdd6bfb69d 100644 --- a/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumerTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumerTest.java @@ -100,7 +100,9 @@ public class DefaultMQPushConsumerTest { @Before public void init() throws Exception { ConcurrentMap factoryTable = (ConcurrentMap) FieldUtils.readDeclaredField(MQClientManager.getInstance(), "factoryTable", true); - factoryTable.forEach((s, instance) -> instance.shutdown()); + for (MQClientInstance instance : factoryTable.values()) { + instance.shutdown(); + } factoryTable.clear(); consumerGroup = "FooBarGroup" + System.currentTimeMillis(); @@ -121,12 +123,15 @@ public ConsumeConcurrentlyStatus consumeMessage(List msgs, // suppress updateTopicRouteInfoFromNameServer pushConsumer.changeInstanceNameToPID(); - mQClientFactory = spy(MQClientManager.getInstance().getOrCreateMQClientInstance(pushConsumer, (RPCHook) FieldUtils.readDeclaredField(pushConsumerImpl, "rpcHook", true))); + mQClientFactory = MQClientManager.getInstance().getOrCreateMQClientInstance(pushConsumer, (RPCHook) FieldUtils.readDeclaredField(pushConsumerImpl, "rpcHook", true)); + FieldUtils.writeDeclaredField(mQClientFactory, "mQClientAPIImpl", mQClientAPIImpl, true); + mQClientFactory = spy(mQClientFactory); factoryTable.put(pushConsumer.buildMQClientId(), mQClientFactory); doReturn(false).when(mQClientFactory).updateTopicRouteInfoFromNameServer(anyString()); doReturn(null).when(mQClientFactory).queryAssignment(anyString(), anyString(), anyString(), any(MessageModel.class), anyInt()); + doReturn(new FindBrokerResult("127.0.0.1:10911", false)).when(mQClientFactory).findBrokerAddressInSubscribe(anyString(), anyLong(), anyBoolean()); - rebalanceImpl = spy(pushConsumer.getDefaultMQPushConsumerImpl().getRebalanceImpl()); + rebalanceImpl = spy(pushConsumerImpl.getRebalanceImpl()); doReturn(123L).when(rebalanceImpl).computePullFromWhereWithException(any(MessageQueue.class)); Field field = DefaultMQPushConsumerImpl.class.getDeclaredField("rebalanceImpl"); field.setAccessible(true); @@ -136,25 +141,16 @@ public ConsumeConcurrentlyStatus consumeMessage(List msgs, field.setAccessible(true); field.set(null, true); - pushConsumer.subscribe(topic, "*"); - pushConsumer.start(); - - field = DefaultMQPushConsumerImpl.class.getDeclaredField("mQClientFactory"); - field.setAccessible(true); - field.set(pushConsumerImpl, mQClientFactory); + Set messageQueueSet = new HashSet(); + messageQueueSet.add(createPullRequest().getMessageQueue()); + pushConsumerImpl.updateTopicSubscribeInfo(topic, messageQueueSet); - field = MQClientInstance.class.getDeclaredField("mQClientAPIImpl"); - field.setAccessible(true); - field.set(mQClientFactory, mQClientAPIImpl); + pushConsumerImpl.setmQClientFactory(mQClientFactory); pullAPIWrapper = spy(new PullAPIWrapper(mQClientFactory, consumerGroup, false)); - field = DefaultMQPushConsumerImpl.class.getDeclaredField("pullAPIWrapper"); - field.setAccessible(true); - field.set(pushConsumerImpl, pullAPIWrapper); + FieldUtils.writeDeclaredField(pushConsumerImpl, "pullAPIWrapper", pullAPIWrapper, true); - mQClientFactory.registerConsumer(consumerGroup, pushConsumerImpl); - - when(mQClientFactory.getMQClientAPIImpl().pullMessage(anyString(), any(PullMessageRequestHeader.class), + when(mQClientAPIImpl.pullMessage(anyString(), any(PullMessageRequestHeader.class), anyLong(), any(CommunicationMode.class), nullable(PullCallback.class))) .thenAnswer(new Answer() { @Override @@ -175,10 +171,10 @@ public PullResult answer(InvocationOnMock mock) throws Throwable { } }); - doReturn(new FindBrokerResult("127.0.0.1:10911", false)).when(mQClientFactory).findBrokerAddressInSubscribe(anyString(), anyLong(), anyBoolean()); - Set messageQueueSet = new HashSet(); - messageQueueSet.add(createPullRequest().getMessageQueue()); - pushConsumer.getDefaultMQPushConsumerImpl().updateTopicSubscribeInfo(topic, messageQueueSet); + pushConsumer.subscribe(topic, "*"); + pushConsumer.start(); + + mQClientFactory.registerConsumer(consumerGroup, pushConsumerImpl); } @After @@ -194,7 +190,7 @@ public void testStart_OffsetShouldNotNUllAfterStart() { @Test public void testPullMessage_Success() throws InterruptedException, RemotingException, MQBrokerException { final CountDownLatch countDownLatch = new CountDownLatch(1); - final AtomicReference messageAtomic = new AtomicReference<>(); + final AtomicReference messageAtomic = new AtomicReference(); pushConsumer.getDefaultMQPushConsumerImpl().setConsumeMessageService(new ConsumeMessageConcurrentlyService(pushConsumer.getDefaultMQPushConsumerImpl(), new MessageListenerConcurrently() { @Override public ConsumeConcurrentlyStatus consumeMessage(List msgs, @@ -217,7 +213,7 @@ public ConsumeConcurrentlyStatus consumeMessage(List msgs, @Test(timeout = 20000) public void testPullMessage_SuccessWithOrderlyService() throws Exception { final CountDownLatch countDownLatch = new CountDownLatch(1); - final AtomicReference messageAtomic = new AtomicReference<>(); + final AtomicReference messageAtomic = new AtomicReference(); MessageListenerOrderly listenerOrderly = new MessageListenerOrderly() { @Override @@ -355,11 +351,14 @@ public void testPullMessage_ExceptionOccursWhenComputePullFromWhere() throws MQC final CountDownLatch countDownLatch = new CountDownLatch(1); final MessageExt[] messageExts = new MessageExt[1]; pushConsumer.getDefaultMQPushConsumerImpl().setConsumeMessageService( - new ConsumeMessageConcurrentlyService(pushConsumer.getDefaultMQPushConsumerImpl(), - (msgs, context) -> { - messageExts[0] = msgs.get(0); - return null; - })); + new ConsumeMessageConcurrentlyService(pushConsumer.getDefaultMQPushConsumerImpl(), + new MessageListenerConcurrently() { + @Override public ConsumeConcurrentlyStatus consumeMessage(List msgs, + ConsumeConcurrentlyContext context) { + messageExts[0] = msgs.get(0); + return null; + } + })); pushConsumer.getDefaultMQPushConsumerImpl().setConsumeOrderly(true); PullMessageService pullMessageService = mQClientFactory.getPullMessageService(); diff --git a/client/src/test/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageConcurrentlyServiceTest.java b/client/src/test/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageConcurrentlyServiceTest.java index e8feb80dd99..0faf72469cd 100644 --- a/client/src/test/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageConcurrentlyServiceTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageConcurrentlyServiceTest.java @@ -149,7 +149,7 @@ public PullResult answer(InvocationOnMock mock) throws Throwable { @Test public void testPullMessage_ConsumeSuccess() throws InterruptedException, RemotingException, MQBrokerException, NoSuchFieldException,Exception { final CountDownLatch countDownLatch = new CountDownLatch(1); - final AtomicReference messageAtomic = new AtomicReference<>(); + final AtomicReference messageAtomic = new AtomicReference(); ConsumeMessageConcurrentlyService normalServie = new ConsumeMessageConcurrentlyService(pushConsumer.getDefaultMQPushConsumerImpl(), new MessageListenerConcurrently() { @Override diff --git a/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerLogicalQueueTest.java b/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerLogicalQueueTest.java new file mode 100644 index 00000000000..12d5cbaab82 --- /dev/null +++ b/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerLogicalQueueTest.java @@ -0,0 +1,311 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.client.producer; + +import com.alibaba.fastjson.JSON; +import com.google.common.collect.ImmutableList; +import com.google.common.util.concurrent.SettableFuture; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Set; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import org.apache.commons.lang3.reflect.FieldUtils; +import org.apache.rocketmq.client.ClientConfig; +import org.apache.rocketmq.client.exception.MQBrokerException; +import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.client.exception.MQRedirectException; +import org.apache.rocketmq.client.hook.SendMessageContext; +import org.apache.rocketmq.client.impl.CommunicationMode; +import org.apache.rocketmq.client.impl.MQClientAPIImpl; +import org.apache.rocketmq.client.impl.MQClientManager; +import org.apache.rocketmq.client.impl.factory.MQClientInstance; +import org.apache.rocketmq.client.impl.producer.DefaultMQProducerImpl; +import org.apache.rocketmq.client.impl.producer.TopicPublishInfo; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.message.Message; +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.common.protocol.header.SendMessageRequestHeader; +import org.apache.rocketmq.common.protocol.route.BrokerData; +import org.apache.rocketmq.common.protocol.route.LogicalQueueRouteData; +import org.apache.rocketmq.common.protocol.route.LogicalQueuesInfo; +import org.apache.rocketmq.common.protocol.route.MessageQueueRouteState; +import org.apache.rocketmq.common.protocol.route.QueueData; +import org.apache.rocketmq.common.protocol.route.TopicRouteData; +import org.apache.rocketmq.remoting.exception.RemotingConnectException; +import org.assertj.core.api.ThrowableAssert; +import org.assertj.core.util.Lists; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.ArgumentMatchers; +import org.mockito.Mock; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.junit.MockitoJUnitRunner; +import org.mockito.stubbing.Answer; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.isNull; +import static org.mockito.ArgumentMatchers.nullable; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class DefaultMQProducerLogicalQueueTest { + private MQClientInstance mQClientFactory; + @Mock + private MQClientAPIImpl mQClientAPIImpl; + + private DefaultMQProducer producer; + private Message message; + private String topic; + + private MessageQueue messageQueue; + + private static final String cluster = "DefaultCluster"; + private static final String broker1Name = "broker1"; + private static final String broker2Name = "broker2"; + private static final String broker1Addr = "127.0.0.2:10911"; + private static final String broker2Addr = "127.0.0.3:10911"; + + @Before + public void init() throws Exception { + topic = "Foobar" + System.nanoTime(); + messageQueue = new MessageQueue(topic, MixAll.LOGICAL_QUEUE_MOCK_BROKER_NAME, 0); + + ConcurrentMap factoryTable = (ConcurrentMap) FieldUtils.readDeclaredField(MQClientManager.getInstance(), "factoryTable", true); + for (MQClientInstance instance : factoryTable.values()) { + instance.shutdown(); + } + factoryTable.clear(); + + mQClientFactory = spy(MQClientManager.getInstance().getOrCreateMQClientInstance(new ClientConfig())); + factoryTable.put(new ClientConfig().buildMQClientId(), mQClientFactory); + + String producerGroupTemp = "FooBar_PID" + System.nanoTime(); + producer = new DefaultMQProducer(producerGroupTemp); + producer.setNamesrvAddr("127.0.0.1:9876"); + producer.setCompressMsgBodyOverHowmuch(Integer.MAX_VALUE); + message = new Message(topic, new byte[] {'a'}); + + mQClientFactory.registerProducer(producerGroupTemp, producer.getDefaultMQProducerImpl()); + + producer.start(); + + FieldUtils.writeDeclaredField(producer.getDefaultMQProducerImpl(), "mQClientFactory", mQClientFactory, true); + FieldUtils.writeField(mQClientFactory, "mQClientAPIImpl", mQClientAPIImpl, true); + + when(mQClientAPIImpl.sendMessage(anyString(), anyString(), any(Message.class), any(SendMessageRequestHeader.class), anyLong(), any(CommunicationMode.class), + nullable(SendMessageContext.class), any(DefaultMQProducerImpl.class))).thenCallRealMethod(); + when(mQClientAPIImpl.sendMessage(anyString(), anyString(), any(Message.class), any(SendMessageRequestHeader.class), anyLong(), any(CommunicationMode.class), + (SendCallback) isNull(), nullable(TopicPublishInfo.class), nullable(MQClientInstance.class), anyInt(), nullable(SendMessageContext.class), any(DefaultMQProducerImpl.class))) + .thenReturn(createSendResult(SendStatus.SEND_OK)); + when(mQClientAPIImpl.sendMessage(anyString(), anyString(), any(Message.class), any(SendMessageRequestHeader.class), anyLong(), any(CommunicationMode.class), + any(SendCallback.class), nullable(TopicPublishInfo.class), nullable(MQClientInstance.class), anyInt(), nullable(SendMessageContext.class), any(DefaultMQProducerImpl.class))) + .thenAnswer(new Answer() { + @Override public SendResult answer(InvocationOnMock invocation) throws Throwable { + SendCallback sendCallback = invocation.getArgument(6); + sendCallback.onSuccess(DefaultMQProducerLogicalQueueTest.this.createSendResult(SendStatus.SEND_OK)); + return null; + } + }); + } + + @After + public void terminate() { + producer.shutdown(); + } + + @Test + public void testSendMessageSync_Success() throws Exception { + when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong(), anyBoolean(), ArgumentMatchers.>any())).thenReturn(createTopicRoute()); + SendResult sendResult = producer.send(message, messageQueue); + + assertThat(sendResult.getSendStatus()).isEqualTo(SendStatus.SEND_OK); + assertThat(sendResult.getOffsetMsgId()).isEqualTo("123"); + assertThat(sendResult.getQueueOffset()).isEqualTo(456L); + } + + @Test + public void testSendMessageSync_Redirect() throws Exception { + when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong(), anyBoolean(), ArgumentMatchers.>any())).thenReturn(createTopicRoute()); + + when(mQClientAPIImpl.sendMessage(eq(broker1Addr), eq(broker1Name), any(Message.class), any(SendMessageRequestHeader.class), anyLong(), eq(CommunicationMode.SYNC), + (SendCallback) isNull(), nullable(TopicPublishInfo.class), nullable(MQClientInstance.class), anyInt(), nullable(SendMessageContext.class), any(DefaultMQProducerImpl.class))) + .thenThrow(new MQRedirectException(null)); + + assertThatThrownBy(new ThrowableAssert.ThrowingCallable() { + @Override public void call() throws Throwable { + producer.send(message, messageQueue); + } + }).isInstanceOf(MQBrokerException.class).hasMessageContaining("redirect"); + + when(mQClientAPIImpl.sendMessage(eq(broker1Addr), eq(broker1Name), any(Message.class), any(SendMessageRequestHeader.class), anyLong(), eq(CommunicationMode.SYNC), + (SendCallback) isNull(), nullable(TopicPublishInfo.class), nullable(MQClientInstance.class), anyInt(), nullable(SendMessageContext.class), any(DefaultMQProducerImpl.class))) + .thenThrow(new MQRedirectException(JSON.toJSONBytes(ImmutableList.of( + new LogicalQueueRouteData(0, 0, new MessageQueue(topic, broker1Name, 0), MessageQueueRouteState.Expired, 0, 0, 0, 0, broker1Addr), + new LogicalQueueRouteData(0, 10, new MessageQueue(topic, broker2Name, 0), MessageQueueRouteState.Normal, 0, -1, -1, -1, broker2Addr))))); + when(mQClientAPIImpl.sendMessage(eq(broker2Addr), eq(broker2Name), any(Message.class), any(SendMessageRequestHeader.class), anyLong(), eq(CommunicationMode.SYNC), + (SendCallback) isNull(), nullable(TopicPublishInfo.class), nullable(MQClientInstance.class), anyInt(), nullable(SendMessageContext.class), any(DefaultMQProducerImpl.class))) + .thenReturn(createSendResult(SendStatus.SEND_OK)); + + SendResult sendResult = producer.send(message, messageQueue); + assertThat(sendResult.getSendStatus()).isEqualTo(SendStatus.SEND_OK); + assertThat(sendResult.getOffsetMsgId()).isEqualTo("123"); + assertThat(sendResult.getQueueOffset()).isEqualTo(466L); + } + + @Test + public void testSendMessageSync_RemotingException() throws Exception { + TopicRouteData topicRouteData = createTopicRoute(); + when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong(), anyBoolean(), ArgumentMatchers.>any())).thenReturn(topicRouteData); + + when(mQClientAPIImpl.sendMessage(eq(broker1Addr), eq(broker1Name), any(Message.class), any(SendMessageRequestHeader.class), anyLong(), eq(CommunicationMode.SYNC), + (SendCallback) isNull(), nullable(TopicPublishInfo.class), nullable(MQClientInstance.class), anyInt(), nullable(SendMessageContext.class), any(DefaultMQProducerImpl.class))) + .thenThrow(new RemotingConnectException(broker1Addr)); + SendResult returnSendResult = createSendResult(SendStatus.SEND_OK); + when(mQClientAPIImpl.sendMessage(eq(broker2Addr), eq(broker2Name), any(Message.class), any(SendMessageRequestHeader.class), anyLong(), eq(CommunicationMode.SYNC), + (SendCallback) isNull(), nullable(TopicPublishInfo.class), nullable(MQClientInstance.class), anyInt(), nullable(SendMessageContext.class), any(DefaultMQProducerImpl.class))) + .thenReturn(returnSendResult); + + assertThatThrownBy(new ThrowableAssert.ThrowingCallable() { + @Override public void call() throws Throwable { + producer.send(message, messageQueue); + } + }).isInstanceOf(RemotingConnectException.class).hasMessageContaining(broker1Addr); + + topicRouteData.getLogicalQueuesInfo().get(0).add(new LogicalQueueRouteData(0, -1, new MessageQueue(topic, broker2Name, 1), MessageQueueRouteState.WriteOnly, 0, -1, -1, -1, broker2Addr)); + + SendResult sendResult = producer.send(message, messageQueue); + assertThat(sendResult.getSendStatus()).isEqualTo(SendStatus.SEND_OK); + assertThat(sendResult.getOffsetMsgId()).isEqualTo("123"); + assertThat(sendResult.getQueueOffset()).isEqualTo(-1L); + } + + @Test + public void testSendMessageAsync_Success() throws Exception { + when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong(), anyBoolean(), ArgumentMatchers.>any())).thenReturn(createTopicRoute()); + + final SettableFuture future = SettableFuture.create(); + producer.send(message, messageQueue, new SendCallback() { + @Override + public void onSuccess(SendResult sendResult) { + future.set(sendResult); + } + + @Override + public void onException(Throwable e) { + future.setException(e); + } + }); + + SendResult sendResult = future.get(3, TimeUnit.SECONDS); + assertThat(sendResult.getSendStatus()).isEqualTo(SendStatus.SEND_OK); + assertThat(sendResult.getOffsetMsgId()).isEqualTo("123"); + assertThat(sendResult.getQueueOffset()).isEqualTo(456L); + } + + @Test + public void testSendMessageAsync() throws Exception { + when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong(), anyBoolean(), ArgumentMatchers.>any())).thenReturn(createTopicRoute()); + + final AtomicReference> future = new AtomicReference>(); + SendCallback sendCallback = new SendCallback() { + @Override + public void onSuccess(SendResult sendResult) { + future.get().set(sendResult); + } + + @Override + public void onException(Throwable e) { + future.get().setException(e); + } + }; + + Message message = new Message(); + message.setTopic("test"); + message.setBody("hello world".getBytes()); + future.set(SettableFuture.create()); + producer.send(new Message(), messageQueue, sendCallback); + assertThatThrownBy(new ThrowableAssert.ThrowingCallable() { + @Override public void call() throws Throwable { + future.get().get(3, TimeUnit.SECONDS); + } + }).hasCauseInstanceOf(MQClientException.class).hasMessageContaining("The specified topic is blank"); + + //this message is send success + message.setTopic(topic); + future.set(SettableFuture.create()); + producer.send(message, messageQueue, sendCallback, 1000); + future.get().get(3, TimeUnit.SECONDS); + } + + public TopicRouteData createTopicRoute() { + TopicRouteData topicRouteData = new TopicRouteData(); + + topicRouteData.setFilterServerTable(new HashMap>()); + topicRouteData.setBrokerDatas(ImmutableList.of( + new BrokerData(cluster, broker1Name, new HashMap(Collections.singletonMap(MixAll.MASTER_ID, broker1Addr))), + new BrokerData(cluster, broker2Name, new HashMap(Collections.singletonMap(MixAll.MASTER_ID, broker2Addr))) + )); + + List queueDataList = new ArrayList(); + QueueData queueData; + queueData = new QueueData(); + queueData.setBrokerName(broker1Name); + queueData.setPerm(6); + queueData.setReadQueueNums(3); + queueData.setWriteQueueNums(4); + queueData.setTopicSysFlag(0); + queueDataList.add(queueData); + queueData = new QueueData(); + queueData.setBrokerName(broker2Name); + queueData.setPerm(6); + queueData.setReadQueueNums(3); + queueData.setWriteQueueNums(4); + queueData.setTopicSysFlag(0); + queueDataList.add(queueData); + topicRouteData.setQueueDatas(queueDataList); + + LogicalQueuesInfo info = new LogicalQueuesInfo(); + info.put(0, Lists.newArrayList(new LogicalQueueRouteData(0, 0, new MessageQueue(topic, broker1Name, 0), MessageQueueRouteState.Normal, 0, 0, 0, 0, broker1Addr))); + topicRouteData.setLogicalQueuesInfo(info); + return topicRouteData; + } + + private SendResult createSendResult(SendStatus sendStatus) { + SendResult sendResult = new SendResult(); + sendResult.setMsgId("123"); + sendResult.setOffsetMsgId("123"); + sendResult.setQueueOffset(456); + sendResult.setSendStatus(sendStatus); + sendResult.setRegionId("HZ"); + sendResult.setMessageQueue(new MessageQueue(topic, broker1Name, 0)); + return sendResult; + } +} diff --git a/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java b/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java index 6d507269659..a8906b33d25 100644 --- a/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java @@ -22,6 +22,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; @@ -53,6 +54,7 @@ import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.ArgumentMatchers; import org.mockito.Mock; import org.mockito.Spy; import org.mockito.junit.MockitoJUnitRunner; @@ -60,6 +62,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Fail.failBecauseExceptionWasNotThrown; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; @@ -150,7 +153,7 @@ public void testSendMessage_NoRoute() throws RemotingException, InterruptedExcep @Test public void testSendMessageSync_Success() throws RemotingException, InterruptedException, MQBrokerException, MQClientException { - when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute()); + when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong(), anyBoolean(), ArgumentMatchers.>any())).thenReturn(createTopicRoute()); SendResult sendResult = producer.send(message); assertThat(sendResult.getSendStatus()).isEqualTo(SendStatus.SEND_OK); @@ -160,7 +163,7 @@ public void testSendMessageSync_Success() throws RemotingException, InterruptedE @Test public void testSendMessageSync_WithBodyCompressed() throws RemotingException, InterruptedException, MQBrokerException, MQClientException { - when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute()); + when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong(), anyBoolean(), ArgumentMatchers.>any())).thenReturn(createTopicRoute()); SendResult sendResult = producer.send(bigMessage); assertThat(sendResult.getSendStatus()).isEqualTo(SendStatus.SEND_OK); @@ -171,7 +174,7 @@ public void testSendMessageSync_WithBodyCompressed() throws RemotingException, I @Test public void testSendMessageAsync_Success() throws RemotingException, InterruptedException, MQBrokerException, MQClientException { final CountDownLatch countDownLatch = new CountDownLatch(1); - when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute()); + when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong(), anyBoolean(), ArgumentMatchers.>any())).thenReturn(createTopicRoute()); producer.send(message, new SendCallback() { @Override public void onSuccess(SendResult sendResult) { @@ -194,7 +197,7 @@ public void testSendMessageAsync() throws RemotingException, MQClientException, final AtomicInteger cc = new AtomicInteger(0); final CountDownLatch countDownLatch = new CountDownLatch(6); - when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute()); + when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong(), anyBoolean(), ArgumentMatchers.>any())).thenReturn(createTopicRoute()); SendCallback sendCallback = new SendCallback() { @Override public void onSuccess(SendResult sendResult) { @@ -236,7 +239,7 @@ public void testBatchSendMessageAsync() final AtomicInteger cc = new AtomicInteger(0); final CountDownLatch countDownLatch = new CountDownLatch(4); - when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute()); + when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong(), anyBoolean(), ArgumentMatchers.>any())).thenReturn(createTopicRoute()); SendCallback sendCallback = new SendCallback() { @Override public void onSuccess(SendResult sendResult) { @@ -278,7 +281,7 @@ public MessageQueue select(List mqs, Message msg, Object arg) { @Test public void testSendMessageAsync_BodyCompressed() throws RemotingException, InterruptedException, MQBrokerException, MQClientException { final CountDownLatch countDownLatch = new CountDownLatch(1); - when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute()); + when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong(), anyBoolean(), ArgumentMatchers.>any())).thenReturn(createTopicRoute()); producer.send(bigMessage, new SendCallback() { @Override public void onSuccess(SendResult sendResult) { @@ -297,7 +300,7 @@ public void onException(Throwable e) { @Test public void testSendMessageSync_SuccessWithHook() throws Throwable { - when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute()); + when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong(), anyBoolean(), ArgumentMatchers.>any())).thenReturn(createTopicRoute()); final Throwable[] assertionErrors = new Throwable[1]; final CountDownLatch countDownLatch = new CountDownLatch(2); producer.getDefaultMQProducerImpl().registerSendMessageHook(new SendMessageHook() { @@ -365,7 +368,7 @@ public void testSetCallbackExecutor() throws MQClientException { @Test public void testRequestMessage() throws RemotingException, RequestTimeoutException, MQClientException, InterruptedException, MQBrokerException { - when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute()); + when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong(), anyBoolean(), ArgumentMatchers.>any())).thenReturn(createTopicRoute()); final AtomicBoolean finish = new AtomicBoolean(false); new Thread(new Runnable() { @Override public void run() { @@ -391,13 +394,13 @@ public void testRequestMessage() throws RemotingException, RequestTimeoutExcepti @Test(expected = RequestTimeoutException.class) public void testRequestMessage_RequestTimeoutException() throws RemotingException, RequestTimeoutException, MQClientException, InterruptedException, MQBrokerException { - when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute()); + when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong(), anyBoolean(), ArgumentMatchers.>any())).thenReturn(createTopicRoute()); Message result = producer.request(message, 3 * 1000L); } @Test public void testAsyncRequest_OnSuccess() throws Exception { - when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute()); + when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong(), anyBoolean(), ArgumentMatchers.>any())).thenReturn(createTopicRoute()); final CountDownLatch countDownLatch = new CountDownLatch(1); RequestCallback requestCallback = new RequestCallback() { @Override public void onSuccess(Message message) { diff --git a/client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQConsumerWithOpenTracingTest.java b/client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQConsumerWithOpenTracingTest.java index ecf72ae44cf..258b122d520 100644 --- a/client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQConsumerWithOpenTracingTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQConsumerWithOpenTracingTest.java @@ -26,6 +26,7 @@ import java.util.HashSet; import java.util.List; import java.util.Set; +import java.util.concurrent.Callable; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -102,7 +103,9 @@ public class DefaultMQConsumerWithOpenTracingTest { @Before public void init() throws Exception { ConcurrentMap factoryTable = (ConcurrentMap) FieldUtils.readDeclaredField(MQClientManager.getInstance(), "factoryTable", true); - factoryTable.forEach((s, instance) -> instance.shutdown()); + for (MQClientInstance instance : factoryTable.values()) { + instance.shutdown(); + } factoryTable.clear(); when(mQClientAPIImpl.pullMessage(anyString(), any(PullMessageRequestHeader.class), @@ -173,7 +176,7 @@ public void terminate() { @Test public void testPullMessage_WithTrace_Success() throws InterruptedException, RemotingException, MQBrokerException, MQClientException { final CountDownLatch countDownLatch = new CountDownLatch(1); - final AtomicReference messageAtomic = new AtomicReference<>(); + final AtomicReference messageAtomic = new AtomicReference(); pushConsumer.getDefaultMQPushConsumerImpl().setConsumeMessageService(new ConsumeMessageConcurrentlyService(pushConsumer.getDefaultMQPushConsumerImpl(), new MessageListenerConcurrently() { @Override public ConsumeConcurrentlyStatus consumeMessage(List msgs, @@ -193,7 +196,11 @@ public ConsumeConcurrentlyStatus consumeMessage(List msgs, assertThat(msg.getBody()).isEqualTo(new byte[]{'a'}); // wait until consumeMessageAfter hook of tracer is done surely. - waitAtMost(1, TimeUnit.SECONDS).until(() -> tracer.finishedSpans().size() == 1); + waitAtMost(1, TimeUnit.SECONDS).until(new Callable() { + @Override public Boolean call() throws Exception { + return tracer.finishedSpans().size() == 1; + } + }); MockSpan span = tracer.finishedSpans().get(0); assertThat(span.tags().get(Tags.MESSAGE_BUS_DESTINATION.getKey())).isEqualTo(topic); assertThat(span.tags().get(Tags.SPAN_KIND.getKey())).isEqualTo(Tags.SPAN_KIND_CONSUMER); diff --git a/client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQConsumerWithTraceTest.java b/client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQConsumerWithTraceTest.java index aec7d2cb0e2..1bfc28405d5 100644 --- a/client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQConsumerWithTraceTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQConsumerWithTraceTest.java @@ -115,7 +115,9 @@ public class DefaultMQConsumerWithTraceTest { @Before public void init() throws Exception { ConcurrentMap factoryTable = (ConcurrentMap) FieldUtils.readDeclaredField(MQClientManager.getInstance(), "factoryTable", true); - factoryTable.forEach((s, instance) -> instance.shutdown()); + for (MQClientInstance instance : factoryTable.values()) { + instance.shutdown(); + } factoryTable.clear(); consumerGroup = "FooBarGroup" + System.currentTimeMillis(); @@ -216,7 +218,7 @@ public void testPullMessage_WithTrace_Success() throws InterruptedException, Rem traceProducer.getDefaultMQProducerImpl().getmQClientFactory().registerProducer(producerGroupTraceTemp, traceProducer.getDefaultMQProducerImpl()); final CountDownLatch countDownLatch = new CountDownLatch(1); - final AtomicReference messageAtomic = new AtomicReference<>(); + final AtomicReference messageAtomic = new AtomicReference(); pushConsumer.getDefaultMQPushConsumerImpl().setConsumeMessageService(new ConsumeMessageConcurrentlyService(pushConsumer.getDefaultMQPushConsumerImpl(), new MessageListenerConcurrently() { @Override public ConsumeConcurrentlyStatus consumeMessage(List msgs, diff --git a/client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQProducerWithOpenTracingTest.java b/client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQProducerWithOpenTracingTest.java index 5d64a93f039..0a1f6850264 100644 --- a/client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQProducerWithOpenTracingTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQProducerWithOpenTracingTest.java @@ -20,6 +20,11 @@ import io.opentracing.mock.MockSpan; import io.opentracing.mock.MockTracer; import io.opentracing.tag.Tags; +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Set; import org.apache.rocketmq.client.ClientConfig; import org.apache.rocketmq.client.exception.MQBrokerException; import org.apache.rocketmq.client.exception.MQClientException; @@ -47,17 +52,14 @@ import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.ArgumentMatchers; import org.mockito.Mock; import org.mockito.Spy; import org.mockito.junit.MockitoJUnitRunner; -import java.lang.reflect.Field; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; - import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; @@ -113,7 +115,7 @@ public void init() throws Exception { @Test public void testSendMessageSync_WithTrace_Success() throws RemotingException, InterruptedException, MQBrokerException, MQClientException { producer.getDefaultMQProducerImpl().getmQClientFactory().registerProducer(producerGroupTraceTemp, producer.getDefaultMQProducerImpl()); - when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute()); + when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong(), anyBoolean(), ArgumentMatchers.>any())).thenReturn(createTopicRoute()); producer.send(message); assertThat(tracer.finishedSpans().size()).isEqualTo(1); MockSpan span = tracer.finishedSpans().get(0); diff --git a/client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQProducerWithTraceTest.java b/client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQProducerWithTraceTest.java index 62b34175aa2..c371694fb11 100644 --- a/client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQProducerWithTraceTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQProducerWithTraceTest.java @@ -17,6 +17,13 @@ package org.apache.rocketmq.client.trace; +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; import org.apache.rocketmq.client.ClientConfig; import org.apache.rocketmq.client.exception.MQBrokerException; import org.apache.rocketmq.client.exception.MQClientException; @@ -42,18 +49,17 @@ import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.ArgumentMatchers; import org.mockito.Mock; import org.mockito.Spy; import org.mockito.junit.MockitoJUnitRunner; -import java.lang.reflect.Field; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; - -import static org.mockito.ArgumentMatchers.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.nullable; import static org.mockito.Mockito.when; @RunWith(MockitoJUnitRunner.class) @@ -121,7 +127,7 @@ public void init() throws Exception { @Test public void testSendMessageSync_WithTrace_Success() throws RemotingException, InterruptedException, MQBrokerException, MQClientException { traceProducer.getDefaultMQProducerImpl().getmQClientFactory().registerProducer(producerGroupTraceTemp, traceProducer.getDefaultMQProducerImpl()); - when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute()); + when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong(), anyBoolean(), ArgumentMatchers.>any())).thenReturn(createTopicRoute()); final CountDownLatch countDownLatch = new CountDownLatch(1); try { producer.send(message); @@ -133,7 +139,7 @@ public void testSendMessageSync_WithTrace_Success() throws RemotingException, In @Test public void testSendMessageSync_WithTrace_NoBrokerSet_Exception() throws RemotingException, InterruptedException, MQBrokerException, MQClientException { - when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute()); + when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong(), anyBoolean(), ArgumentMatchers.>any())).thenReturn(createTopicRoute()); final CountDownLatch countDownLatch = new CountDownLatch(1); try { producer.send(message); diff --git a/client/src/test/java/org/apache/rocketmq/client/trace/TransactionMQProducerWithOpenTracingTest.java b/client/src/test/java/org/apache/rocketmq/client/trace/TransactionMQProducerWithOpenTracingTest.java index dd6d1083ce0..aca62544d8b 100644 --- a/client/src/test/java/org/apache/rocketmq/client/trace/TransactionMQProducerWithOpenTracingTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/trace/TransactionMQProducerWithOpenTracingTest.java @@ -20,6 +20,12 @@ import io.opentracing.mock.MockSpan; import io.opentracing.mock.MockTracer; import io.opentracing.tag.Tags; +import java.lang.reflect.Field; +import java.net.InetSocketAddress; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Set; import org.apache.rocketmq.client.ClientConfig; import org.apache.rocketmq.client.exception.MQBrokerException; import org.apache.rocketmq.client.exception.MQClientException; @@ -53,18 +59,14 @@ import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.ArgumentMatchers; import org.mockito.Mock; import org.mockito.Spy; import org.mockito.junit.MockitoJUnitRunner; -import java.lang.reflect.Field; -import java.net.InetSocketAddress; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; - import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; @@ -131,7 +133,7 @@ public LocalTransactionState checkLocalTransaction(MessageExt msg) { @Test public void testSendMessageSync_WithTrace_Success() throws RemotingException, InterruptedException, MQBrokerException, MQClientException { producer.getDefaultMQProducerImpl().getmQClientFactory().registerProducer(producerGroupTraceTemp, producer.getDefaultMQProducerImpl()); - when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute()); + when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong(), anyBoolean(), ArgumentMatchers.>any())).thenReturn(createTopicRoute()); producer.sendMessageInTransaction(message, null); assertThat(tracer.finishedSpans().size()).isEqualTo(2); diff --git a/client/src/test/java/org/apache/rocketmq/client/trace/TransactionMQProducerWithTraceTest.java b/client/src/test/java/org/apache/rocketmq/client/trace/TransactionMQProducerWithTraceTest.java index f838817bf9d..b3a441433cc 100644 --- a/client/src/test/java/org/apache/rocketmq/client/trace/TransactionMQProducerWithTraceTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/trace/TransactionMQProducerWithTraceTest.java @@ -17,6 +17,13 @@ package org.apache.rocketmq.client.trace; +import java.lang.reflect.Field; +import java.net.InetSocketAddress; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Set; +import java.util.concurrent.atomic.AtomicReference; import org.apache.rocketmq.client.ClientConfig; import org.apache.rocketmq.client.exception.MQBrokerException; import org.apache.rocketmq.client.exception.MQClientException; @@ -50,19 +57,20 @@ import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.ArgumentMatchers; import org.mockito.Mock; import org.mockito.Spy; +import org.mockito.invocation.InvocationOnMock; import org.mockito.junit.MockitoJUnitRunner; - -import java.lang.reflect.Field; -import java.net.InetSocketAddress; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.concurrent.atomic.AtomicReference; +import org.mockito.stubbing.Answer; import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.ArgumentMatchers.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.nullable; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.when; @@ -127,7 +135,7 @@ public LocalTransactionState checkLocalTransaction(MessageExt msg) { Field fieldHooks = DefaultMQProducerImpl.class.getDeclaredField("endTransactionHookList"); fieldHooks.setAccessible(true); - Listhooks = new ArrayList<>(); + Listhooks = new ArrayList(); hooks.add(endTransactionHook); fieldHooks.set(producer.getDefaultMQProducerImpl(), hooks); @@ -142,12 +150,14 @@ public LocalTransactionState checkLocalTransaction(MessageExt msg) { @Test public void testSendMessageSync_WithTrace_Success() throws RemotingException, InterruptedException, MQBrokerException, MQClientException { traceProducer.getDefaultMQProducerImpl().getmQClientFactory().registerProducer(producerGroupTraceTemp, traceProducer.getDefaultMQProducerImpl()); - when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute()); - AtomicReference context = new AtomicReference<>(); - doAnswer(mock -> { - context.set(mock.getArgument(0)); - return null; - }).when(endTransactionHook).endTransaction(any()); + when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong(), anyBoolean(), ArgumentMatchers.>any())).thenReturn(createTopicRoute()); + final AtomicReference context = new AtomicReference(); + doAnswer(new Answer() { + @Override public Object answer(InvocationOnMock mock) throws Throwable { + context.set(mock.getArgument(0)); + return null; + } + }).when(endTransactionHook).endTransaction(ArgumentMatchers.any()); producer.sendMessageInTransaction(message, null); EndTransactionContext ctx = context.get(); diff --git a/common/pom.xml b/common/pom.xml index ac1d086b4cc..ff32ddfc1d2 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -27,6 +27,19 @@ rocketmq-common rocketmq-common ${project.version} + + + + org.apache.maven.plugins + maven-compiler-plugin + + 6 + 6 + + + + + ${project.groupId} @@ -40,5 +53,9 @@ commons-validator commons-validator + + com.google.guava + guava + diff --git a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java index 488f2132daf..17cc2a1bebc 100644 --- a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java +++ b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java @@ -222,6 +222,8 @@ public class BrokerConfig { private boolean autoDeleteUnusedStats = false; + private long forwardTimeout = 3 * 1000; + public static String localHostName() { try { return InetAddress.getLocalHost().getHostName(); @@ -908,7 +910,6 @@ public void setAutoDeleteUnusedStats(boolean autoDeleteUnusedStats) { this.autoDeleteUnusedStats = autoDeleteUnusedStats; } - public long getLoadBalancePollNameServerInterval() { return loadBalancePollNameServerInterval; } @@ -958,4 +959,12 @@ public int getDefaultPopShareQueueNum() { public void setDefaultPopShareQueueNum(int defaultPopShareQueueNum) { this.defaultPopShareQueueNum = defaultPopShareQueueNum; } + + public long getForwardTimeout() { + return forwardTimeout; + } + + public void setForwardTimeout(long timeout) { + this.forwardTimeout = timeout; + } } diff --git a/common/src/main/java/org/apache/rocketmq/common/ConfigManager.java b/common/src/main/java/org/apache/rocketmq/common/ConfigManager.java index 99b5f0c667b..13d5b6be441 100644 --- a/common/src/main/java/org/apache/rocketmq/common/ConfigManager.java +++ b/common/src/main/java/org/apache/rocketmq/common/ConfigManager.java @@ -17,6 +17,7 @@ package org.apache.rocketmq.common; import java.io.IOException; +import java.util.Map; import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.logging.InternalLoggerFactory; @@ -67,6 +68,16 @@ private boolean loadBak() { public abstract void decode(final String jsonString); + public synchronized void persist(String topicName, T t) { + // stub for future + this.persist(); + } + + public synchronized void persist(Map m) { + // stub for future + this.persist(); + } + public synchronized void persist() { String jsonString = this.encode(true); if (jsonString != null) { diff --git a/common/src/main/java/org/apache/rocketmq/common/MixAll.java b/common/src/main/java/org/apache/rocketmq/common/MixAll.java index 9d95ecb5ea4..8aae04b2674 100644 --- a/common/src/main/java/org/apache/rocketmq/common/MixAll.java +++ b/common/src/main/java/org/apache/rocketmq/common/MixAll.java @@ -16,6 +16,7 @@ */ package org.apache.rocketmq.common; +import com.alibaba.fastjson.TypeReference; import java.io.ByteArrayInputStream; import java.io.File; import java.io.FileInputStream; @@ -26,6 +27,7 @@ import java.lang.reflect.Field; import java.lang.reflect.Method; import java.lang.reflect.Modifier; +import java.lang.reflect.Type; import java.net.Inet6Address; import java.net.InetAddress; import java.net.NetworkInterface; @@ -41,6 +43,7 @@ import org.apache.rocketmq.common.annotation.ImportantField; import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.help.FAQUrl; +import org.apache.rocketmq.common.protocol.route.LogicalQueueRouteData; import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.logging.InternalLoggerFactory; @@ -83,6 +86,9 @@ public class MixAll { public static final String ACL_CONF_TOOLS_FILE = "/conf/tools.yml"; public static final String REPLY_MESSAGE_FLAG = "reply"; private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.COMMON_LOGGER_NAME); + public static final String LOGICAL_QUEUE_MOCK_BROKER_NAME = "__logical_queue_broker__"; + public static final Type TYPE_LIST_LOGICAL_QUEUE_ROUTE_DATA = new TypeReference>() { + }.getType(); public static String getWSAddr() { String wsDomainName = System.getProperty("rocketmq.namesrv.domain", DEFAULT_NAMESRV_ADDR_LOOKUP); @@ -443,4 +449,11 @@ public static String humanReadableByteCount(long bytes, boolean si) { return String.format("%.1f %sB", bytes / Math.pow(unit, exp), pre); } + public static int compareInteger(int x, int y) { + return (x < y) ? -1 : ((x == y) ? 0 : 1); + } + + public static int compareLong(long x, long y) { + return (x < y) ? -1 : ((x == y) ? 0 : 1); + } } diff --git a/common/src/main/java/org/apache/rocketmq/common/TopicConfig.java b/common/src/main/java/org/apache/rocketmq/common/TopicConfig.java index 4795cced62d..c082ba66165 100644 --- a/common/src/main/java/org/apache/rocketmq/common/TopicConfig.java +++ b/common/src/main/java/org/apache/rocketmq/common/TopicConfig.java @@ -37,6 +37,16 @@ public TopicConfig(String topicName) { this.topicName = topicName; } + public TopicConfig(TopicConfig other) { + this.topicName = other.topicName; + this.readQueueNums = other.readQueueNums; + this.writeQueueNums = other.writeQueueNums; + this.perm = other.perm; + this.topicFilterType = other.topicFilterType; + this.topicSysFlag = other.topicSysFlag; + this.order = other.order; + } + public TopicConfig(String topicName, int readQueueNums, int writeQueueNums, int perm) { this.topicName = topicName; this.readQueueNums = readQueueNums; diff --git a/common/src/main/java/org/apache/rocketmq/common/TopicQueueId.java b/common/src/main/java/org/apache/rocketmq/common/TopicQueueId.java new file mode 100644 index 00000000000..ee215af0f14 --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/TopicQueueId.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.common; + +import com.google.common.base.Objects; + +public class TopicQueueId { + private final String topic; + private final int queueId; + + private final int hash; + + public TopicQueueId(String topic, int queueId) { + this.topic = topic; + this.queueId = queueId; + + this.hash = Objects.hashCode(topic, queueId); + } + + @Override public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + TopicQueueId broker = (TopicQueueId) o; + return queueId == broker.queueId && Objects.equal(topic, broker.topic); + } + + @Override public int hashCode() { + return hash; + } + + @Override public String toString() { + final StringBuilder sb = new StringBuilder("MessageQueueInBroker{"); + sb.append("topic='").append(topic).append('\''); + sb.append(", queueId=").append(queueId); + sb.append('}'); + return sb.toString(); + } +} diff --git a/common/src/main/java/org/apache/rocketmq/common/constant/LoggerName.java b/common/src/main/java/org/apache/rocketmq/common/constant/LoggerName.java index 589200b2094..c29eccd4cbe 100644 --- a/common/src/main/java/org/apache/rocketmq/common/constant/LoggerName.java +++ b/common/src/main/java/org/apache/rocketmq/common/constant/LoggerName.java @@ -38,4 +38,5 @@ public class LoggerName { public static final String WATER_MARK_LOGGER_NAME = "RocketmqWaterMark"; public static final String FILTER_LOGGER_NAME = "RocketmqFilter"; public static final String ROCKETMQ_POP_LOGGER_NAME = "RocketmqPop"; + public static final String STDOUT_LOGGER_NAME = "STDOUT"; } diff --git a/common/src/main/java/org/apache/rocketmq/common/fastjson/GenericMapSuperclassDeserializer.java b/common/src/main/java/org/apache/rocketmq/common/fastjson/GenericMapSuperclassDeserializer.java new file mode 100644 index 00000000000..e4388eb25a0 --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/fastjson/GenericMapSuperclassDeserializer.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.common.fastjson; + +import com.alibaba.fastjson.JSONException; +import com.alibaba.fastjson.parser.DefaultJSONParser; +import com.alibaba.fastjson.parser.JSONToken; +import com.alibaba.fastjson.parser.deserializer.MapDeserializer; +import com.alibaba.fastjson.parser.deserializer.ObjectDeserializer; +import java.lang.reflect.ParameterizedType; +import java.lang.reflect.Type; +import java.util.Map; + +/** + * workaround https://github.com/alibaba/fastjson/issues/3730 + */ +public class GenericMapSuperclassDeserializer implements ObjectDeserializer { + public static final GenericMapSuperclassDeserializer INSTANCE = new GenericMapSuperclassDeserializer(); + + @SuppressWarnings({"unchecked", "rawtypes"}) + @Override public T deserialze(DefaultJSONParser parser, Type type, Object fieldName) { + Class clz = (Class) type; + Type genericSuperclass = clz.getGenericSuperclass(); + Map map; + try { + map = (Map) clz.newInstance(); + } catch (Exception e) { + throw new JSONException("unsupport type " + type, e); + } + ParameterizedType parameterizedType = (ParameterizedType) genericSuperclass; + Type keyType = parameterizedType.getActualTypeArguments()[0]; + Type valueType = parameterizedType.getActualTypeArguments()[1]; + if (String.class == keyType) { + return (T) MapDeserializer.parseMap(parser, (Map) map, valueType, fieldName); + } else { + return (T) MapDeserializer.parseMap(parser, map, keyType, valueType, fieldName); + } + } + + @Override public int getFastMatchToken() { + return JSONToken.LBRACE; + } +} diff --git a/common/src/main/java/org/apache/rocketmq/common/message/MessageConst.java b/common/src/main/java/org/apache/rocketmq/common/message/MessageConst.java index 0922c5f6759..628bf4e613d 100644 --- a/common/src/main/java/org/apache/rocketmq/common/message/MessageConst.java +++ b/common/src/main/java/org/apache/rocketmq/common/message/MessageConst.java @@ -54,6 +54,8 @@ public class MessageConst { public static final String PROPERTY_MESSAGE_TYPE = "MSG_TYPE"; public static final String PROPERTY_POP_CK = "POP_CK"; public static final String PROPERTY_FIRST_POP_TIME = "1ST_POP_TIME"; + public static final String PROPERTY_FORWARD_QUEUE_ID = "PROPERTY_FORWARD_QUEUE_ID"; + public static final String PROPERTY_REDIRECT = "REDIRECT"; public static final String KEY_SEPARATOR = " "; diff --git a/common/src/main/java/org/apache/rocketmq/common/message/MessageQueue.java b/common/src/main/java/org/apache/rocketmq/common/message/MessageQueue.java index 03ba2027e03..7926b7327d6 100644 --- a/common/src/main/java/org/apache/rocketmq/common/message/MessageQueue.java +++ b/common/src/main/java/org/apache/rocketmq/common/message/MessageQueue.java @@ -28,6 +28,12 @@ public MessageQueue() { } + public MessageQueue(MessageQueue other) { + this.topic = other.topic; + this.brokerName = other.brokerName; + this.queueId = other.queueId; + } + public MessageQueue(String topic, String brokerName, int queueId) { this.topic = topic; this.brokerName = brokerName; diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/RequestCode.java b/common/src/main/java/org/apache/rocketmq/common/protocol/RequestCode.java index 9446caa0735..3613049c0f9 100644 --- a/common/src/main/java/org/apache/rocketmq/common/protocol/RequestCode.java +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/RequestCode.java @@ -193,6 +193,18 @@ public class RequestCode { public static final int PUSH_REPLY_MESSAGE_TO_CLIENT = 326; + public static final int GET_TOPIC_CONFIG = 351; + public static final int QUERY_ASSIGNMENT = 400; public static final int SET_MESSAGE_REQUEST_MODE = 401; + + public static final int UPDATE_TOPIC_LOGICAL_QUEUE_MAPPING = 411; + public static final int DELETE_TOPIC_LOGICAL_QUEUE_MAPPING = 422; + public static final int QUERY_TOPIC_LOGICAL_QUEUE_MAPPING = 413; + public static final int SEAL_TOPIC_LOGICAL_QUEUE = 414; + public static final int REUSE_TOPIC_LOGICAL_QUEUE = 415; + public static final int CREATE_MESSAGE_QUEUE_FOR_LOGICAL_QUEUE = 416; + public static final int MIGRATE_TOPIC_LOGICAL_QUEUE_PREPARE = 417; + public static final int MIGRATE_TOPIC_LOGICAL_QUEUE_COMMIT = 418; + public static final int MIGRATE_TOPIC_LOGICAL_QUEUE_NOTIFY = 419; } diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/ResponseCode.java b/common/src/main/java/org/apache/rocketmq/common/protocol/ResponseCode.java index df0ccbe95f9..42b9c4fa0fc 100644 --- a/common/src/main/java/org/apache/rocketmq/common/protocol/ResponseCode.java +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/ResponseCode.java @@ -20,6 +20,7 @@ import org.apache.rocketmq.remoting.protocol.RemotingSysResponseCode; public class ResponseCode extends RemotingSysResponseCode { + public static final int ASYNC_AND_RETURN_NULL = -2; public static final int FLUSH_DISK_TIMEOUT = 10; diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/body/ClusterInfo.java b/common/src/main/java/org/apache/rocketmq/common/protocol/body/ClusterInfo.java index 76c64a8508c..222e51a3a64 100644 --- a/common/src/main/java/org/apache/rocketmq/common/protocol/body/ClusterInfo.java +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/body/ClusterInfo.java @@ -17,6 +17,7 @@ package org.apache.rocketmq.common.protocol.body; +import com.google.common.base.Objects; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -62,4 +63,17 @@ public String[] retrieveAllAddrByCluster(String cluster) { public String[] retrieveAllClusterNames() { return clusterAddrTable.keySet().toArray(new String[] {}); } + + @Override public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + ClusterInfo info = (ClusterInfo) o; + return Objects.equal(brokerAddrTable, info.brokerAddrTable) && Objects.equal(clusterAddrTable, info.clusterAddrTable); + } + + @Override public int hashCode() { + return Objects.hashCode(brokerAddrTable, clusterAddrTable); + } } diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/body/CreateMessageQueueForLogicalQueueRequestBody.java b/common/src/main/java/org/apache/rocketmq/common/protocol/body/CreateMessageQueueForLogicalQueueRequestBody.java new file mode 100644 index 00000000000..e446d9ba4c6 --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/body/CreateMessageQueueForLogicalQueueRequestBody.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.common.protocol.body; + +import org.apache.rocketmq.common.protocol.route.MessageQueueRouteState; +import org.apache.rocketmq.remoting.protocol.RemotingSerializable; + +public class CreateMessageQueueForLogicalQueueRequestBody extends RemotingSerializable { + private String topic; + private int logicalQueueIndex; + private MessageQueueRouteState messageQueueStatus; + + public String getTopic() { + return topic; + } + + public void setTopic(String topic) { + this.topic = topic; + } + + public int getLogicalQueueIndex() { + return logicalQueueIndex; + } + + public void setLogicalQueueIndex(int logicalQueueIndex) { + this.logicalQueueIndex = logicalQueueIndex; + } + + public MessageQueueRouteState getMessageQueueStatus() { + return messageQueueStatus; + } + + public void setMessageQueueStatus(MessageQueueRouteState messageQueueStatuses) { + this.messageQueueStatus = messageQueueStatuses; + } +} diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/body/MigrateLogicalQueueBody.java b/common/src/main/java/org/apache/rocketmq/common/protocol/body/MigrateLogicalQueueBody.java new file mode 100644 index 00000000000..6eb06a5d973 --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/body/MigrateLogicalQueueBody.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.common.protocol.body; + +import org.apache.rocketmq.common.protocol.route.LogicalQueueRouteData; +import org.apache.rocketmq.remoting.protocol.RemotingSerializable; + +public class MigrateLogicalQueueBody extends RemotingSerializable { + private LogicalQueueRouteData fromQueueRouteData; + private LogicalQueueRouteData toQueueRouteData; + + public LogicalQueueRouteData getFromQueueRouteData() { + return fromQueueRouteData; + } + + public void setFromQueueRouteData( + LogicalQueueRouteData fromQueueRouteData) { + this.fromQueueRouteData = fromQueueRouteData; + } + + public LogicalQueueRouteData getToQueueRouteData() { + return toQueueRouteData; + } + + public void setToQueueRouteData(LogicalQueueRouteData toQueueRouteData) { + this.toQueueRouteData = toQueueRouteData; + } +} diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/body/ReuseTopicLogicalQueueRequestBody.java b/common/src/main/java/org/apache/rocketmq/common/protocol/body/ReuseTopicLogicalQueueRequestBody.java new file mode 100644 index 00000000000..22ab4522927 --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/body/ReuseTopicLogicalQueueRequestBody.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.common.protocol.body; + +import org.apache.rocketmq.common.protocol.route.MessageQueueRouteState; +import org.apache.rocketmq.remoting.protocol.RemotingSerializable; + +public class ReuseTopicLogicalQueueRequestBody extends RemotingSerializable { + private String topic; + private int queueId; + private int logicalQueueIndex; + private MessageQueueRouteState messageQueueRouteState; + + public String getTopic() { + return topic; + } + + public void setTopic(String topic) { + this.topic = topic; + } + + public int getQueueId() { + return queueId; + } + + public void setQueueId(int queueId) { + this.queueId = queueId; + } + + public int getLogicalQueueIndex() { + return logicalQueueIndex; + } + + public void setLogicalQueueIndex(int logicalQueueIndex) { + this.logicalQueueIndex = logicalQueueIndex; + } + + public void setMessageQueueRouteState(MessageQueueRouteState messageQueueRouteState) { + this.messageQueueRouteState = messageQueueRouteState; + } + + public MessageQueueRouteState getMessageQueueRouteState() { + return messageQueueRouteState; + } +} diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/body/SealTopicLogicalQueueRequestBody.java b/common/src/main/java/org/apache/rocketmq/common/protocol/body/SealTopicLogicalQueueRequestBody.java new file mode 100644 index 00000000000..edb521f3d01 --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/body/SealTopicLogicalQueueRequestBody.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.common.protocol.body; + +import org.apache.rocketmq.remoting.protocol.RemotingSerializable; + +public class SealTopicLogicalQueueRequestBody extends RemotingSerializable { + private String topic; + private int queueId; + private int logicalQueueIndex; + + public String getTopic() { + return topic; + } + + public void setTopic(String topic) { + this.topic = topic; + } + + public int getQueueId() { + return queueId; + } + + public void setQueueId(int queueId) { + this.queueId = queueId; + } + + public int getLogicalQueueIndex() { + return logicalQueueIndex; + } + + public void setLogicalQueueIndex(int logicalQueueIndex) { + this.logicalQueueIndex = logicalQueueIndex; + } +} diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/body/TopicConfigSerializeWrapper.java b/common/src/main/java/org/apache/rocketmq/common/protocol/body/TopicConfigSerializeWrapper.java index ce123021d45..13896636fc4 100644 --- a/common/src/main/java/org/apache/rocketmq/common/protocol/body/TopicConfigSerializeWrapper.java +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/body/TopicConfigSerializeWrapper.java @@ -17,15 +17,18 @@ package org.apache.rocketmq.common.protocol.body; +import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import org.apache.rocketmq.common.DataVersion; import org.apache.rocketmq.common.TopicConfig; +import org.apache.rocketmq.common.protocol.route.LogicalQueuesInfo; import org.apache.rocketmq.remoting.protocol.RemotingSerializable; public class TopicConfigSerializeWrapper extends RemotingSerializable { private ConcurrentMap topicConfigTable = new ConcurrentHashMap(); + private Map logicalQueuesInfoMap; private DataVersion dataVersion = new DataVersion(); public ConcurrentMap getTopicConfigTable() { @@ -43,4 +46,12 @@ public DataVersion getDataVersion() { public void setDataVersion(DataVersion dataVersion) { this.dataVersion = dataVersion; } + + public Map getLogicalQueuesInfoMap() { + return logicalQueuesInfoMap; + } + + public void setLogicalQueuesInfoMap(Map logicalQueuesInfoMap) { + this.logicalQueuesInfoMap = logicalQueuesInfoMap; + } } diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/body/UpdateTopicLogicalQueueMappingRequestBody.java b/common/src/main/java/org/apache/rocketmq/common/protocol/body/UpdateTopicLogicalQueueMappingRequestBody.java new file mode 100644 index 00000000000..67c6fd2a1ae --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/body/UpdateTopicLogicalQueueMappingRequestBody.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.common.protocol.body; + +import org.apache.rocketmq.remoting.protocol.RemotingSerializable; + +public class UpdateTopicLogicalQueueMappingRequestBody extends RemotingSerializable { + private String topic; + private int queueId; + private int logicalQueueIdx; + + public int getLogicalQueueIdx() { + return logicalQueueIdx; + } + + public void setLogicalQueueIdx(int logicalQueueIdx) { + this.logicalQueueIdx = logicalQueueIdx; + } + + public String getTopic() { + return topic; + } + + public void setTopic(String topic) { + this.topic = topic; + } + + public int getQueueId() { + return queueId; + } + + public void setQueueId(int queueId) { + this.queueId = queueId; + } +} diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/header/DeleteTopicLogicalQueueRequestHeader.java b/common/src/main/java/org/apache/rocketmq/common/protocol/header/DeleteTopicLogicalQueueRequestHeader.java new file mode 100644 index 00000000000..fa8d50d4192 --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/header/DeleteTopicLogicalQueueRequestHeader.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.common.protocol.header; + +import org.apache.rocketmq.remoting.CommandCustomHeader; +import org.apache.rocketmq.remoting.annotation.CFNotNull; +import org.apache.rocketmq.remoting.exception.RemotingCommandException; + +public class DeleteTopicLogicalQueueRequestHeader implements CommandCustomHeader { + @CFNotNull + private String topic; + + @Override public void checkFields() throws RemotingCommandException { + } + + public String getTopic() { + return topic; + } + + public void setTopic(String topic) { + this.topic = topic; + } +} diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/header/GetMaxOffsetRequestHeader.java b/common/src/main/java/org/apache/rocketmq/common/protocol/header/GetMaxOffsetRequestHeader.java index 871309de6ce..6963195f912 100644 --- a/common/src/main/java/org/apache/rocketmq/common/protocol/header/GetMaxOffsetRequestHeader.java +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/header/GetMaxOffsetRequestHeader.java @@ -29,6 +29,8 @@ public class GetMaxOffsetRequestHeader implements CommandCustomHeader { private String topic; @CFNotNull private Integer queueId; + private boolean committed; + private boolean logicalQueue; @Override public void checkFields() throws RemotingCommandException { @@ -49,4 +51,20 @@ public Integer getQueueId() { public void setQueueId(Integer queueId) { this.queueId = queueId; } + + public void setCommitted(boolean committed) { + this.committed = committed; + } + + public boolean isCommitted() { + return committed; + } + + public void setLogicalQueue(boolean logicalQueue) { + this.logicalQueue = logicalQueue; + } + + public boolean getLogicalQueue() { + return logicalQueue; + } } diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/header/GetTopicConfigRequestHeader.java b/common/src/main/java/org/apache/rocketmq/common/protocol/header/GetTopicConfigRequestHeader.java new file mode 100644 index 00000000000..ea9d17c3c70 --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/header/GetTopicConfigRequestHeader.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.common.protocol.header; + +import org.apache.rocketmq.remoting.CommandCustomHeader; +import org.apache.rocketmq.remoting.annotation.CFNotNull; +import org.apache.rocketmq.remoting.exception.RemotingCommandException; + +public class GetTopicConfigRequestHeader implements CommandCustomHeader { + @Override + public void checkFields() throws RemotingCommandException { + } + + @CFNotNull + private String topic; + + /** + * @return the topic + */ + public String getTopic() { + return topic; + } + + /** + * @param topic the topic to set + */ + public void setTopic(String topic) { + this.topic = topic; + } +} \ No newline at end of file diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/header/QueryTopicLogicalQueueMappingRequestHeader.java b/common/src/main/java/org/apache/rocketmq/common/protocol/header/QueryTopicLogicalQueueMappingRequestHeader.java new file mode 100644 index 00000000000..b7e0c46d055 --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/header/QueryTopicLogicalQueueMappingRequestHeader.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.common.protocol.header; + +import org.apache.rocketmq.remoting.CommandCustomHeader; +import org.apache.rocketmq.remoting.annotation.CFNotNull; +import org.apache.rocketmq.remoting.exception.RemotingCommandException; + +public class QueryTopicLogicalQueueMappingRequestHeader implements CommandCustomHeader { + @CFNotNull + private String topic; + + @Override public void checkFields() throws RemotingCommandException { + } + + public String getTopic() { + return topic; + } + + public void setTopic(String topic) { + this.topic = topic; + } +} diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/header/namesrv/GetRouteInfoRequestHeader.java b/common/src/main/java/org/apache/rocketmq/common/protocol/header/namesrv/GetRouteInfoRequestHeader.java index a2806e62897..ad776c855bb 100644 --- a/common/src/main/java/org/apache/rocketmq/common/protocol/header/namesrv/GetRouteInfoRequestHeader.java +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/header/namesrv/GetRouteInfoRequestHeader.java @@ -20,6 +20,7 @@ */ package org.apache.rocketmq.common.protocol.header.namesrv; +import java.util.Set; import org.apache.rocketmq.remoting.CommandCustomHeader; import org.apache.rocketmq.remoting.annotation.CFNotNull; import org.apache.rocketmq.remoting.exception.RemotingCommandException; @@ -28,6 +29,9 @@ public class GetRouteInfoRequestHeader implements CommandCustomHeader { @CFNotNull private String topic; + private int sysFlag; + private Set logicalQueueIdsFilter; + @Override public void checkFields() throws RemotingCommandException { } @@ -39,4 +43,20 @@ public String getTopic() { public void setTopic(String topic) { this.topic = topic; } + + public int getSysFlag() { + return sysFlag; + } + + public void setSysFlag(int sysFlag) { + this.sysFlag = sysFlag; + } + + public void setLogicalQueueIdsFilter(Set filter) { + this.logicalQueueIdsFilter = filter; + } + + public Set getLogicalQueueIdsFilter() { + return logicalQueueIdsFilter; + } } diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/route/LogicalQueueRouteData.java b/common/src/main/java/org/apache/rocketmq/common/protocol/route/LogicalQueueRouteData.java new file mode 100644 index 00000000000..aed4d5d669e --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/route/LogicalQueueRouteData.java @@ -0,0 +1,309 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.common.protocol.route; + +import com.alibaba.fastjson.annotation.JSONField; +import com.google.common.base.Objects; +import com.google.common.base.Predicate; +import com.google.common.collect.Lists; +import java.util.List; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.message.MessageQueue; + +/** + * logical queue offset -> message queue offset mapping + */ +public class LogicalQueueRouteData implements Comparable { + private volatile int logicalQueueIndex = -1; /* -1 means not set */ + private volatile long logicalQueueDelta = -1; /* inclusive, -1 means not set, occurred in writeOnly state */ + + private MessageQueue messageQueue; + + private volatile MessageQueueRouteState state = MessageQueueRouteState.Normal; + + private volatile long offsetDelta = 0; // valid when Normal/WriteOnly/ReadOnly + private volatile long offsetMax = -1; // exclusive, valid when ReadOnly + + private volatile long firstMsgTimeMillis = -1; // valid when ReadOnly + private volatile long lastMsgTimeMillis = -1; // valid when ReadOnly + + private String brokerAddr; /* not always set, only used by high availability forward */ + + public LogicalQueueRouteData() { + } + + public LogicalQueueRouteData(int logicalQueueIndex, long logicalQueueDelta, + MessageQueue messageQueue, MessageQueueRouteState state, long offsetDelta, long offsetMax, + long firstMsgTimeMillis, + long lastMsgTimeMillis, String brokerAddr) { + this.logicalQueueIndex = logicalQueueIndex; + this.logicalQueueDelta = logicalQueueDelta; + this.messageQueue = messageQueue; + this.state = state; + this.offsetDelta = offsetDelta; + this.offsetMax = offsetMax; + this.firstMsgTimeMillis = firstMsgTimeMillis; + this.lastMsgTimeMillis = lastMsgTimeMillis; + this.brokerAddr = brokerAddr; + } + + public LogicalQueueRouteData(LogicalQueueRouteData queueRouteData) { + copyFrom(queueRouteData); + } + + public int getLogicalQueueIndex() { + return logicalQueueIndex; + } + + public void setLogicalQueueIndex(int logicalQueueIndex) { + this.logicalQueueIndex = logicalQueueIndex; + } + + public long getLogicalQueueDelta() { + return logicalQueueDelta; + } + + public void setLogicalQueueDelta(long logicalQueueDelta) { + this.logicalQueueDelta = logicalQueueDelta; + } + + public MessageQueue getMessageQueue() { + return messageQueue; + } + + public void setMessageQueue(MessageQueue messageQueue) { + this.messageQueue = messageQueue; + } + + public MessageQueueRouteState getState() { + return state; + } + + @JSONField(serialize = false) + public int getStateOrdinal() { + return state.ordinal(); + } + + public void setState(MessageQueueRouteState state) { + this.state = state; + } + + public String getBrokerAddr() { + return brokerAddr; + } + + public void setBrokerAddr(String brokerAddr) { + this.brokerAddr = brokerAddr; + } + + public long getOffsetDelta() { + return offsetDelta; + } + + public void setOffsetDelta(long offsetDelta) { + this.offsetDelta = offsetDelta; + } + + public long getOffsetMax() { + return offsetMax; + } + + public void setOffsetMax(long offsetMax) { + this.offsetMax = offsetMax; + } + + public long getFirstMsgTimeMillis() { + return firstMsgTimeMillis; + } + + public void setFirstMsgTimeMillis(long firstMsgTimeMillis) { + this.firstMsgTimeMillis = firstMsgTimeMillis; + } + + public long getLastMsgTimeMillis() { + return lastMsgTimeMillis; + } + + public void setLastMsgTimeMillis(long lastMsgTimeMillis) { + this.lastMsgTimeMillis = lastMsgTimeMillis; + } + + @Override public String toString() { + return "LogicalQueueRouteData{" + + "logicalQueueIndex=" + logicalQueueIndex + + ", logicalQueueDelta=" + logicalQueueDelta + + ", messageQueue=" + messageQueue + + ", state=" + state + + ", offsetDelta=" + offsetDelta + + ", offsetMax=" + offsetMax + + ", firstMsgTimeMillis=" + firstMsgTimeMillis + + ", lastMsgTimeMillis=" + lastMsgTimeMillis + + ", brokerAddr='" + brokerAddr + '\'' + + '}'; + } + + public void copyFrom(LogicalQueueRouteData queueRouteData) { + this.logicalQueueIndex = queueRouteData.logicalQueueIndex; + this.logicalQueueDelta = queueRouteData.logicalQueueDelta; + this.messageQueue = new MessageQueue(queueRouteData.getMessageQueue()); + this.state = queueRouteData.state; + this.offsetDelta = queueRouteData.offsetDelta; + this.offsetMax = queueRouteData.offsetMax; + this.firstMsgTimeMillis = queueRouteData.firstMsgTimeMillis; + this.lastMsgTimeMillis = queueRouteData.lastMsgTimeMillis; + this.brokerAddr = queueRouteData.brokerAddr; + } + + public long toLogicalQueueOffset(long messageQueueOffset) { + return this.logicalQueueDelta < 0 ? -1 : messageQueueOffset - this.offsetDelta + this.logicalQueueDelta; + } + + public long toMessageQueueOffset(long logicalQueueOffset) { + return logicalQueueOffset - this.logicalQueueDelta + this.offsetDelta; + } + + @Override public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + LogicalQueueRouteData that = (LogicalQueueRouteData) o; + return logicalQueueIndex == that.logicalQueueIndex && logicalQueueDelta == that.logicalQueueDelta && offsetDelta == that.offsetDelta && offsetMax == that.offsetMax && firstMsgTimeMillis == that.firstMsgTimeMillis && lastMsgTimeMillis == that.lastMsgTimeMillis && Objects.equal(messageQueue, that.messageQueue) && state == that.state && Objects.equal(brokerAddr, that.brokerAddr); + } + + @Override public int hashCode() { + return Objects.hashCode(logicalQueueIndex, logicalQueueDelta, messageQueue, state, offsetDelta, offsetMax, firstMsgTimeMillis, lastMsgTimeMillis, brokerAddr); + } + + @JSONField(serialize = false) + public long getMessagesCount() { + return this.offsetDelta >= 0 && this.offsetMax >= 0 ? this.offsetMax - this.offsetDelta : 0L; + } + + @JSONField(serialize = false) + public boolean isWritable() { + return MessageQueueRouteState.Normal.equals(state) || MessageQueueRouteState.WriteOnly.equals(state); + } + + @JSONField(serialize = false) + public boolean isReadable() { + return MessageQueueRouteState.Normal.equals(state) || MessageQueueRouteState.ReadOnly.equals(state); + } + + @JSONField(serialize = false) + public boolean isExpired() { + return MessageQueueRouteState.Expired.equals(state); + } + + @JSONField(serialize = false) + public boolean isWriteOnly() { + return MessageQueueRouteState.WriteOnly.equals(state); + } + + @JSONField(serialize = false) + public int getQueueId() { + return messageQueue.getQueueId(); + } + + @JSONField(serialize = false) + public String getBrokerName() { + return messageQueue.getBrokerName(); + } + + @JSONField(serialize = false) + public String getTopic() { + return messageQueue.getTopic(); + } + + public boolean isSameTo(LogicalQueueRouteData o) { + if (o == null) { + return false; + } + return isSameTo(o.getMessageQueue(), o.offsetDelta); + } + + public boolean isSameTo(MessageQueue mq, long offsetDelta) { + return Objects.equal(this.messageQueue, mq) && this.offsetDelta == offsetDelta; + } + + /** + * First compare logicalQueueDelta, negative delta must be ordered in the last; then compare state's ordinal; then + * compare messageQueue, nulls first; then compare offsetDelta. + */ + @Override + public int compareTo(LogicalQueueRouteData o) { + long x = this.getLogicalQueueDelta(); + long y = o.getLogicalQueueDelta(); + int result; + { + if (x >= 0 && y >= 0) { + result = MixAll.compareLong(x, y); + } else if (x < 0 && y < 0) { + result = MixAll.compareLong(-x, -y); + } else if (x < 0) { + // o1 < 0 && o2 >= 0 + result = 1; + } else { + // o1 >= 0 && o2 < 0 + result = -1; + } + } + if (result == 0) { + result = MixAll.compareInteger(this.state.ordinal(), o.state.ordinal()); + } + if (result == 0) { + if (this.messageQueue == null) { + if (o.messageQueue != null) { + result = -1; + } + } else { + if (o.messageQueue != null) { + result = this.messageQueue.compareTo(o.messageQueue); + } else { + result = 1; + } + } + } + if (result == 0) { + result = MixAll.compareLong(this.offsetDelta, o.offsetDelta); + } + return result; + } + + public static final Predicate READABLE_PREDICT = new Predicate() { + @Override + public boolean apply(LogicalQueueRouteData input) { + return input != null && input.isReadable(); + } + }; + + public List filterMessages(List list) { + if (this.offsetMax < 0 || list == null || list.isEmpty()) { + return list; + } + List result = Lists.newArrayListWithExpectedSize(list.size()); + for (MessageExt m : list) { + if (m.getQueueOffset() >= this.offsetMax) { + break; + } else { + result.add(m); + } + } + return result; + } +} diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/route/LogicalQueuesInfo.java b/common/src/main/java/org/apache/rocketmq/common/protocol/route/LogicalQueuesInfo.java new file mode 100644 index 00000000000..de62b435d6a --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/route/LogicalQueuesInfo.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.common.protocol.route; + +import com.alibaba.fastjson.parser.ParserConfig; +import com.google.common.base.Objects; +import com.google.common.collect.Lists; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; +import org.apache.rocketmq.common.fastjson.GenericMapSuperclassDeserializer; + +public class LogicalQueuesInfo extends HashMap> { + // TODO whether here needs more fine-grained locks like per logical queue lock? + private final ReadWriteLock lock = new ReentrantReadWriteLock(); + + public LogicalQueuesInfo() { + super(); + } + + public LogicalQueuesInfo(Map> m) { + super(m); + } + + public Lock readLock() { + return lock.readLock(); + } + + public Lock writeLock() { + return lock.writeLock(); + } + + public void updateLogicalQueueRouteDataList(int logicalQueueIdx, + List logicalQueueRouteDataList) { + this.writeLock().lock(); + try { + logicalQueueRouteDataList = Lists.newLinkedList(logicalQueueRouteDataList); + List queueRouteDataList = this.get(logicalQueueIdx); + for (LogicalQueueRouteData logicalQueueRouteData : queueRouteDataList) { + for (Iterator it = logicalQueueRouteDataList.iterator(); it.hasNext(); ) { + LogicalQueueRouteData newQueueRouteData = it.next(); + if (Objects.equal(newQueueRouteData.getMessageQueue(), logicalQueueRouteData.getMessageQueue()) && newQueueRouteData.getOffsetDelta() == logicalQueueRouteData.getOffsetDelta()) { + logicalQueueRouteData.copyFrom(newQueueRouteData); + it.remove(); + break; + } + } + if (logicalQueueRouteDataList.isEmpty()) { + break; + } + } + for (LogicalQueueRouteData queueRouteData : logicalQueueRouteDataList) { + int idx = Collections.binarySearch(queueRouteDataList, queueRouteData); + if (idx < 0) { + idx = -idx - 1; + } + queueRouteDataList.add(idx, queueRouteData); + } + } finally { + this.writeLock().unlock(); + } + } + + static { + // workaround https://github.com/alibaba/fastjson/issues/3730 + ParserConfig.getGlobalInstance().putDeserializer(LogicalQueuesInfo.class, GenericMapSuperclassDeserializer.INSTANCE); + } +} diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/route/LogicalQueuesInfoUnordered.java b/common/src/main/java/org/apache/rocketmq/common/protocol/route/LogicalQueuesInfoUnordered.java new file mode 100644 index 00000000000..73d6061eac8 --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/route/LogicalQueuesInfoUnordered.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.common.protocol.route; + +import com.alibaba.fastjson.parser.ParserConfig; +import com.google.common.base.Objects; +import com.google.common.collect.Lists; +import org.apache.rocketmq.common.fastjson.GenericMapSuperclassDeserializer; + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +/** + * Only used inside namesrv, between client and namesrv, to reduce cpu usage of namesrv + */ +public class LogicalQueuesInfoUnordered extends ConcurrentHashMap> { + static { + // workaround https://github.com/alibaba/fastjson/issues/3730 + ParserConfig.getGlobalInstance().putDeserializer(LogicalQueuesInfoUnordered.class, GenericMapSuperclassDeserializer.INSTANCE); + } + + public LogicalQueuesInfoUnordered() { + super(); + } + + public LogicalQueuesInfoUnordered(int size) { + super(size); + } + + public LogicalQueuesInfo toLogicalQueuesInfoOrdered() { + LogicalQueuesInfo logicalQueuesInfoOrdered = new LogicalQueuesInfo(); + for (Map.Entry> entry : this.entrySet()) { + List list = Lists.newArrayListWithExpectedSize(entry.getValue().size()); + for (LogicalQueueRouteData d : entry.getValue().values()) { + list.add(new LogicalQueueRouteData(d)); + } + Collections.sort(list); + logicalQueuesInfoOrdered.put(entry.getKey(), list); + } + return logicalQueuesInfoOrdered; + } + + public static class Key { + private final String brokerName; + private final int queueId; + + private final long offsetDelta; + + private final int hash; + + public Key(String brokerName, int queueId, long offsetDelta) { + this.brokerName = brokerName; + this.queueId = queueId; + this.offsetDelta = offsetDelta; + + this.hash = Objects.hashCode(brokerName, queueId, this.offsetDelta); + } + + public String getBrokerName() { + return brokerName; + } + + public int getQueueId() { + return queueId; + } + + public long getOffsetDelta() { + return offsetDelta; + } + + @Override public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + Key id = (Key) o; + return queueId == id.queueId && offsetDelta == id.offsetDelta && Objects.equal(brokerName, id.brokerName); + } + + @Override public int hashCode() { + return hash; + } + + @Override public String toString() { + return "Key{" + + "brokerName='" + brokerName + '\'' + + ", queueId=" + queueId + + ", offsetDelta=" + offsetDelta + + '}'; + } + } +} diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/route/MessageQueueRouteState.java b/common/src/main/java/org/apache/rocketmq/common/protocol/route/MessageQueueRouteState.java new file mode 100644 index 00000000000..e6b48fcbc26 --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/route/MessageQueueRouteState.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.common.protocol.route; + +public enum MessageQueueRouteState { + // do not change below order, since ordinal() is used + Expired, + ReadOnly, + Normal, + WriteOnly, + ; +} diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/route/TopicRouteData.java b/common/src/main/java/org/apache/rocketmq/common/protocol/route/TopicRouteData.java index e8f54b8d73e..4470a2a5169 100644 --- a/common/src/main/java/org/apache/rocketmq/common/protocol/route/TopicRouteData.java +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/route/TopicRouteData.java @@ -30,27 +30,32 @@ public class TopicRouteData extends RemotingSerializable { private List queueDatas; private List brokerDatas; private HashMap/* Filter Server */> filterServerTable; + private LogicalQueuesInfo logicalQueuesInfo; - public TopicRouteData cloneTopicRouteData() { - TopicRouteData topicRouteData = new TopicRouteData(); - topicRouteData.setQueueDatas(new ArrayList()); - topicRouteData.setBrokerDatas(new ArrayList()); - topicRouteData.setFilterServerTable(new HashMap>()); - topicRouteData.setOrderTopicConf(this.orderTopicConf); + public TopicRouteData() { + } + + public TopicRouteData(TopicRouteData topicRouteData) { + this.queueDatas = new ArrayList(); + this.brokerDatas = new ArrayList(); + this.filterServerTable = new HashMap>(); + this.orderTopicConf = topicRouteData.orderTopicConf; - if (this.queueDatas != null) { - topicRouteData.getQueueDatas().addAll(this.queueDatas); + if (topicRouteData.queueDatas != null) { + this.queueDatas.addAll(topicRouteData.queueDatas); } - if (this.brokerDatas != null) { - topicRouteData.getBrokerDatas().addAll(this.brokerDatas); + if (topicRouteData.brokerDatas != null) { + this.brokerDatas.addAll(topicRouteData.brokerDatas); } - if (this.filterServerTable != null) { - topicRouteData.getFilterServerTable().putAll(this.filterServerTable); + if (topicRouteData.filterServerTable != null) { + this.filterServerTable.putAll(topicRouteData.filterServerTable); } - return topicRouteData; + if (topicRouteData.logicalQueuesInfo != null) { + this.logicalQueuesInfo = new LogicalQueuesInfo(topicRouteData.logicalQueuesInfo); + } } public List getQueueDatas() { @@ -85,6 +90,14 @@ public void setOrderTopicConf(String orderTopicConf) { this.orderTopicConf = orderTopicConf; } + public LogicalQueuesInfo getLogicalQueuesInfo() { + return logicalQueuesInfo; + } + + public void setLogicalQueuesInfo(LogicalQueuesInfo logicalQueuesInfo) { + this.logicalQueuesInfo = logicalQueuesInfo; + } + @Override public int hashCode() { final int prime = 31; @@ -93,6 +106,7 @@ public int hashCode() { result = prime * result + ((orderTopicConf == null) ? 0 : orderTopicConf.hashCode()); result = prime * result + ((queueDatas == null) ? 0 : queueDatas.hashCode()); result = prime * result + ((filterServerTable == null) ? 0 : filterServerTable.hashCode()); + result = prime * result + ((logicalQueuesInfo == null) ? 0 : logicalQueuesInfo.hashCode()); return result; } @@ -125,12 +139,17 @@ public boolean equals(Object obj) { return false; } else if (!filterServerTable.equals(other.filterServerTable)) return false; + if (logicalQueuesInfo == null) { + if (other.logicalQueuesInfo != null) + return false; + } else if (!logicalQueuesInfo.equals(other.logicalQueuesInfo)) + return false; return true; } @Override public String toString() { return "TopicRouteData [orderTopicConf=" + orderTopicConf + ", queueDatas=" + queueDatas - + ", brokerDatas=" + brokerDatas + ", filterServerTable=" + filterServerTable + "]"; + + ", brokerDatas=" + brokerDatas + ", filterServerTable=" + filterServerTable + ", logicalQueuesInfo=" + logicalQueuesInfo + "]"; } } diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/route/TopicRouteDataNameSrv.java b/common/src/main/java/org/apache/rocketmq/common/protocol/route/TopicRouteDataNameSrv.java new file mode 100644 index 00000000000..e9fb84e9750 --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/route/TopicRouteDataNameSrv.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.common.protocol.route; + +import com.google.common.base.Objects; + +public class TopicRouteDataNameSrv extends TopicRouteData { + private LogicalQueuesInfoUnordered logicalQueuesInfoUnordered; + + public TopicRouteDataNameSrv() { + } + + public LogicalQueuesInfoUnordered getLogicalQueuesInfoUnordered() { + return logicalQueuesInfoUnordered; + } + + public void setLogicalQueuesInfoUnordered( + LogicalQueuesInfoUnordered logicalQueuesInfoUnordered) { + this.logicalQueuesInfoUnordered = logicalQueuesInfoUnordered; + } + + @Override public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + if (!super.equals(o)) + return false; + TopicRouteDataNameSrv srv = (TopicRouteDataNameSrv) o; + return Objects.equal(logicalQueuesInfoUnordered, srv.logicalQueuesInfoUnordered); + } + + @Override public int hashCode() { + return Objects.hashCode(super.hashCode(), logicalQueuesInfoUnordered); + } + + @Override public String toString() { + return "TopicRouteDataNameSrv{" + + "logicalQueuesInfoUnordered=" + logicalQueuesInfoUnordered + + "} " + super.toString(); + } + + public TopicRouteData toTopicRouteData() { + TopicRouteData topicRouteData = new TopicRouteData(this); + if (this.logicalQueuesInfoUnordered != null) { + topicRouteData.setLogicalQueuesInfo(this.logicalQueuesInfoUnordered.toLogicalQueuesInfoOrdered()); + } + return topicRouteData; + } +} diff --git a/common/src/main/java/org/apache/rocketmq/common/sysflag/MessageSysFlag.java b/common/src/main/java/org/apache/rocketmq/common/sysflag/MessageSysFlag.java index d534571eb55..9f39f483e69 100644 --- a/common/src/main/java/org/apache/rocketmq/common/sysflag/MessageSysFlag.java +++ b/common/src/main/java/org/apache/rocketmq/common/sysflag/MessageSysFlag.java @@ -25,6 +25,7 @@ public class MessageSysFlag { public final static int TRANSACTION_ROLLBACK_TYPE = 0x3 << 2; public final static int BORNHOST_V6_FLAG = 0x1 << 4; public final static int STOREHOSTADDRESS_V6_FLAG = 0x1 << 5; + public final static int LOGICAL_QUEUE_FLAG = 0x1 << 6; public static int getTransactionValue(final int flag) { return flag & TRANSACTION_ROLLBACK_TYPE; diff --git a/common/src/test/java/org/apache/rocketmq/common/fastjson/GenericMapSuperclassDeserializerTest.java b/common/src/test/java/org/apache/rocketmq/common/fastjson/GenericMapSuperclassDeserializerTest.java new file mode 100644 index 00000000000..5aa550130d7 --- /dev/null +++ b/common/src/test/java/org/apache/rocketmq/common/fastjson/GenericMapSuperclassDeserializerTest.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.common.fastjson; + +import com.alibaba.fastjson.JSON; +import java.util.Collections; +import java.util.concurrent.ConcurrentHashMap; +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.common.protocol.route.LogicalQueueRouteData; +import org.apache.rocketmq.common.protocol.route.LogicalQueuesInfo; +import org.apache.rocketmq.common.protocol.route.LogicalQueuesInfoUnordered; +import org.apache.rocketmq.common.protocol.route.MessageQueueRouteState; +import org.assertj.core.util.Lists; +import org.junit.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +public class GenericMapSuperclassDeserializerTest { + @Test + public void testLogicalQueuesInfo() throws Exception { + LogicalQueuesInfo logicalQueuesInfo = new LogicalQueuesInfo(); + logicalQueuesInfo.put(0, Lists.newArrayList(new LogicalQueueRouteData(1, 2, new MessageQueue("topic", "broker", 3), MessageQueueRouteState.Normal, 4, 5, 6, 7, "127.1.2.3"))); + + byte[] buf = JSON.toJSONBytes(logicalQueuesInfo); + + LogicalQueuesInfo newLogicalQueuesInfo = JSON.parseObject(buf, LogicalQueuesInfo.class); + + assertThat(newLogicalQueuesInfo).isEqualTo(logicalQueuesInfo); + } + + @Test + public void testLogicalQueuesInfoUnordered() throws Exception { + LogicalQueuesInfoUnordered logicalQueuesInfoUnordered = new LogicalQueuesInfoUnordered(); + MessageQueue mq = new MessageQueue("topic", "broker", 3); + logicalQueuesInfoUnordered.put(0, new ConcurrentHashMap(Collections.singletonMap(new LogicalQueuesInfoUnordered.Key(mq.getBrokerName(), mq.getQueueId(), 4), new LogicalQueueRouteData(1, 2, mq, MessageQueueRouteState.Normal, 4, 5, 6, 7, "127.1.2.3")))); + + byte[] buf = JSON.toJSONBytes(logicalQueuesInfoUnordered); + + LogicalQueuesInfoUnordered newLogicalQueuesInfoUnordered = JSON.parseObject(buf, LogicalQueuesInfoUnordered.class); + + assertThat(newLogicalQueuesInfoUnordered).isEqualTo(logicalQueuesInfoUnordered); + } +} \ No newline at end of file diff --git a/common/src/test/java/org/apache/rocketmq/common/protocol/route/TopicRouteDataTest.java b/common/src/test/java/org/apache/rocketmq/common/protocol/route/TopicRouteDataTest.java index 5c7c6d1d1ff..f72f8f488f0 100644 --- a/common/src/test/java/org/apache/rocketmq/common/protocol/route/TopicRouteDataTest.java +++ b/common/src/test/java/org/apache/rocketmq/common/protocol/route/TopicRouteDataTest.java @@ -18,18 +18,13 @@ package org.apache.rocketmq.common.protocol.route; -import org.apache.rocketmq.common.protocol.route.BrokerData; -import org.apache.rocketmq.common.protocol.route.QueueData; -import org.apache.rocketmq.common.protocol.route.TopicRouteData; -import org.apache.rocketmq.remoting.protocol.RemotingSerializable; -import org.junit.Test; - import java.util.ArrayList; import java.util.HashMap; import java.util.List; +import org.apache.rocketmq.remoting.protocol.RemotingSerializable; +import org.junit.Test; import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.within; public class TopicRouteDataTest { @@ -64,7 +59,7 @@ public void testTopicRouteDataClone() throws Exception { topicRouteData.setFilterServerTable(new HashMap>()); topicRouteData.setQueueDatas(queueDataList); - assertThat(topicRouteData.cloneTopicRouteData()).isEqualTo(topicRouteData); + assertThat(new TopicRouteData(topicRouteData)).isEqualTo(topicRouteData); } diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/RemotingSerializable.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/RemotingSerializable.java index f80ff14c107..46f47976e83 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/RemotingSerializable.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/RemotingSerializable.java @@ -18,9 +18,10 @@ import com.alibaba.fastjson.JSON; import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; public abstract class RemotingSerializable { - private final static Charset CHARSET_UTF8 = Charset.forName("UTF-8"); + public final static Charset CHARSET_UTF8 = StandardCharsets.UTF_8; public static byte[] encode(final Object obj) { final String json = toJson(obj, false); @@ -35,14 +36,17 @@ public static String toJson(final Object obj, boolean prettyFormat) { } public static T decode(final byte[] data, Class classOfT) { - final String json = new String(data, CHARSET_UTF8); - return fromJson(json, classOfT); + return fromJson(data, classOfT); } public static T fromJson(String json, Class classOfT) { return JSON.parseObject(json, classOfT); } + private static T fromJson(byte[] data, Class classOfT) { + return JSON.parseObject(data, classOfT); + } + public byte[] encode() { final String json = this.toJson(); if (json != null) { From 238f9bccc2c3d7a4eec28b4f6c60f9bc5047aeaf Mon Sep 17 00:00:00 2001 From: chenzlalvin Date: Wed, 23 Jun 2021 17:31:52 +0800 Subject: [PATCH 2/5] [RIP-21] submodule store & broker & srvutil --- .../rocketmq/broker/BrokerController.java | 104 ++- .../domain/LogicalQueuesInfoInBroker.java | 116 +++ .../rocketmq/broker/out/BrokerOuterAPI.java | 21 + .../plugin/AbstractPluginMessageStore.java | 5 + .../AbstractSendMessageProcessor.java | 173 ++++- .../processor/AdminBrokerProcessor.java | 714 +++++++++++++++++- .../processor/PullMessageProcessor.java | 141 +++- .../processor/SendMessageProcessor.java | 93 +-- .../broker/topic/TopicConfigManager.java | 117 ++- .../rocketmq/broker/BrokerOuterAPITest.java | 29 + .../processor/AdminBrokerProcessorTest.java | 368 ++++++++- .../processor/PullMessageProcessorTest.java | 101 +++ .../processor/SendMessageProcessorTest.java | 77 +- .../broker/topic/TopicConfigManagerTest.java | 138 ++++ .../srvutil/ConcurrentHashMapUtil.java | 50 ++ .../apache/rocketmq/store/CleanFilesHook.java | 23 + .../org/apache/rocketmq/store/CommitLog.java | 12 +- .../rocketmq/store/DefaultMessageStore.java | 64 +- .../rocketmq/store/MappedFileQueue.java | 29 +- .../apache/rocketmq/store/MessageStore.java | 10 + .../rocketmq/store/MappedFileQueueTest.java | 2 +- 21 files changed, 2196 insertions(+), 191 deletions(-) create mode 100644 broker/src/main/java/org/apache/rocketmq/broker/domain/LogicalQueuesInfoInBroker.java create mode 100644 broker/src/test/java/org/apache/rocketmq/broker/topic/TopicConfigManagerTest.java create mode 100644 srvutil/src/main/java/org/apache/rocketmq/srvutil/ConcurrentHashMapUtil.java create mode 100644 store/src/main/java/org/apache/rocketmq/store/CleanFilesHook.java diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java index e83bea23975..5eb916922f6 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java @@ -16,13 +16,18 @@ */ package org.apache.rocketmq.broker; +import com.google.common.collect.Maps; import java.io.IOException; import java.net.InetSocketAddress; +import java.util.AbstractMap; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Objects; +import java.util.Optional; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -33,6 +38,8 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import java.util.function.Function; +import java.util.stream.Collectors; import org.apache.rocketmq.acl.AccessValidator; import org.apache.rocketmq.broker.client.ClientHousekeepingService; import org.apache.rocketmq.broker.client.ConsumerIdsChangeListener; @@ -42,6 +49,7 @@ import org.apache.rocketmq.broker.client.net.Broker2Client; import org.apache.rocketmq.broker.client.rebalance.RebalanceLockManager; import org.apache.rocketmq.broker.dledger.DLedgerRoleChangeHandler; +import org.apache.rocketmq.broker.domain.LogicalQueuesInfoInBroker; import org.apache.rocketmq.broker.filter.CommitLogDispatcherCalcBitMap; import org.apache.rocketmq.broker.filter.ConsumerFilterManager; import org.apache.rocketmq.broker.filtersrv.FilterServerManager; @@ -79,9 +87,11 @@ import org.apache.rocketmq.broker.transaction.queue.TransactionalMessageBridge; import org.apache.rocketmq.broker.transaction.queue.TransactionalMessageServiceImpl; import org.apache.rocketmq.broker.util.ServiceProvider; +import org.apache.rocketmq.client.exception.MQBrokerException; import org.apache.rocketmq.common.BrokerConfig; import org.apache.rocketmq.common.Configuration; import org.apache.rocketmq.common.DataVersion; +import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.ThreadFactoryImpl; import org.apache.rocketmq.common.TopicConfig; import org.apache.rocketmq.common.UtilAll; @@ -89,13 +99,18 @@ import org.apache.rocketmq.common.constant.PermName; import org.apache.rocketmq.common.namesrv.RegisterBrokerResult; import org.apache.rocketmq.common.protocol.RequestCode; +import org.apache.rocketmq.common.protocol.body.ClusterInfo; import org.apache.rocketmq.common.protocol.body.TopicConfigSerializeWrapper; +import org.apache.rocketmq.common.protocol.route.LogicalQueuesInfo; import org.apache.rocketmq.common.stats.MomentStatsItem; import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.remoting.RPCHook; import org.apache.rocketmq.remoting.RemotingServer; import org.apache.rocketmq.remoting.common.TlsMode; +import org.apache.rocketmq.remoting.exception.RemotingConnectException; +import org.apache.rocketmq.remoting.exception.RemotingSendRequestException; +import org.apache.rocketmq.remoting.exception.RemotingTimeoutException; import org.apache.rocketmq.remoting.netty.NettyClientConfig; import org.apache.rocketmq.remoting.netty.NettyRemotingServer; import org.apache.rocketmq.remoting.netty.NettyRequestProcessor; @@ -159,6 +174,7 @@ public class BrokerController { private final BrokerStatsManager brokerStatsManager; private final List sendMessageHookList = new ArrayList(); private final List consumeMessageHookList = new ArrayList(); + private final ConcurrentMap brokerName2AddrMap = Maps.newConcurrentMap(); private MessageStore messageStore; private RemotingServer remotingServer; private RemotingServer fastRemotingServer; @@ -277,9 +293,9 @@ public boolean initialize() throws CloneNotSupportedException { if (result) { try { - this.messageStore = - new DefaultMessageStore(this.messageStoreConfig, this.brokerStatsManager, this.messageArrivingListener, - this.brokerConfig); + DefaultMessageStore messageStore = new DefaultMessageStore(this.messageStoreConfig, this.brokerStatsManager, this.messageArrivingListener, this.brokerConfig); + messageStore.registerCleanFileHook(topicConfigManager.getLogicalQueueCleanHook()); + this.messageStore = messageStore; if (messageStoreConfig.isEnableDLegerCommitLog()) { DLedgerRoleChangeHandler roleChangeHandler = new DLedgerRoleChangeHandler(this, (DefaultMessageStore) messageStore); ((DLedgerCommitLog)((DefaultMessageStore) messageStore).getCommitLog()).getdLedgerServer().getdLedgerLeaderElector().addRoleChangeHandler(roleChangeHandler); @@ -467,6 +483,14 @@ public void run() { }, 1000 * 10, 1000 * 60 * 2, TimeUnit.MILLISECONDS); } + this.scheduledExecutorService.scheduleAtFixedRate(() -> { + try { + BrokerController.this.refreshBrokerNameMapping(); + } catch (Exception e) { + log.error("ScheduledTask examineBrokerClusterInfo exception", e); + } + }, 10, 10, TimeUnit.SECONDS); + if (!messageStoreConfig.isEnableDLegerCommitLog()) { if (BrokerRole.SLAVE == this.messageStoreConfig.getBrokerRole()) { if (this.messageStoreConfig.getHaMasterAddress() != null && this.messageStoreConfig.getHaMasterAddress().length() >= 6) { @@ -593,6 +617,18 @@ private void initialRpcHooks() { } } + private void refreshBrokerNameMapping() throws InterruptedException, MQBrokerException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException { + ClusterInfo brokerClusterInfo = this.brokerOuterAPI.getBrokerClusterInfo(); + brokerClusterInfo.getBrokerAddrTable().forEach((brokerName, data) -> { + String masterBrokerAddr = data.getBrokerAddrs().get(MixAll.MASTER_ID); + this.brokerName2AddrMap.put(brokerName, masterBrokerAddr); + }); + } + + public String getBrokerAddrByName(String brokerName) { + return this.brokerName2AddrMap.get(brokerName); + } + public void registerProcessor() { /** * SendMessageProcessor @@ -1009,20 +1045,54 @@ public void run() { } public synchronized void registerIncrementBrokerData(TopicConfig topicConfig, DataVersion dataVersion) { - TopicConfig registerTopicConfig = topicConfig; - if (!PermName.isWriteable(this.getBrokerConfig().getBrokerPermission()) - || !PermName.isReadable(this.getBrokerConfig().getBrokerPermission())) { - registerTopicConfig = - new TopicConfig(topicConfig.getTopicName(), topicConfig.getReadQueueNums(), topicConfig.getWriteQueueNums(), - this.brokerConfig.getBrokerPermission()); + this.registerIncrementBrokerData(Collections.singletonList(topicConfig), dataVersion); + } + + public synchronized void registerIncrementBrokerData(List topicConfigList, DataVersion dataVersion) { + if (topicConfigList == null || topicConfigList.isEmpty()) { + return; } - ConcurrentMap topicConfigTable = new ConcurrentHashMap(); - topicConfigTable.put(topicConfig.getTopicName(), registerTopicConfig); TopicConfigSerializeWrapper topicConfigSerializeWrapper = new TopicConfigSerializeWrapper(); topicConfigSerializeWrapper.setDataVersion(dataVersion); + + ConcurrentMap topicConfigTable = topicConfigList.stream() + .map(topicConfig -> { + TopicConfig registerTopicConfig; + if (!PermName.isWriteable(this.getBrokerConfig().getBrokerPermission()) + || !PermName.isReadable(this.getBrokerConfig().getBrokerPermission())) { + registerTopicConfig = + new TopicConfig(topicConfig.getTopicName(), + topicConfig.getReadQueueNums(), + topicConfig.getWriteQueueNums(), + this.brokerConfig.getBrokerPermission()); + } else { + registerTopicConfig = new TopicConfig(topicConfig); + } + return registerTopicConfig; + }) + .collect(Collectors.toConcurrentMap(TopicConfig::getTopicName, Function.identity())); topicConfigSerializeWrapper.setTopicConfigTable(topicConfigTable); + String brokerName = this.brokerConfig.getBrokerName(); + Map logicalQueuesInfoMap = topicConfigList.stream() + .map(TopicConfig::getTopicName) + .map(topicName -> Optional.ofNullable(this.topicConfigManager.selectLogicalQueuesInfo(topicName)) + .map(info -> { + info.readLock().lock(); + try { + return new AbstractMap.SimpleImmutableEntry<>(topicName, new LogicalQueuesInfoInBroker(info, data -> Objects.equals(data.getBrokerName(), brokerName))); + } finally { + info.readLock().unlock(); + } + }) + .orElse(null)) + .filter(Objects::nonNull) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + if (!logicalQueuesInfoMap.isEmpty()) { + topicConfigSerializeWrapper.setLogicalQueuesInfoMap(logicalQueuesInfoMap); + } + doRegisterBrokerAll(true, false, topicConfigSerializeWrapper); } @@ -1032,13 +1102,21 @@ public synchronized void registerBrokerAll(final boolean checkOrderConfig, boole if (!PermName.isWriteable(this.getBrokerConfig().getBrokerPermission()) || !PermName.isReadable(this.getBrokerConfig().getBrokerPermission())) { ConcurrentHashMap topicConfigTable = new ConcurrentHashMap(); + Map logicalQueuesInfoMap = Maps.newHashMapWithExpectedSize(topicConfigWrapper.getTopicConfigTable().size()); for (TopicConfig topicConfig : topicConfigWrapper.getTopicConfigTable().values()) { + String topicName = topicConfig.getTopicName(); TopicConfig tmp = - new TopicConfig(topicConfig.getTopicName(), topicConfig.getReadQueueNums(), topicConfig.getWriteQueueNums(), + new TopicConfig(topicName, topicConfig.getReadQueueNums(), topicConfig.getWriteQueueNums(), this.brokerConfig.getBrokerPermission()); - topicConfigTable.put(topicConfig.getTopicName(), tmp); + topicConfigTable.put(topicName, tmp); + LogicalQueuesInfoInBroker logicalQueuesInfo = this.topicConfigManager.selectLogicalQueuesInfo(topicName); + if (logicalQueuesInfo != null) { + String brokerName = this.brokerConfig.getBrokerName(); + logicalQueuesInfoMap.put(topicName, new LogicalQueuesInfoInBroker(logicalQueuesInfo, data -> Objects.equals(data.getBrokerName(), brokerName))); + } } topicConfigWrapper.setTopicConfigTable(topicConfigTable); + topicConfigWrapper.setLogicalQueuesInfoMap(logicalQueuesInfoMap); } if (forceRegister || needRegister(this.brokerConfig.getBrokerClusterName(), diff --git a/broker/src/main/java/org/apache/rocketmq/broker/domain/LogicalQueuesInfoInBroker.java b/broker/src/main/java/org/apache/rocketmq/broker/domain/LogicalQueuesInfoInBroker.java new file mode 100644 index 00000000000..a6728c828b0 --- /dev/null +++ b/broker/src/main/java/org/apache/rocketmq/broker/domain/LogicalQueuesInfoInBroker.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.broker.domain; + +import com.alibaba.fastjson.parser.ParserConfig; +import com.google.common.collect.Maps; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ConcurrentNavigableMap; +import java.util.concurrent.ConcurrentSkipListMap; +import java.util.function.Predicate; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.apache.rocketmq.common.fastjson.GenericMapSuperclassDeserializer; +import org.apache.rocketmq.common.protocol.route.LogicalQueueRouteData; +import org.apache.rocketmq.common.protocol.route.LogicalQueuesInfo; +import org.apache.rocketmq.srvutil.ConcurrentHashMapUtil; + +import static java.util.Optional.ofNullable; + +public class LogicalQueuesInfoInBroker extends LogicalQueuesInfo { + private final ConcurrentMap> queueId2LogicalQueueMap = Maps.newConcurrentMap(); + + public LogicalQueuesInfoInBroker() { + } + + public LogicalQueuesInfoInBroker(LogicalQueuesInfoInBroker other) { + this(other, null); + } + + // deep copy + public LogicalQueuesInfoInBroker(LogicalQueuesInfoInBroker other, Predicate predicate) { + other.readLock().lock(); + try { + for (Entry> entry : other.entrySet()) { + Stream stream = entry.getValue().stream(); + if (predicate != null) { + stream = stream.filter(predicate); + } + this.put(entry.getKey(), stream.map(LogicalQueueRouteData::new).collect(Collectors.toList())); + } + } finally { + other.readLock().unlock(); + } + } + + public void updateQueueRouteDataByQueueId(int queueId, LogicalQueueRouteData queueRouteData) { + if (queueRouteData == null) { + return; + } + ConcurrentHashMapUtil.computeIfAbsent(queueId2LogicalQueueMap, queueId, k -> new ConcurrentSkipListMap<>()).put(queueRouteData.getOffsetDelta(), queueRouteData); + } + + /** + * find logical queue route data for message queues owned by this broker + */ + public LogicalQueueRouteData queryQueueRouteDataByQueueId(int queueId, long offset) { + ConcurrentNavigableMap m = this.queueId2LogicalQueueMap.get(queueId); + if (m == null || m.isEmpty()) { + return null; + } + Entry entry = m.floorEntry(offset); + if (entry == null) { + return null; + } + return entry.getValue(); + } + + public void deleteQueueRouteData(LogicalQueueRouteData logicalQueueRouteData) { + ConcurrentNavigableMap m = this.queueId2LogicalQueueMap.get(logicalQueueRouteData.getQueueId()); + if (m != null) { + m.remove(logicalQueueRouteData.getOffsetDelta(), logicalQueueRouteData); + } + } + + public LogicalQueueRouteData nextAvailableLogicalRouteData(LogicalQueueRouteData queueRouteData, + Predicate predicate) { + this.readLock().lock(); + try { + List queueRouteDataList = ofNullable(this.get(queueRouteData.getLogicalQueueIndex())).orElse(Collections.emptyList()); + int idx = Collections.binarySearch(queueRouteDataList, queueRouteData); + if (idx >= 0) { + for (int i = idx + 1, size = queueRouteDataList.size(); i < size; i++) { + LogicalQueueRouteData tmp = queueRouteDataList.get(i); + if (predicate.test(tmp)) { + return tmp; + } + } + } + } finally { + this.readLock().unlock(); + } + return null; + } + + static { + // workaround https://github.com/alibaba/fastjson/issues/3730 + ParserConfig.getGlobalInstance().putDeserializer(LogicalQueuesInfoInBroker.class, GenericMapSuperclassDeserializer.INSTANCE); + } +} diff --git a/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java b/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java index 252201a058e..4d33663bb90 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java @@ -34,6 +34,7 @@ import org.apache.rocketmq.common.namesrv.TopAddressing; import org.apache.rocketmq.common.protocol.RequestCode; import org.apache.rocketmq.common.protocol.ResponseCode; +import org.apache.rocketmq.common.protocol.body.ClusterInfo; import org.apache.rocketmq.common.protocol.body.ConsumerOffsetSerializeWrapper; import org.apache.rocketmq.common.protocol.body.KVTable; import org.apache.rocketmq.common.protocol.body.RegisterBrokerBody; @@ -48,6 +49,7 @@ import org.apache.rocketmq.common.protocol.route.TopicRouteData; import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.logging.InternalLoggerFactory; +import org.apache.rocketmq.remoting.InvokeCallback; import org.apache.rocketmq.remoting.RPCHook; import org.apache.rocketmq.remoting.RemotingClient; import org.apache.rocketmq.remoting.exception.RemotingCommandException; @@ -432,4 +434,23 @@ public TopicRouteData getTopicRouteInfoFromNameServer(final String topic, final throw new MQBrokerException(response.getCode(), response.getRemark()); } + + public ClusterInfo getBrokerClusterInfo() throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException, MQBrokerException { + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_BROKER_CLUSTER_INFO, null); + RemotingCommand response = this.remotingClient.invokeSync(null, request, 3_000); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + return ClusterInfo.decode(response.getBody(), ClusterInfo.class); + } + default: + break; + } + + throw new MQBrokerException(response.getCode(), response.getRemark()); + } + + public void forwardRequest(String brokerAddr, RemotingCommand request, long timeoutMillis, InvokeCallback invokeCallback) throws InterruptedException, RemotingSendRequestException, RemotingTimeoutException, RemotingTooMuchRequestException, RemotingConnectException { + this.remotingClient.invokeAsync(brokerAddr, request, timeoutMillis, invokeCallback); + } } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/plugin/AbstractPluginMessageStore.java b/broker/src/main/java/org/apache/rocketmq/broker/plugin/AbstractPluginMessageStore.java index 1db019bec81..db1b6267b09 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/plugin/AbstractPluginMessageStore.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/plugin/AbstractPluginMessageStore.java @@ -109,6 +109,11 @@ public long getMaxOffsetInQueue(String topic, int queueId) { return next.getMaxOffsetInQueue(topic, queueId); } + @Override + public long getMaxOffsetInQueue(String topic, int queueId, boolean committed) { + return next.getMaxOffsetInQueue(topic, queueId, committed); + } + @Override public long getMinOffsetInQueue(String topic, int queueId) { return next.getMinOffsetInQueue(topic, queueId); diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/AbstractSendMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/AbstractSendMessageProcessor.java index ced7c2014f4..4daa832dd0a 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/AbstractSendMessageProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/AbstractSendMessageProcessor.java @@ -16,19 +16,28 @@ */ package org.apache.rocketmq.broker.processor; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Maps; import io.netty.channel.ChannelHandlerContext; import java.net.InetSocketAddress; import java.net.SocketAddress; import java.util.List; +import java.util.Locale; import java.util.Map; +import java.util.Optional; import java.util.Random; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.LongAdder; import org.apache.rocketmq.broker.BrokerController; +import org.apache.rocketmq.broker.domain.LogicalQueuesInfoInBroker; import org.apache.rocketmq.broker.mqtrace.SendMessageContext; import org.apache.rocketmq.broker.mqtrace.SendMessageHook; -import org.apache.rocketmq.common.topic.TopicValidator; +import org.apache.rocketmq.broker.topic.TopicConfigManager; import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.TopicConfig; import org.apache.rocketmq.common.TopicFilterType; +import org.apache.rocketmq.common.TopicQueueId; import org.apache.rocketmq.common.constant.DBMsgConstants; import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.constant.PermName; @@ -42,17 +51,23 @@ import org.apache.rocketmq.common.protocol.header.SendMessageRequestHeader; import org.apache.rocketmq.common.protocol.header.SendMessageRequestHeaderV2; import org.apache.rocketmq.common.protocol.header.SendMessageResponseHeader; +import org.apache.rocketmq.common.protocol.route.LogicalQueueRouteData; import org.apache.rocketmq.common.sysflag.MessageSysFlag; import org.apache.rocketmq.common.sysflag.TopicSysFlag; +import org.apache.rocketmq.common.topic.TopicValidator; import org.apache.rocketmq.common.utils.ChannelUtil; import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.logging.InternalLoggerFactory; +import org.apache.rocketmq.remoting.CommandCustomHeader; import org.apache.rocketmq.remoting.common.RemotingHelper; import org.apache.rocketmq.remoting.exception.RemotingCommandException; import org.apache.rocketmq.remoting.netty.AsyncNettyRequestProcessor; import org.apache.rocketmq.remoting.netty.NettyRequestProcessor; import org.apache.rocketmq.remoting.protocol.RemotingCommand; +import org.apache.rocketmq.remoting.protocol.RemotingSerializable; +import org.apache.rocketmq.srvutil.ConcurrentHashMapUtil; import org.apache.rocketmq.store.MessageExtBrokerInner; +import org.apache.rocketmq.store.PutMessageResult; public abstract class AbstractSendMessageProcessor extends AsyncNettyRequestProcessor implements NettyRequestProcessor { protected static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); @@ -63,6 +78,8 @@ public abstract class AbstractSendMessageProcessor extends AsyncNettyRequestProc protected final SocketAddress storeHost; private List sendMessageHookList; + private final ConcurrentMap inFlyWritingCounterMap = Maps.newConcurrentMap(); + public AbstractSendMessageProcessor(final BrokerController brokerController) { this.brokerController = brokerController; this.storeHost = @@ -330,4 +347,158 @@ public void executeSendMessageHookAfter(final RemotingCommand response, final Se public boolean rejectRequest() { return false; } + + + public ConcurrentMap getInFlyWritingCounterMap() { + return inFlyWritingCounterMap; + } + + protected LogicalQueueContext buildLogicalQueueContext(String topic, int queueId, + RemotingCommand response) { + TopicConfigManager topicConfigManager = this.brokerController.getTopicConfigManager(); + LogicalQueuesInfoInBroker logicalQueuesInfo = topicConfigManager.selectLogicalQueuesInfo(topic); + if (logicalQueuesInfo == null) { + return noopLogicalQueueContext; + } + // writable route data will has largest offset + LogicalQueueRouteData curQueueRouteData = logicalQueuesInfo.queryQueueRouteDataByQueueId(queueId, Long.MAX_VALUE); + if (curQueueRouteData == null) { + // topic enabled logical queue, but some message queues are not converted or being converted + String msg = String.format(Locale.ENGLISH, "queueId %d not included in logical queue", queueId); + log.debug("buildLogicalQueueContext unexpected error, topic {} {}", topic, msg); + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark(msg); + return noopLogicalQueueContext; + } + LongAdder inFlyWritingCounter = ConcurrentHashMapUtil.computeIfAbsent(inFlyWritingCounterMap, new TopicQueueId(topic, queueId), ignore -> new LongAdder()); + return new LogicalQueueContext(topic, queueId, logicalQueuesInfo, curQueueRouteData, inFlyWritingCounter); + } + + protected class LogicalQueueContext { + private final String topic; + private final int queueId; + private final LogicalQueuesInfoInBroker logicalQueuesInfo; + private final LogicalQueueRouteData curQueueRouteData; + private final LongAdder inFlyWritingCounter; + + public LogicalQueueContext(String topic, int queueId, + LogicalQueuesInfoInBroker logicalQueuesInfo, + LogicalQueueRouteData curQueueRouteData, LongAdder inFlyWritingCounter) { + this.topic = topic; + this.queueId = queueId; + this.logicalQueuesInfo = logicalQueuesInfo; + this.curQueueRouteData = curQueueRouteData; + this.inFlyWritingCounter = inFlyWritingCounter; + } + + public CompletableFuture hookBeforePut(ChannelHandlerContext ctx, SendMessageRequestHeader requestHeader, + RemotingCommand request, RemotingCommand response) { + if (curQueueRouteData.isWritable()) { + this.inFlyWritingCounter.increment(); + return null; + } + int logicalQueueIdx = curQueueRouteData.getLogicalQueueIndex(); + List queueRouteDataList = logicalQueuesInfo.get(logicalQueueIdx); + LogicalQueueRouteData writableQueueRouteData = null; + for (int i = queueRouteDataList.size() - 1; i >= 0; i--) { + LogicalQueueRouteData queueRouteData = queueRouteDataList.get(i); + if (queueRouteData.isWritable()) { + writableQueueRouteData = queueRouteData; + break; + } + } + if (writableQueueRouteData == null) { + response.setCode(ResponseCode.NO_PERMISSION); + response.setRemark(String.format(Locale.ENGLISH, "broker[%s] topic[%s] queueId[%d] logicalQueueIdx[%d] not writable", AbstractSendMessageProcessor.this.brokerController.getBrokerConfig().getBrokerIP1(), topic, queueId, logicalQueueIdx)); + return CompletableFuture.completedFuture(response); + } + if ((Optional.ofNullable(requestHeader.getSysFlag()).orElse(0) & MessageSysFlag.LOGICAL_QUEUE_FLAG) > 0) { + // new client, use redirect + response.setCode(ResponseCode.NO_PERMISSION); + response.addExtField(MessageConst.PROPERTY_REDIRECT, "1"); + response.setBody(RemotingSerializable.encode(ImmutableList.of(curQueueRouteData, writableQueueRouteData))); + return CompletableFuture.completedFuture(response); + } else { + // old client, use forward + this.logicalQueueHookForward(ctx, writableQueueRouteData, requestHeader, request, response); + } + if (response.getCode() != -1) { + return CompletableFuture.completedFuture(response); + } else if (response.getCode() == ResponseCode.ASYNC_AND_RETURN_NULL) { + return CompletableFuture.completedFuture(null); + } + return null; + } + + private void logicalQueueHookForward(ChannelHandlerContext ctx, + LogicalQueueRouteData writableQueueRouteData, + SendMessageRequestHeader requestHeader, RemotingCommand request, + RemotingCommand response) { + response.setCode(ResponseCode.SUCCESS); + requestHeader.setQueueId(writableQueueRouteData.getQueueId()); + request.writeCustomHeader(requestHeader); + String brokerName = writableQueueRouteData.getBrokerName(); + BrokerController brokerController = AbstractSendMessageProcessor.this.brokerController; + String brokerAddr = brokerController.getBrokerAddrByName(brokerName); + if (brokerAddr == null) { + log.warn("getBrokerAddrByName brokerName={} got null, fallback to queueRouteData.getBrokerAddr()", brokerName); + brokerAddr = writableQueueRouteData.getBrokerAddr(); + } + if (brokerAddr == null) { + response.setCode(ResponseCode.SYSTEM_ERROR); + String msg = String.format(Locale.ENGLISH, "unknown brokerName %s", brokerName); + response.setRemark(msg); + log.warn("logicalQueueHookForward can not look up brokerName={}: {}", brokerName, requestHeader); + return; + } + try { + String finalBrokerAddr = brokerAddr; + brokerController.getBrokerOuterAPI().forwardRequest(brokerAddr, request, brokerController.getBrokerConfig().getForwardTimeout(), responseFuture -> { + RemotingCommand forwardResponse = responseFuture.getResponseCommand(); + if (forwardResponse == null) { + forwardResponse = response; + forwardResponse.setCode(ResponseCode.SYSTEM_ERROR); + if (!responseFuture.isSendRequestOK()) { + forwardResponse.setRemark(String.format(Locale.ENGLISH, "send request failed to %s: %s", finalBrokerAddr, responseFuture.getCause())); + } else if (responseFuture.isTimeout()) { + forwardResponse.setRemark(String.format(Locale.ENGLISH, "wait response from %s timeout: %dms", finalBrokerAddr, responseFuture.getTimeoutMillis())); + } else { + forwardResponse.setRemark(String.format(Locale.ENGLISH, "unknown reason. addr: %s, timeoutMillis: %d: %s", finalBrokerAddr, responseFuture.getTimeoutMillis(), responseFuture.getCause())); + } + } else { + CommandCustomHeader customHeader = forwardResponse.readCustomHeader(); + if (customHeader instanceof SendMessageResponseHeader) { + SendMessageResponseHeader responseHeader = (SendMessageResponseHeader) customHeader; + Integer forwardQueueId = responseHeader.getQueueId(); + forwardResponse.addExtField(MessageConst.PROPERTY_FORWARD_QUEUE_ID, forwardQueueId != null ? Integer.toString(forwardQueueId) : "null"); + responseHeader.setQueueId(requestHeader.getQueueId()); + // queueOffset should not be changed since forwarded broker will add delta to it. + } + } + AbstractSendMessageProcessor.this.doResponse(ctx, request, forwardResponse); + }); + response.setCode(ResponseCode.ASYNC_AND_RETURN_NULL); + } catch (Exception e) { + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("forward error"); + log.warn(String.format(Locale.ENGLISH, "logicalQueueHookForward to %s error", brokerAddr), e); + } + } + + public void hookAfterPut(CompletableFuture putMessageResult) { + Optional.ofNullable(putMessageResult).orElse(CompletableFuture.completedFuture(null)).whenComplete((result, throwable) -> { + this.inFlyWritingCounter.decrement(); + }); + } + } + + private final LogicalQueueContext noopLogicalQueueContext = new LogicalQueueContext(null, 0, null, null, null) { + @Override public CompletableFuture hookBeforePut(ChannelHandlerContext ctx, SendMessageRequestHeader requestHeader, + RemotingCommand request, RemotingCommand response) { + return null; + } + + @Override public void hookAfterPut(CompletableFuture putMessageResult) { + } + }; } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java index 0a1d214b87f..b3309e1e2e9 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java @@ -17,22 +17,49 @@ package org.apache.rocketmq.broker.processor; import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONObject; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import io.netty.channel.Channel; import io.netty.channel.ChannelHandlerContext; +import java.io.UnsupportedEncodingException; +import java.net.UnknownHostException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Properties; +import java.util.Set; +import java.util.TreeMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.LongAdder; +import java.util.stream.Collectors; import org.apache.rocketmq.acl.AccessValidator; import org.apache.rocketmq.acl.plain.PlainAccessValidator; import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.broker.client.ClientChannelInfo; import org.apache.rocketmq.broker.client.ConsumerGroupInfo; +import org.apache.rocketmq.broker.domain.LogicalQueuesInfoInBroker; import org.apache.rocketmq.broker.filter.ConsumerFilterData; import org.apache.rocketmq.broker.filter.ExpressionMessageFilter; -import org.apache.rocketmq.common.topic.TopicValidator; +import org.apache.rocketmq.broker.topic.TopicConfigManager; import org.apache.rocketmq.broker.transaction.queue.TransactionalMessageUtil; import org.apache.rocketmq.common.AclConfig; import org.apache.rocketmq.common.MQVersion; import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.PlainAccessConfig; import org.apache.rocketmq.common.TopicConfig; +import org.apache.rocketmq.common.TopicQueueId; import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.admin.ConsumeStats; import org.apache.rocketmq.common.admin.OffsetWrapper; @@ -53,23 +80,29 @@ import org.apache.rocketmq.common.protocol.body.ConsumeQueueData; import org.apache.rocketmq.common.protocol.body.ConsumeStatsList; import org.apache.rocketmq.common.protocol.body.ConsumerConnection; +import org.apache.rocketmq.common.protocol.body.CreateMessageQueueForLogicalQueueRequestBody; import org.apache.rocketmq.common.protocol.body.GroupList; import org.apache.rocketmq.common.protocol.body.KVTable; import org.apache.rocketmq.common.protocol.body.LockBatchRequestBody; import org.apache.rocketmq.common.protocol.body.LockBatchResponseBody; +import org.apache.rocketmq.common.protocol.body.MigrateLogicalQueueBody; import org.apache.rocketmq.common.protocol.body.ProducerConnection; import org.apache.rocketmq.common.protocol.body.QueryConsumeQueueResponseBody; import org.apache.rocketmq.common.protocol.body.QueryConsumeTimeSpanBody; import org.apache.rocketmq.common.protocol.body.QueryCorrectionOffsetBody; import org.apache.rocketmq.common.protocol.body.QueueTimeSpan; +import org.apache.rocketmq.common.protocol.body.ReuseTopicLogicalQueueRequestBody; +import org.apache.rocketmq.common.protocol.body.SealTopicLogicalQueueRequestBody; import org.apache.rocketmq.common.protocol.body.TopicList; import org.apache.rocketmq.common.protocol.body.UnlockBatchRequestBody; +import org.apache.rocketmq.common.protocol.body.UpdateTopicLogicalQueueMappingRequestBody; import org.apache.rocketmq.common.protocol.header.CloneGroupOffsetRequestHeader; import org.apache.rocketmq.common.protocol.header.ConsumeMessageDirectlyResultRequestHeader; import org.apache.rocketmq.common.protocol.header.CreateAccessConfigRequestHeader; import org.apache.rocketmq.common.protocol.header.CreateTopicRequestHeader; import org.apache.rocketmq.common.protocol.header.DeleteAccessConfigRequestHeader; import org.apache.rocketmq.common.protocol.header.DeleteSubscriptionGroupRequestHeader; +import org.apache.rocketmq.common.protocol.header.DeleteTopicLogicalQueueRequestHeader; import org.apache.rocketmq.common.protocol.header.DeleteTopicRequestHeader; import org.apache.rocketmq.common.protocol.header.GetAllTopicConfigResponseHeader; import org.apache.rocketmq.common.protocol.header.GetBrokerAclConfigResponseHeader; @@ -88,11 +121,13 @@ import org.apache.rocketmq.common.protocol.header.GetMinOffsetRequestHeader; import org.apache.rocketmq.common.protocol.header.GetMinOffsetResponseHeader; import org.apache.rocketmq.common.protocol.header.GetProducerConnectionListRequestHeader; +import org.apache.rocketmq.common.protocol.header.GetTopicConfigRequestHeader; import org.apache.rocketmq.common.protocol.header.GetTopicStatsInfoRequestHeader; import org.apache.rocketmq.common.protocol.header.QueryConsumeQueueRequestHeader; import org.apache.rocketmq.common.protocol.header.QueryConsumeTimeSpanRequestHeader; import org.apache.rocketmq.common.protocol.header.QueryCorrectionOffsetHeader; import org.apache.rocketmq.common.protocol.header.QueryTopicConsumeByWhoRequestHeader; +import org.apache.rocketmq.common.protocol.header.QueryTopicLogicalQueueMappingRequestHeader; import org.apache.rocketmq.common.protocol.header.ResetOffsetRequestHeader; import org.apache.rocketmq.common.protocol.header.ResumeCheckHalfMessageRequestHeader; import org.apache.rocketmq.common.protocol.header.SearchOffsetRequestHeader; @@ -102,9 +137,12 @@ import org.apache.rocketmq.common.protocol.header.filtersrv.RegisterFilterServerRequestHeader; import org.apache.rocketmq.common.protocol.header.filtersrv.RegisterFilterServerResponseHeader; import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData; +import org.apache.rocketmq.common.protocol.route.LogicalQueueRouteData; +import org.apache.rocketmq.common.protocol.route.MessageQueueRouteState; import org.apache.rocketmq.common.stats.StatsItem; import org.apache.rocketmq.common.stats.StatsSnapshot; import org.apache.rocketmq.common.subscription.SubscriptionGroupConfig; +import org.apache.rocketmq.common.topic.TopicValidator; import org.apache.rocketmq.filter.util.BitsArray; import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.logging.InternalLoggerFactory; @@ -116,6 +154,7 @@ import org.apache.rocketmq.remoting.protocol.LanguageCode; import org.apache.rocketmq.remoting.protocol.RemotingCommand; import org.apache.rocketmq.remoting.protocol.RemotingSerializable; +import org.apache.rocketmq.srvutil.ConcurrentHashMapUtil; import org.apache.rocketmq.store.ConsumeQueue; import org.apache.rocketmq.store.ConsumeQueueExt; import org.apache.rocketmq.store.DefaultMessageStore; @@ -126,18 +165,6 @@ import org.apache.rocketmq.store.PutMessageStatus; import org.apache.rocketmq.store.SelectMappedBufferResult; -import java.io.UnsupportedEncodingException; -import java.net.UnknownHostException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Properties; -import java.util.Set; -import java.util.concurrent.ConcurrentMap; - public class AdminBrokerProcessor extends AsyncNettyRequestProcessor implements NettyRequestProcessor { private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); private final BrokerController brokerController; @@ -234,6 +261,26 @@ public RemotingCommand processRequest(ChannelHandlerContext ctx, return resumeCheckHalfMessage(ctx, request); case RequestCode.GET_BROKER_CLUSTER_ACL_CONFIG: return getBrokerClusterAclConfig(ctx, request); + case RequestCode.GET_TOPIC_CONFIG: + return getTopicConfig(ctx, request); + case RequestCode.UPDATE_TOPIC_LOGICAL_QUEUE_MAPPING: + return updateTopicLogicalQueueMapping(ctx, request); + case RequestCode.DELETE_TOPIC_LOGICAL_QUEUE_MAPPING: + return deleteTopicLogicalQueueMapping(ctx, request); + case RequestCode.QUERY_TOPIC_LOGICAL_QUEUE_MAPPING: + return queryTopicLogicalQueueMapping(ctx, request); + case RequestCode.SEAL_TOPIC_LOGICAL_QUEUE: + return sealTopicLogicalQueue(ctx, request); + case RequestCode.REUSE_TOPIC_LOGICAL_QUEUE: + return reuseTopicLogicalQueue(ctx, request); + case RequestCode.CREATE_MESSAGE_QUEUE_FOR_LOGICAL_QUEUE: + return createMessageQueueForLogicalQueue(ctx, request); + case RequestCode.MIGRATE_TOPIC_LOGICAL_QUEUE_PREPARE: + return migrateTopicLogicalQueuePrepare(ctx, request); + case RequestCode.MIGRATE_TOPIC_LOGICAL_QUEUE_COMMIT: + return migrateTopicLogicalQueueCommit(ctx, request); + case RequestCode.MIGRATE_TOPIC_LOGICAL_QUEUE_NOTIFY: + return migrateTopicLogicalQueueNotify(ctx, request); default: break; } @@ -299,6 +346,7 @@ private synchronized RemotingCommand deleteTopic(ChannelHandlerContext ctx, if (this.brokerController.getBrokerConfig().isAutoDeleteUnusedStats()) { this.brokerController.getBrokerStatsManager().onTopicDeleted(requestHeader.getTopic()); } + this.brokerController.getTopicConfigManager().deleteQueueRouteData(requestHeader.getTopic()); response.setCode(ResponseCode.SUCCESS); response.setRemark(null); return response; @@ -573,7 +621,33 @@ private RemotingCommand getMaxOffset(ChannelHandlerContext ctx, final GetMaxOffsetRequestHeader requestHeader = (GetMaxOffsetRequestHeader) request.decodeCommandCustomHeader(GetMaxOffsetRequestHeader.class); - long offset = this.brokerController.getMessageStore().getMaxOffsetInQueue(requestHeader.getTopic(), requestHeader.getQueueId()); + String topic = requestHeader.getTopic(); + int queueId = requestHeader.getQueueId(); + + if (requestHeader.getLogicalQueue()) { + LogicalQueuesInfoInBroker logicalQueuesInfo = this.brokerController.getTopicConfigManager().selectLogicalQueuesInfo(topic); + if (logicalQueuesInfo != null) { + // max offset must be in the queue route with largest offset + LogicalQueueRouteData requestLogicalQueueRouteData = logicalQueuesInfo.queryQueueRouteDataByQueueId(queueId, Long.MAX_VALUE); + if (requestLogicalQueueRouteData != null) { + logicalQueuesInfo.readLock().lock(); + try { + List queueRouteDataList = logicalQueuesInfo.get(requestLogicalQueueRouteData.getLogicalQueueIndex()); + if (queueRouteDataList != null && !queueRouteDataList.isEmpty()) { + LogicalQueueRouteData selectedLogicalQueueRouteData = queueRouteDataList.get(queueRouteDataList.size() - 1); + if (!Objects.equals(selectedLogicalQueueRouteData.getMessageQueue(), new MessageQueue(topic, this.brokerController.getBrokerConfig().getBrokerName(), queueId))) { + log.info("getMaxOffset topic={} queueId={} not latest, redirect: {}", topic, queueId, selectedLogicalQueueRouteData); + response.addExtField(MessageConst.PROPERTY_REDIRECT, "1"); + } + } + } finally { + logicalQueuesInfo.readLock().unlock(); + } + } + } + } + + long offset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId, requestHeader.isCommitted()); responseHeader.setOffset(offset); @@ -1622,4 +1696,616 @@ private MessageExtBrokerInner toMessageExtBrokerInner(MessageExt msgExt) { inner.setWaitStoreMsgOK(false); return inner; } + + private RemotingCommand getTopicConfig(ChannelHandlerContext ctx, + RemotingCommand request) throws RemotingCommandException { + GetTopicConfigRequestHeader requestHeader = (GetTopicConfigRequestHeader) request.decodeCommandCustomHeader(GetTopicConfigRequestHeader.class); + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + + TopicConfig topicConfig = this.brokerController.getTopicConfigManager().getTopicConfigTable().get(requestHeader.getTopic()); + if (topicConfig == null) { + log.error("No topic in this broker, client: {} topic: {}", ctx.channel().remoteAddress(), requestHeader.getTopic()); + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("No topic in this broker. topic: " + requestHeader.getTopic()); + return response; + } + String content = JSONObject.toJSONString(topicConfig); + try { + response.setBody(content.getBytes(MixAll.DEFAULT_CHARSET)); + } catch (UnsupportedEncodingException e) { + log.error("UnsupportedEncodingException getTopicConfig: topic=" + topicConfig.getTopicName(), e); + + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("UnsupportedEncodingException " + e.getMessage()); + return response; + } + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + + return response; + } + + private RemotingCommand updateTopicLogicalQueueMapping(ChannelHandlerContext ctx, + RemotingCommand request) throws RemotingCommandException { + UpdateTopicLogicalQueueMappingRequestBody requestBody = RemotingSerializable.decode(request.getBody(), UpdateTopicLogicalQueueMappingRequestBody.class); + RemotingCommand response = RemotingCommand.createResponseCommand(null); + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("unknown error"); + if (requestBody == null) { + response.setRemark("decode null"); + return response; + } + String topic = requestBody.getTopic(); + int queueId = requestBody.getQueueId(); + String brokerName = this.brokerController.getBrokerConfig().getBrokerName(); + int logicalQueueIdx = requestBody.getLogicalQueueIdx(); + log.info("updateTopicLogicalQueueMapping topic={} queueId={} logicalQueueIndex={}", topic, queueId, logicalQueueIdx); + TopicConfigManager topicConfigManager = this.brokerController.getTopicConfigManager(); + TopicConfig topicConfig = topicConfigManager.selectTopicConfig(topic); + if (topicConfig == null) { + log.warn("updateTopicLogicalQueueMapping topic={} queueId={} logicalQueueIndex={} topic not exist", topic, queueId, logicalQueueIdx); + response.setRemark("topic not exist"); + return response; + } + + LogicalQueuesInfoInBroker logicalQueuesInfo; + LogicalQueueRouteData newQueueRouteData = new LogicalQueueRouteData(); + newQueueRouteData.setBrokerAddr(this.brokerController.getBrokerAddr()); + newQueueRouteData.setMessageQueue(new MessageQueue(topic, brokerName, queueId)); + if (logicalQueueIdx >= 0) { + // add logical queue + newQueueRouteData.setLogicalQueueIndex(logicalQueueIdx); + newQueueRouteData.setLogicalQueueDelta(0L); + newQueueRouteData.setState(MessageQueueRouteState.Normal); + logicalQueuesInfo = topicConfigManager.getOrCreateLogicalQueuesInfo(topic); + logicalQueuesInfo.writeLock().lock(); + try { + // verify whether this message queue is already set up + for (List queueRouteDataList : logicalQueuesInfo.values()) { + for (Iterator iterator = queueRouteDataList.iterator(); iterator.hasNext(); ) { + LogicalQueueRouteData queueRouteData = iterator.next(); + if (Objects.equals(queueRouteData.getMessageQueue(), newQueueRouteData.getMessageQueue())) { + if (queueRouteData.getLogicalQueueIndex() == logicalQueueIdx) { + log.info("updateTopicLogicalQueueMapping topic={} queueId={} logicalQueueIndex={} already set up", topic, queueId, logicalQueueIdx); + response.setCode(ResponseCode.SUCCESS); + response.setRemark("already set up"); + return response; + } else { + log.warn("updateTopicLogicalQueueMapping topic={} queueId={} already assigned logicalQueueIdx={}, will reassign as logicalQueueIdx={}", topic, queueRouteData.getMessageQueue(), queueRouteData.getLogicalQueueIndex(), newQueueRouteData.getLogicalQueueIndex()); + iterator.remove(); + break; + } + } + } + } + List queueRouteDataList = logicalQueuesInfo.computeIfAbsent(logicalQueueIdx, ignore -> Lists.newArrayListWithExpectedSize(1)); + int idx = Collections.binarySearch(queueRouteDataList, newQueueRouteData, Comparator.comparingLong(LogicalQueueRouteData::getLogicalQueueDelta).thenComparingInt(LogicalQueueRouteData::getStateOrdinal)); + if (idx >= 0) { + log.warn("updateTopicLogicalQueueMapping topic={} queueId={} logicalQueueIdx={} found same logicalQueueOffset and will replace, exist {}, new {}", topic, queueId, logicalQueueIdx, queueRouteDataList.get(idx), newQueueRouteData); + queueRouteDataList.set(idx, newQueueRouteData); + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("duplicate logicalQueueOffset found"); + } else { + idx = -idx - 1; + queueRouteDataList.add(idx, newQueueRouteData); + logicalQueuesInfo.updateQueueRouteDataByQueueId(newQueueRouteData.getQueueId(), newQueueRouteData); + response.setCode(ResponseCode.SUCCESS); + response.setRemark("set up"); + log.info("updateTopicLogicalQueueMapping topic={} queueId={} logicalQueueIdx={} added as #{}", topic, queueId, logicalQueueIdx, idx); + } + } finally { + logicalQueuesInfo.writeLock().unlock(); + } + } else { + // delete logical queue + logicalQueuesInfo = topicConfigManager.getOrCreateLogicalQueuesInfo(topic); + logicalQueuesInfo.writeLock().lock(); + try { + for (List queueRouteDataList : logicalQueuesInfo.values()) { + queueRouteDataList.removeIf(queueRouteData -> Objects.equals(queueRouteData.getMessageQueue(), newQueueRouteData.getMessageQueue())); + } + } finally { + logicalQueuesInfo.writeLock().unlock(); + } + logicalQueuesInfo.updateQueueRouteDataByQueueId(newQueueRouteData.getQueueId(), null); + this.brokerController.getSendMessageProcessor().getInFlyWritingCounterMap().remove(new TopicQueueId(topic, queueId)); + response.setCode(ResponseCode.SUCCESS); + response.setRemark("deleted"); + log.info("updateTopicLogicalQueueMapping topic={} queueId={} deleted as logical queue", topic, queueId, logicalQueueIdx); + } + + topicConfigManager.getDataVersion().nextVersion(); + topicConfigManager.persist(topic, logicalQueuesInfo); + this.brokerController.registerIncrementBrokerData(topicConfig, topicConfigManager.getDataVersion()); + + return response; + } + + private RemotingCommand deleteTopicLogicalQueueMapping(ChannelHandlerContext ctx, + RemotingCommand request) throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + DeleteTopicLogicalQueueRequestHeader requestHeader = + (DeleteTopicLogicalQueueRequestHeader) request.decodeCommandCustomHeader(DeleteTopicLogicalQueueRequestHeader.class); + String topic = requestHeader.getTopic(); + log.info("deleteTopicLogicalQueueMapping topic={}", topic); + + TopicConfigManager topicConfigManager = this.brokerController.getTopicConfigManager(); + LogicalQueuesInfoInBroker logicalQueuesInfo = topicConfigManager.selectLogicalQueuesInfo(topic); + if (logicalQueuesInfo == null) { + response.setCode(ResponseCode.SUCCESS); + response.setRemark("already deleted"); + log.info("deleteTopicLogicalQueueMapping topic={} already deleted", topic); + return response; + } + String brokerName = this.brokerController.getBrokerConfig().getBrokerName(); + long size = logicalQueuesInfo.values().stream().flatMap(Collection::stream).filter(v -> Objects.equals(v.getBrokerName(), brokerName)).count(); + if (size > 0) { + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark(String.format(Locale.ENGLISH, "still %d message queues", size)); + log.info("deleteTopicLogicalQueueMapping topic={} still {} message queues", topic, size); + return response; + } + topicConfigManager.deleteQueueRouteData(topic); + response.setCode(ResponseCode.SUCCESS); + response.setRemark("deleted"); + + topicConfigManager.getDataVersion().nextVersion(); + topicConfigManager.persist(topic, logicalQueuesInfo); + this.brokerController.registerIncrementBrokerData(topicConfigManager.selectTopicConfig(topic), topicConfigManager.getDataVersion()); + + log.info("deleteTopicLogicalQueueMapping topic={} deleted", topic); + return response; + } + + private RemotingCommand queryTopicLogicalQueueMapping(ChannelHandlerContext ctx, + RemotingCommand request) throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + QueryTopicLogicalQueueMappingRequestHeader requestHeader = + (QueryTopicLogicalQueueMappingRequestHeader) request.decodeCommandCustomHeader(QueryTopicLogicalQueueMappingRequestHeader.class); + String topic = requestHeader.getTopic(); + log.info("queryTopicLogicalQueueMapping topic={}", topic); + LogicalQueuesInfoInBroker logicalQueuesInfo = brokerController.getTopicConfigManager().selectLogicalQueuesInfo(topic); + TreeMap> result = null; + if (logicalQueuesInfo != null) { + result = Maps.newTreeMap(); + logicalQueuesInfo.readLock().lock(); + try { + for (Map.Entry> entry : logicalQueuesInfo.entrySet()) { + Integer k = entry.getKey(); + List v = entry.getValue(); + result.put(k, ImmutableList.copyOf(v)); + } + } finally { + logicalQueuesInfo.readLock().unlock(); + } + } + response.setCode(ResponseCode.SUCCESS); + response.setBody(RemotingSerializable.encode(result)); + return response; + } + + private void sealLogicalQueueRouteData(LogicalQueueRouteData queueRouteData, + MessageStore messageStore) throws TimeoutException, InterruptedException { + queueRouteData.setState(MessageQueueRouteState.ReadOnly); + + String topic = queueRouteData.getTopic(); + int queueId = queueRouteData.getQueueId(); + + // busy wait for all in-fly messages to be finished + TopicQueueId key = new TopicQueueId(topic, queueId); + long startTime = System.currentTimeMillis(); + while (true) { + LongAdder counter = this.brokerController.getSendMessageProcessor().getInFlyWritingCounterMap().get(key); + if (counter == null || counter.sum() == 0) { + break; + } + if (System.currentTimeMillis() - startTime > 10_000) { + throw new TimeoutException(); + } + Thread.sleep(100); + } + // busy wait for all CQ to be finished + while (messageStore.getMaxOffsetInQueue(topic, queueId, true) != messageStore.getMaxOffsetInQueue(topic, queueId, false)) { + if (System.currentTimeMillis() - startTime > 10_000) { + throw new TimeoutException(); + } + Thread.sleep(100); + } + + long firstMsgQueueOffset = messageStore.getMinOffsetInQueue(topic, queueId); + long lastMsgQueueOffset = messageStore.getMaxOffsetInQueue(topic, queueId, false) - 1; + long firstMsgTimeMillis = 0L; + long lastMsgTimeMillis = 0L; + boolean expired = false; + if (firstMsgQueueOffset == lastMsgQueueOffset) { + // no message at all + expired = true; + } else { + long minPhyOffset = messageStore.getMinPhyOffset(); + long lastMsgCommitLogOffset = lastMsgQueueOffset >= 0 ? messageStore.getCommitLogOffsetInQueue(topic, queueId, lastMsgQueueOffset) : -1; + if (lastMsgCommitLogOffset < minPhyOffset) { + // commitLog already cleaned + expired = true; + } else { + long firstMsgCommitLogOffset = firstMsgQueueOffset >= 0 ? messageStore.getCommitLogOffsetInQueue(topic, queueId, firstMsgQueueOffset) : -1; + MessageExt firstMsg = firstMsgCommitLogOffset >= 0 ? messageStore.lookMessageByOffset(firstMsgCommitLogOffset) : null; + firstMsgTimeMillis = firstMsg != null ? firstMsg.getStoreTimestamp() : 0L; + + MessageExt lastMsg = lastMsgCommitLogOffset >= 0 ? messageStore.lookMessageByOffset(lastMsgCommitLogOffset) : null; + lastMsgTimeMillis = lastMsg != null ? lastMsg.getStoreTimestamp() : 0L; + } + } + + queueRouteData.setOffsetMax(lastMsgQueueOffset + 1); + queueRouteData.setFirstMsgTimeMillis(firstMsgTimeMillis); + queueRouteData.setLastMsgTimeMillis(lastMsgTimeMillis); + + if (expired) { + queueRouteData.setState(MessageQueueRouteState.Expired); + } + } + + private RemotingCommand sealTopicLogicalQueue(ChannelHandlerContext ctx, RemotingCommand request) { + SealTopicLogicalQueueRequestBody requestBody = RemotingSerializable.decode(request.getBody(), SealTopicLogicalQueueRequestBody.class); + if (requestBody == null) { + return RemotingCommand.createResponseCommand(ResponseCode.SYSTEM_ERROR, "decode error"); + } + String topic = requestBody.getTopic(); + int queueId = requestBody.getQueueId(); + int logicalQueueIdx = requestBody.getLogicalQueueIndex(); + log.info("sealTopicLogicalQueue topic={} queueId={}", topic, queueId); + TopicConfigManager topicConfigManager = this.brokerController.getTopicConfigManager(); + TopicConfig topicConfig = topicConfigManager.selectTopicConfig(topic); + if (topicConfig == null) { + log.warn("sealTopicLogicalQueue topic={} queueId={} topic not exist", topic, queueId); + return RemotingCommand.createResponseCommand(ResponseCode.SYSTEM_ERROR, "topic not exist"); + } + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + response.setCode(ResponseCode.SUCCESS); + + LogicalQueueRouteData resultQueueRouteData; + + LogicalQueuesInfoInBroker logicalQueuesInfo = topicConfigManager.getOrCreateLogicalQueuesInfo(topic); + logicalQueuesInfo.readLock().lock(); + try { + List queueRouteDataList = logicalQueuesInfo.get(logicalQueueIdx); + if (queueRouteDataList == null) { + log.info("sealTopicLogicalQueue topic={} queueId={} logicalQueueIdx={} not exist", topic, queueId, logicalQueueIdx); + response.setRemark("logical queue not exist"); + return response; + } + List foundQueues = queueRouteDataList.stream() + .filter(queueRouteData -> queueId == queueRouteData.getQueueId()).collect(Collectors.toList()); + if (foundQueues.isEmpty()) { + log.info("sealTopicLogicalQueue topic={} queueId={} logicalQueueIdx={} queueId={} not exist", topic, queueId, logicalQueueIdx, queueId); + response.setRemark("message queue not exist"); + return response; + } + Optional firstMainQueueRouteDataOptional = foundQueues.stream().filter(LogicalQueueRouteData::isWritable).findFirst(); + if (!firstMainQueueRouteDataOptional.isPresent()) { + log.info("sealTopicLogicalQueue topic={} queueId={} logicalQueueIdx={} queueId={} already sealed", topic, queueId, logicalQueueIdx, queueId); + response.setRemark("message queue already sealed"); + return response; + } + resultQueueRouteData = firstMainQueueRouteDataOptional.get(); + } finally { + logicalQueuesInfo.readLock().unlock(); + } + try { + sealLogicalQueueRouteData(resultQueueRouteData, brokerController.getMessageStore()); + } catch (InterruptedException e) { + return RemotingCommand.createResponseCommand(ResponseCode.SYSTEM_ERROR, "thread interrupted"); + } catch (TimeoutException e) { + return RemotingCommand.createResponseCommand(ResponseCode.SYSTEM_ERROR, "seal timeout"); + } + + topicConfigManager.getDataVersion().nextVersion(); + topicConfigManager.persist(topic, logicalQueuesInfo); + this.brokerController.registerIncrementBrokerData(topicConfigManager.selectTopicConfig(topic), topicConfigManager.getDataVersion()); + + response.setBody(RemotingSerializable.encode(resultQueueRouteData)); + return response; + } + + private RemotingCommand reuseTopicLogicalQueue(ChannelHandlerContext ctx, RemotingCommand request) { + ReuseTopicLogicalQueueRequestBody requestBody = RemotingSerializable.decode(request.getBody(), ReuseTopicLogicalQueueRequestBody.class); + if (requestBody == null) { + return RemotingCommand.createResponseCommand(ResponseCode.SYSTEM_ERROR, "decode error"); + } + String topic = requestBody.getTopic(); + int queueId = requestBody.getQueueId(); + String brokerName = this.brokerController.getBrokerConfig().getBrokerName(); + MessageQueue mq = new MessageQueue(topic, brokerName, queueId); + int logicalQueueIdx = requestBody.getLogicalQueueIndex(); + MessageQueueRouteState messageQueueRouteState = requestBody.getMessageQueueRouteState(); + log.info("reuseTopicLogicalQueue topic={} queueId={} logicalQueueIdx={} messageQueueRouteState={}", topic, queueId, logicalQueueIdx, messageQueueRouteState); + TopicConfigManager topicConfigManager = this.brokerController.getTopicConfigManager(); + LogicalQueuesInfoInBroker logicalQueuesInfo = topicConfigManager.selectLogicalQueuesInfo(topic); + if (logicalQueuesInfo == null) { + return RemotingCommand.createResponseCommand(ResponseCode.SYSTEM_ERROR, "queue route data not found"); + } + if (queueId >= topicConfigManager.selectTopicConfig(topic).getWriteQueueNums()) { + return RemotingCommand.createResponseCommand(ResponseCode.SYSTEM_ERROR, "found no queue"); + } + logicalQueuesInfo.writeLock().lock(); + LogicalQueueRouteData queueRouteData = new LogicalQueueRouteData( + logicalQueueIdx, + MessageQueueRouteState.WriteOnly.equals(messageQueueRouteState) ? -1 : 0, + new MessageQueue(topic, brokerName, queueId), + messageQueueRouteState, + this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId, false), + -1, + -1, + -1, + this.brokerController.getBrokerAddr() + ); + try { + if (logicalQueuesInfo.values().stream().flatMap(Collection::stream).filter(v -> Objects.equals(v.getMessageQueue(), mq)).anyMatch(LogicalQueueRouteData::isWritable)) { + return RemotingCommand.createResponseCommand(ResponseCode.SYSTEM_ERROR, "queue writable"); + } + logicalQueuesInfo.computeIfAbsent(logicalQueueIdx, ignore -> Lists.newArrayListWithExpectedSize(1)).add(queueRouteData); + logicalQueuesInfo.updateQueueRouteDataByQueueId(queueId, queueRouteData); + this.brokerController.getSendMessageProcessor().getInFlyWritingCounterMap().remove(new TopicQueueId(topic, queueId)); + } finally { + logicalQueuesInfo.writeLock().unlock(); + } + + topicConfigManager.getDataVersion().nextVersion(); + topicConfigManager.persist(topic, logicalQueuesInfo); + this.brokerController.registerIncrementBrokerData(topicConfigManager.selectTopicConfig(topic), topicConfigManager.getDataVersion()); + + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + response.setCode(ResponseCode.SUCCESS); + response.setBody(RemotingSerializable.encode(queueRouteData)); + return response; + } + + private RemotingCommand createMessageQueueForLogicalQueue(ChannelHandlerContext ctx, RemotingCommand request) { + CreateMessageQueueForLogicalQueueRequestBody requestBody = RemotingSerializable.decode(request.getBody(), CreateMessageQueueForLogicalQueueRequestBody.class); + if (requestBody == null) { + return RemotingCommand.createResponseCommand(ResponseCode.SYSTEM_ERROR, "decode error"); + } + String topic = requestBody.getTopic(); + int logicalQueueIdx = requestBody.getLogicalQueueIndex(); + MessageQueueRouteState messageQueueStatus = requestBody.getMessageQueueStatus(); + log.info("createMessageQueueForLogicalQueue topic={} logicalQueueIdx={}", topic, logicalQueueIdx); + + TopicConfigManager topicConfigManager = this.brokerController.getTopicConfigManager(); + LogicalQueuesInfoInBroker logicalQueuesInfo; + LogicalQueueRouteData queueRouteData; + TopicConfig topicConfig; + while (true) { + topicConfig = topicConfigManager.selectTopicConfig(topic); + if (topicConfig == null || topicConfig.getWriteQueueNums() == 0) { + // create topic if not exist + topicConfig = ConcurrentHashMapUtil.computeIfAbsent(topicConfigManager.getTopicConfigTable(), topic, s -> new TopicConfig(topic, 0, 0, this.brokerController.getBrokerConfig().getBrokerPermission())); + logicalQueuesInfo = topicConfigManager.getOrCreateLogicalQueuesInfo(topic); + } else { + logicalQueuesInfo = topicConfigManager.selectLogicalQueuesInfo(topic); + } + if (topicConfig.getWriteQueueNums() > 0 && logicalQueuesInfo == null) { + return RemotingCommand.createResponseCommand(ResponseCode.SYSTEM_ERROR, "topic not enable logical queue"); + } + TopicConfig newTopicConfig = new TopicConfig(topicConfig); + newTopicConfig.setWriteQueueNums(newTopicConfig.getWriteQueueNums() + 1); + newTopicConfig.setReadQueueNums(newTopicConfig.getReadQueueNums() + 1); + + int queueId = newTopicConfig.getWriteQueueNums() - 1; + queueRouteData = new LogicalQueueRouteData(); + queueRouteData.setLogicalQueueIndex(logicalQueueIdx); + queueRouteData.setBrokerAddr(this.brokerController.getBrokerAddr()); + queueRouteData.setMessageQueue(new MessageQueue(topic, this.brokerController.getBrokerConfig().getBrokerName(), queueId)); + if (messageQueueStatus != null) { + queueRouteData.setState(messageQueueStatus); + switch (messageQueueStatus) { + case WriteOnly: + case Expired: + queueRouteData.setLogicalQueueDelta(-1L); + break; + default: + queueRouteData.setLogicalQueueDelta(0L); + } + } + + logicalQueuesInfo.writeLock().lock(); + try { + List l = logicalQueuesInfo.computeIfAbsent(logicalQueueIdx, i -> Lists.newArrayListWithExpectedSize(1)); + if (MessageQueueRouteState.WriteOnly.equals(messageQueueStatus) && l.stream().anyMatch(LogicalQueueRouteData::isWriteOnly)) { + return RemotingCommand.createResponseCommand(ResponseCode.SYSTEM_ERROR, "multiple WriteOnly queue"); + } else if (MessageQueueRouteState.Normal.equals(messageQueueStatus) && l.stream().anyMatch(LogicalQueueRouteData::isWritable)) { + return RemotingCommand.createResponseCommand(ResponseCode.SYSTEM_ERROR, "multiple writable queue"); + } + if (topicConfigManager.replaceTopicConfig(topic, topicConfig, newTopicConfig)) { + l.add(queueRouteData); + logicalQueuesInfo.updateQueueRouteDataByQueueId(queueId, queueRouteData); + break; + } + } finally { + logicalQueuesInfo.writeLock().unlock(); + } + } + + topicConfigManager.getDataVersion().nextVersion(); + topicConfigManager.persist(topic, logicalQueuesInfo); + this.brokerController.registerIncrementBrokerData(topicConfig, topicConfigManager.getDataVersion()); + + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + response.setCode(ResponseCode.SUCCESS); + response.setBody(RemotingSerializable.encode(queueRouteData)); + return response; + } + + private RemotingCommand migrateTopicLogicalQueuePrepare(ChannelHandlerContext ctx, RemotingCommand request) { + MigrateLogicalQueueBody reqRespBody = RemotingSerializable.decode(request.getBody(), MigrateLogicalQueueBody.class); + if (reqRespBody == null) { + return RemotingCommand.createResponseCommand(ResponseCode.SYSTEM_ERROR, "decode error"); + } + LogicalQueueRouteData fromQueueRouteData = reqRespBody.getFromQueueRouteData(); + LogicalQueueRouteData toQueueRouteData = reqRespBody.getToQueueRouteData(); + log.info("migrateTopicLogicalQueuePrepare fromQueueRouteData={} toQueueRouteData={}", fromQueueRouteData, toQueueRouteData); + final MessageQueue fromMessageQueue = fromQueueRouteData.getMessageQueue(); + final int fromQueueId = fromQueueRouteData.getQueueId(); + final long fromOffsetDelta = fromQueueRouteData.getOffsetDelta(); + int logicalQueueIndex = fromQueueRouteData.getLogicalQueueIndex(); + TopicConfigManager topicConfigManager = this.brokerController.getTopicConfigManager(); + String topic = fromQueueRouteData.getTopic(); + if (!topicConfigManager.getTopicConfigTable().containsKey(topic)) { + return RemotingCommand.createResponseCommand(ResponseCode.SYSTEM_ERROR, "topic not exist"); + } + LogicalQueuesInfoInBroker logicalQueuesInfo = topicConfigManager.selectLogicalQueuesInfo(topic); + if (logicalQueuesInfo == null) { + return RemotingCommand.createResponseCommand(ResponseCode.SYSTEM_ERROR, "topic not enable logical queue"); + } + logicalQueuesInfo.writeLock().lock(); + try { + List queueRouteDataList = logicalQueuesInfo.get(logicalQueueIndex); + if (queueRouteDataList == null) { + return RemotingCommand.createResponseCommand(ResponseCode.SYSTEM_ERROR, String.format(Locale.ENGLISH, "logical queue %d not exist", logicalQueueIndex)); + } + fromQueueRouteData = null; + for (LogicalQueueRouteData v : queueRouteDataList) { + if (v.isSameTo(fromMessageQueue, fromOffsetDelta)) { + fromQueueRouteData = v; + break; + } + } + if (fromQueueRouteData == null) { + return RemotingCommand.createResponseCommand(ResponseCode.SYSTEM_ERROR, String.format(Locale.ENGLISH, "message queue %d not exist", fromQueueId)); + } + if (!MessageQueueRouteState.Normal.equals(fromQueueRouteData.getState())) { + return RemotingCommand.createResponseCommand(ResponseCode.SYSTEM_ERROR, String.format(Locale.ENGLISH, "message queue %d not normal state", fromQueueId)); + } + reqRespBody.setFromQueueRouteData(fromQueueRouteData); + if (fromQueueRouteData.isWritable()) { + sealLogicalQueueRouteData(fromQueueRouteData, brokerController.getMessageStore()); + } + toQueueRouteData.setLogicalQueueDelta(fromQueueRouteData.getLogicalQueueDelta() + fromQueueRouteData.getMessagesCount()); + queueRouteDataList.add(toQueueRouteData); + } catch (InterruptedException e) { + return RemotingCommand.createResponseCommand(ResponseCode.SYSTEM_ERROR, "thread interrupted"); + } catch (TimeoutException e) { + return RemotingCommand.createResponseCommand(ResponseCode.SYSTEM_ERROR, "seal timeout"); + } finally { + logicalQueuesInfo.writeLock().unlock(); + } + + topicConfigManager.getDataVersion().nextVersion(); + topicConfigManager.persist(topic, logicalQueuesInfo); + this.brokerController.registerIncrementBrokerData(topicConfigManager.selectTopicConfig(topic), topicConfigManager.getDataVersion()); + + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + response.setCode(ResponseCode.SUCCESS); + response.setBody(RemotingSerializable.encode(reqRespBody)); + return response; + } + + private RemotingCommand migrateTopicLogicalQueueCommit(ChannelHandlerContext ctx, RemotingCommand request) { + MigrateLogicalQueueBody reqRespBody = RemotingSerializable.decode(request.getBody(), MigrateLogicalQueueBody.class); + if (reqRespBody == null) { + return RemotingCommand.createResponseCommand(ResponseCode.SYSTEM_ERROR, "decode error"); + } + LogicalQueueRouteData fromQueueRouteData = reqRespBody.getFromQueueRouteData(); + LogicalQueueRouteData toQueueRouteData = reqRespBody.getToQueueRouteData(); + log.info("migrateTopicLogicalQueueCommit toQueueRouteData={}", toQueueRouteData); + TopicConfigManager topicConfigManager = this.brokerController.getTopicConfigManager(); + String topic = toQueueRouteData.getTopic(); + int toQueueId = toQueueRouteData.getQueueId(); + if (!topicConfigManager.getTopicConfigTable().containsKey(topic)) { + return RemotingCommand.createResponseCommand(ResponseCode.SYSTEM_ERROR, "topic not exist"); + } + LogicalQueuesInfoInBroker logicalQueuesInfo = topicConfigManager.selectLogicalQueuesInfo(topic); + if (logicalQueuesInfo == null) { + return RemotingCommand.createResponseCommand(ResponseCode.SYSTEM_ERROR, "topic not enable logical queue"); + } + LogicalQueueRouteData queueRouteData; + logicalQueuesInfo.writeLock().lock(); + try { + List queueRouteDataList = logicalQueuesInfo.get(toQueueRouteData.getLogicalQueueIndex()); + if (queueRouteDataList == null) { + return RemotingCommand.createResponseCommand(ResponseCode.SYSTEM_ERROR, String.format(Locale.ENGLISH, "logical queue %d not exist", toQueueRouteData.getLogicalQueueIndex())); + } + queueRouteDataList.stream().filter(fromQueueRouteData::isSameTo).forEach(d -> { + d.copyFrom(fromQueueRouteData); + }); + queueRouteData = queueRouteDataList.stream().filter(toQueueRouteData::isSameTo).findFirst().orElse(null); + if (queueRouteData == null) { + return RemotingCommand.createResponseCommand(ResponseCode.SYSTEM_ERROR, String.format(Locale.ENGLISH, "message queue %d-%d not exist", toQueueId, toQueueRouteData.getOffsetDelta())); + } + queueRouteData.setLogicalQueueDelta(toQueueRouteData.getLogicalQueueDelta()); + queueRouteData.setState(MessageQueueRouteState.Normal); + toQueueRouteData.setState(MessageQueueRouteState.Normal); + if (toQueueRouteData.getBrokerAddr() != null) { + queueRouteData.setBrokerAddr(toQueueRouteData.getBrokerAddr()); + } + } finally { + logicalQueuesInfo.writeLock().unlock(); + } + + topicConfigManager.getDataVersion().nextVersion(); + topicConfigManager.persist(topic, logicalQueuesInfo); + this.brokerController.registerIncrementBrokerData(topicConfigManager.selectTopicConfig(topic), topicConfigManager.getDataVersion()); + + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + response.setCode(ResponseCode.SUCCESS); + response.setBody(RemotingSerializable.encode(reqRespBody)); + return response; + } + + private RemotingCommand migrateTopicLogicalQueueNotify(ChannelHandlerContext ctx, RemotingCommand request) { + MigrateLogicalQueueBody requestBody = RemotingSerializable.decode(request.getBody(), MigrateLogicalQueueBody.class); + if (requestBody == null) { + return RemotingCommand.createResponseCommand(ResponseCode.SYSTEM_ERROR, "decode error"); + } + LogicalQueueRouteData fromQueueRouteData = requestBody.getFromQueueRouteData(); + LogicalQueueRouteData toQueueRouteData = requestBody.getToQueueRouteData(); + log.info("migrateTopicLogicalQueueNotify fromQueueRouteData={} toQueueRouteData={}", fromQueueRouteData, toQueueRouteData); + TopicConfigManager topicConfigManager = this.brokerController.getTopicConfigManager(); + String topic = toQueueRouteData.getTopic(); + if (!topicConfigManager.getTopicConfigTable().containsKey(topic)) { + return RemotingCommand.createResponseCommand(ResponseCode.SYSTEM_ERROR, "topic not exist"); + } + LogicalQueuesInfoInBroker logicalQueuesInfo = topicConfigManager.selectLogicalQueuesInfo(topic); + if (logicalQueuesInfo == null) { + return RemotingCommand.createResponseCommand(ResponseCode.SYSTEM_ERROR, "topic not enable logical queue"); + } + List requestQueueRouteDataList = Lists.newArrayList(fromQueueRouteData, toQueueRouteData); + boolean toQueueRouteDataFound = false; + logicalQueuesInfo.writeLock().lock(); + try { + List queueRouteDataList = logicalQueuesInfo.computeIfAbsent(toQueueRouteData.getLogicalQueueIndex(), ignore -> Lists.newArrayListWithExpectedSize(1)); + for (LogicalQueueRouteData v : queueRouteDataList) { + for (Iterator iterator = requestQueueRouteDataList.iterator(); iterator.hasNext(); ) { + LogicalQueueRouteData queueRouteData = iterator.next(); + if (queueRouteData.isSameTo(v)) { + v.copyFrom(queueRouteData); + if (queueRouteData.getBrokerAddr() != null) { + v.setBrokerAddr(queueRouteData.getBrokerAddr()); + } + if (!toQueueRouteDataFound && toQueueRouteData.isSameTo(v)) { + toQueueRouteDataFound = true; + } + iterator.remove(); + break; + } + } + if (requestQueueRouteDataList.isEmpty()) { + break; + } + } + if (!queueRouteDataList.isEmpty() && !toQueueRouteDataFound) { + // if this broker has this logical queue before, it should add latest writable route here, so that SendMessage request can be proxied + queueRouteDataList.add(toQueueRouteData); + } + } finally { + logicalQueuesInfo.writeLock().unlock(); + } + + topicConfigManager.getDataVersion().nextVersion(); + topicConfigManager.persist(topic, logicalQueuesInfo); + this.brokerController.registerIncrementBrokerData(topicConfigManager.selectTopicConfig(topic), topicConfigManager.getDataVersion()); + + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + response.setCode(ResponseCode.SUCCESS); + return response; + } } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/PullMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/PullMessageProcessor.java index 8879a722fcf..e61ef11309e 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/PullMessageProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/PullMessageProcessor.java @@ -16,6 +16,7 @@ */ package org.apache.rocketmq.broker.processor; +import com.google.common.collect.ImmutableList; import io.netty.channel.Channel; import io.netty.channel.ChannelFuture; import io.netty.channel.ChannelFutureListener; @@ -25,6 +26,7 @@ import java.util.List; import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.broker.client.ConsumerGroupInfo; +import org.apache.rocketmq.broker.domain.LogicalQueuesInfoInBroker; import org.apache.rocketmq.broker.filter.ConsumerFilterData; import org.apache.rocketmq.broker.filter.ConsumerFilterManager; import org.apache.rocketmq.broker.filter.ExpressionForRetryMessageFilter; @@ -41,6 +43,7 @@ import org.apache.rocketmq.common.filter.ExpressionType; import org.apache.rocketmq.common.filter.FilterAPI; import org.apache.rocketmq.common.help.FAQUrl; +import org.apache.rocketmq.common.message.MessageConst; import org.apache.rocketmq.common.message.MessageDecoder; import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.common.protocol.ResponseCode; @@ -48,6 +51,8 @@ import org.apache.rocketmq.common.protocol.header.PullMessageResponseHeader; import org.apache.rocketmq.common.protocol.heartbeat.MessageModel; import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData; +import org.apache.rocketmq.common.protocol.route.LogicalQueueRouteData; +import org.apache.rocketmq.common.protocol.route.MessageQueueRouteState; import org.apache.rocketmq.common.protocol.topic.OffsetMovedEvent; import org.apache.rocketmq.common.subscription.SubscriptionGroupConfig; import org.apache.rocketmq.common.sysflag.MessageSysFlag; @@ -62,6 +67,7 @@ import org.apache.rocketmq.remoting.netty.NettyRequestProcessor; import org.apache.rocketmq.remoting.netty.RequestTask; import org.apache.rocketmq.remoting.protocol.RemotingCommand; +import org.apache.rocketmq.remoting.protocol.RemotingSerializable; import org.apache.rocketmq.store.GetMessageResult; import org.apache.rocketmq.store.MessageExtBrokerInner; import org.apache.rocketmq.store.MessageFilter; @@ -126,23 +132,25 @@ private RemotingCommand processRequest(final Channel channel, RemotingCommand re final long suspendTimeoutMillisLong = hasSuspendFlag ? requestHeader.getSuspendTimeoutMillis() : 0; - TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(requestHeader.getTopic()); + String topic = requestHeader.getTopic(); + TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(topic); if (null == topicConfig) { - log.error("the topic {} not exist, consumer: {}", requestHeader.getTopic(), RemotingHelper.parseChannelRemoteAddr(channel)); + log.error("the topic {} not exist, consumer: {}", topic, RemotingHelper.parseChannelRemoteAddr(channel)); response.setCode(ResponseCode.TOPIC_NOT_EXIST); - response.setRemark(String.format("topic[%s] not exist, apply first please! %s", requestHeader.getTopic(), FAQUrl.suggestTodo(FAQUrl.APPLY_TOPIC_URL))); + response.setRemark(String.format("topic[%s] not exist, apply first please! %s", topic, FAQUrl.suggestTodo(FAQUrl.APPLY_TOPIC_URL))); return response; } if (!PermName.isReadable(topicConfig.getPerm())) { response.setCode(ResponseCode.NO_PERMISSION); - response.setRemark("the topic[" + requestHeader.getTopic() + "] pulling message is forbidden"); + response.setRemark("the topic[" + topic + "] pulling message is forbidden"); return response; } - if (requestHeader.getQueueId() < 0 || requestHeader.getQueueId() >= topicConfig.getReadQueueNums()) { + int queueId = requestHeader.getQueueId(); + if (queueId < 0 || queueId >= topicConfig.getReadQueueNums()) { String errorInfo = String.format("queueId[%d] is illegal, topic:[%s] topicConfig.readQueueNums:[%d] consumer:[%s]", - requestHeader.getQueueId(), requestHeader.getTopic(), topicConfig.getReadQueueNums(), channel.remoteAddress()); + queueId, topic, topicConfig.getReadQueueNums(), channel.remoteAddress()); log.warn(errorInfo); response.setCode(ResponseCode.SYSTEM_ERROR); response.setRemark(errorInfo); @@ -154,11 +162,11 @@ private RemotingCommand processRequest(final Channel channel, RemotingCommand re if (hasSubscriptionFlag) { try { subscriptionData = FilterAPI.build( - requestHeader.getTopic(), requestHeader.getSubscription(), requestHeader.getExpressionType() + topic, requestHeader.getSubscription(), requestHeader.getExpressionType() ); if (!ExpressionType.isTagType(subscriptionData.getExpressionType())) { consumerFilterData = ConsumerFilterManager.build( - requestHeader.getTopic(), requestHeader.getConsumerGroup(), requestHeader.getSubscription(), + topic, requestHeader.getConsumerGroup(), requestHeader.getSubscription(), requestHeader.getExpressionType(), requestHeader.getSubVersion() ); assert consumerFilterData != null; @@ -187,9 +195,9 @@ private RemotingCommand processRequest(final Channel channel, RemotingCommand re return response; } - subscriptionData = consumerGroupInfo.findSubscriptionData(requestHeader.getTopic()); + subscriptionData = consumerGroupInfo.findSubscriptionData(topic); if (null == subscriptionData) { - log.warn("the consumer's subscription not exist, group: {}, topic:{}", requestHeader.getConsumerGroup(), requestHeader.getTopic()); + log.warn("the consumer's subscription not exist, group: {}, topic:{}", requestHeader.getConsumerGroup(), topic); response.setCode(ResponseCode.SUBSCRIPTION_NOT_EXIST); response.setRemark("the consumer's subscription not exist" + FAQUrl.suggestTodo(FAQUrl.SAME_GROUP_DIFFERENT_TOPIC)); return response; @@ -203,7 +211,7 @@ private RemotingCommand processRequest(final Channel channel, RemotingCommand re return response; } if (!ExpressionType.isTagType(subscriptionData.getExpressionType())) { - consumerFilterData = this.brokerController.getConsumerFilterManager().get(requestHeader.getTopic(), + consumerFilterData = this.brokerController.getConsumerFilterManager().get(topic, requestHeader.getConsumerGroup()); if (consumerFilterData == null) { response.setCode(ResponseCode.FILTER_DATA_NOT_EXIST); @@ -212,7 +220,7 @@ private RemotingCommand processRequest(final Channel channel, RemotingCommand re } if (consumerFilterData.getClientVersion() < requestHeader.getSubVersion()) { log.warn("The broker's consumer filter data is not latest, group: {}, topic: {}, serverV: {}, clientV: {}", - requestHeader.getConsumerGroup(), requestHeader.getTopic(), consumerFilterData.getClientVersion(), requestHeader.getSubVersion()); + requestHeader.getConsumerGroup(), topic, consumerFilterData.getClientVersion(), requestHeader.getSubVersion()); response.setCode(ResponseCode.FILTER_DATA_NOT_LATEST); response.setRemark("the consumer's consumer filter data not latest"); return response; @@ -236,13 +244,74 @@ private RemotingCommand processRequest(final Channel channel, RemotingCommand re this.brokerController.getConsumerFilterManager()); } + long offset = requestHeader.getQueueOffset(); + int maxMsgNums = requestHeader.getMaxMsgNums(); + + LogicalQueuesInfoInBroker logicalQueuesInfo = this.brokerController.getTopicConfigManager().selectLogicalQueuesInfo(topic); + LogicalQueueRouteData queueRouteData = null; + if (logicalQueuesInfo != null) { + int responseErrorCode = ResponseCode.SUCCESS; + queueRouteData = logicalQueuesInfo.queryQueueRouteDataByQueueId(queueId, offset); + if (queueRouteData != null) { + if (queueRouteData.isWriteOnly()) { + responseErrorCode = ResponseCode.PULL_NOT_FOUND; + response.setRemark("logical queue write only"); + } else if (queueRouteData.isExpired()) { + responseErrorCode = ResponseCode.PULL_RETRY_IMMEDIATELY; + response.setRemark("logical queue expired"); + prepareRedirectResponse(response, logicalQueuesInfo, queueRouteData); + } else if (MessageQueueRouteState.ReadOnly.equals(queueRouteData.getState()) && queueRouteData.getOffsetMax() >= 0) { + if (offset >= queueRouteData.getOffsetMax()) { + responseErrorCode = ResponseCode.PULL_RETRY_IMMEDIATELY; + response.setRemark("queue offset exceed offsetMax"); + prepareRedirectResponse(response, logicalQueuesInfo, queueRouteData); + } else if (offset + maxMsgNums > queueRouteData.getOffsetMax()) { + if ((queueRouteData.getOffsetMax() - 1 <= this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId)) && + (this.brokerController.getMessageStore().getCommitLogOffsetInQueue(topic, queueId, queueRouteData.getOffsetMax() - 1) < this.brokerController.getMessageStore().getMinPhyOffset())) { + responseErrorCode = ResponseCode.PULL_RETRY_IMMEDIATELY; + response.setRemark("queue offset removed"); + prepareRedirectResponse(response, logicalQueuesInfo, queueRouteData); + } else { + maxMsgNums = (int) (queueRouteData.getOffsetMax() - offset); + if (maxMsgNums <= 0) { + responseErrorCode = ResponseCode.PULL_RETRY_IMMEDIATELY; + response.setRemark("queue offset out of range"); + prepareRedirectResponse(response, logicalQueuesInfo, queueRouteData); + } + } + } + } + } else { + responseErrorCode = ResponseCode.PULL_RETRY_IMMEDIATELY; + response.setRemark("no suitable queue"); + response.addExtField(MessageConst.PROPERTY_REDIRECT, "1"); + // instruct client to refresh all + response.setBody(null); + queueRouteData = logicalQueuesInfo.queryQueueRouteDataByQueueId(queueId, 0L); + } + if (responseErrorCode != ResponseCode.SUCCESS) { + response.setCode(responseErrorCode); + responseHeader.setMinOffset(offset); + responseHeader.setMaxOffset(queueRouteData != null ? queueRouteData.getOffsetMax() : offset); + responseHeader.setNextBeginOffset(queueRouteData != null ? queueRouteData.getOffsetMax() : offset); + responseHeader.setSuggestWhichBrokerId(MixAll.MASTER_ID); + return response; + } + } + final GetMessageResult getMessageResult = - this.brokerController.getMessageStore().getMessage(requestHeader.getConsumerGroup(), requestHeader.getTopic(), - requestHeader.getQueueId(), requestHeader.getQueueOffset(), requestHeader.getMaxMsgNums(), messageFilter); + this.brokerController.getMessageStore().getMessage(requestHeader.getConsumerGroup(), topic, + queueId, offset, maxMsgNums, messageFilter); if (getMessageResult != null) { response.setRemark(getMessageResult.getStatus().name()); - responseHeader.setNextBeginOffset(getMessageResult.getNextBeginOffset()); + long nextBeginOffset = getMessageResult.getNextBeginOffset(); + if (queueRouteData != null && queueRouteData.getOffsetMax() >= 0 && nextBeginOffset > queueRouteData.getOffsetMax()) { + // prevent from pulling messages from next logical queue route data + nextBeginOffset = queueRouteData.getOffsetMax(); + } + responseHeader.setNextBeginOffset(nextBeginOffset); responseHeader.setMinOffset(getMessageResult.getMinOffset()); + // this does not need to be modified since it's not an accurate value under logical queue. responseHeader.setMaxOffset(getMessageResult.getMaxOffset()); if (getMessageResult.isSuggestPullingFromSlave()) { @@ -291,9 +360,9 @@ private RemotingCommand processRequest(final Channel channel, RemotingCommand re // XXX: warn and notify me log.info("the broker store no queue data, fix the request offset {} to {}, Topic: {} QueueId: {} Consumer Group: {}", requestHeader.getQueueOffset(), - getMessageResult.getNextBeginOffset(), - requestHeader.getTopic(), - requestHeader.getQueueId(), + nextBeginOffset, + topic, + queueId, requestHeader.getConsumerGroup() ); } else { @@ -318,7 +387,7 @@ private RemotingCommand processRequest(final Channel channel, RemotingCommand re case OFFSET_TOO_SMALL: response.setCode(ResponseCode.PULL_OFFSET_MOVED); log.info("the request offset too small. group={}, topic={}, requestOffset={}, brokerMinOffset={}, clientIp={}", - requestHeader.getConsumerGroup(), requestHeader.getTopic(), requestHeader.getQueueOffset(), + requestHeader.getConsumerGroup(), topic, requestHeader.getQueueOffset(), getMessageResult.getMinOffset(), channel.remoteAddress()); break; default: @@ -329,8 +398,8 @@ private RemotingCommand processRequest(final Channel channel, RemotingCommand re if (this.hasConsumeMessageHook()) { ConsumeMessageContext context = new ConsumeMessageContext(); context.setConsumerGroup(requestHeader.getConsumerGroup()); - context.setTopic(requestHeader.getTopic()); - context.setQueueId(requestHeader.getQueueId()); + context.setTopic(topic); + context.setQueueId(queueId); String owner = request.getExtFields().get(BrokerStatsManager.COMMERCIAL_OWNER); @@ -414,9 +483,6 @@ public void operationComplete(ChannelFuture future) throws Exception { pollingTimeMills = this.brokerController.getBrokerConfig().getShortPollingTimeMills(); } - String topic = requestHeader.getTopic(); - long offset = requestHeader.getQueueOffset(); - int queueId = requestHeader.getQueueId(); PullRequest pullRequest = new PullRequest(request, channel, pollingTimeMills, this.brokerController.getMessageStore().now(), offset, subscriptionData, messageFilter); this.brokerController.getPullRequestHoldService().suspendPullRequest(topic, queueId, pullRequest); @@ -424,6 +490,20 @@ public void operationComplete(ChannelFuture future) throws Exception { break; } + if (queueRouteData != null) { + logicalQueuesInfo.readLock().lock(); + try { + List queueRouteDataList = logicalQueuesInfo.get(queueRouteData.getLogicalQueueIndex()); + MessageQueue latestMessageQueue = queueRouteDataList.get(queueRouteDataList.size() - 1).getMessageQueue(); + if (!latestMessageQueue.getBrokerName().equals(brokerController.getBrokerConfig().getBrokerName()) || latestMessageQueue.getQueueId() != queueId) { + // There are other newer message queue, instruct client to refresh meta-data to access these + prepareRedirectResponse(response, logicalQueuesInfo, queueRouteData); + } + } finally { + logicalQueuesInfo.readLock().unlock(); + } + } + case ResponseCode.PULL_RETRY_IMMEDIATELY: break; case ResponseCode.PULL_OFFSET_MOVED: @@ -438,7 +518,7 @@ public void operationComplete(ChannelFuture future) throws Exception { event.setConsumerGroup(requestHeader.getConsumerGroup()); event.setMessageQueue(mq); event.setOffsetRequest(requestHeader.getQueueOffset()); - event.setOffsetNew(getMessageResult.getNextBeginOffset()); + event.setOffsetNew(nextBeginOffset); this.generateOffsetMovedEvent(event); log.warn( "PULL_OFFSET_MOVED:correction offset. topic={}, groupId={}, requestOffset={}, newOffset={}, suggestBrokerId={}", @@ -467,11 +547,20 @@ public void operationComplete(ChannelFuture future) throws Exception { && this.brokerController.getMessageStoreConfig().getBrokerRole() != BrokerRole.SLAVE; if (storeOffsetEnable) { this.brokerController.getConsumerOffsetManager().commitOffset(RemotingHelper.parseChannelRemoteAddr(channel), - requestHeader.getConsumerGroup(), requestHeader.getTopic(), requestHeader.getQueueId(), requestHeader.getCommitOffset()); + requestHeader.getConsumerGroup(), topic, queueId, requestHeader.getCommitOffset()); } return response; } + private void prepareRedirectResponse(RemotingCommand response, LogicalQueuesInfoInBroker logicalQueuesInfo, + LogicalQueueRouteData queueRouteData) { + LogicalQueueRouteData nextReadableLogicalQueueRouteData = logicalQueuesInfo.nextAvailableLogicalRouteData(queueRouteData, LogicalQueueRouteData::isReadable); + if (nextReadableLogicalQueueRouteData != null) { + response.addExtField(MessageConst.PROPERTY_REDIRECT, "1"); + response.setBody(RemotingSerializable.encode(ImmutableList.of(queueRouteData, nextReadableLogicalQueueRouteData))); + } + } + public boolean hasConsumeMessageHook() { return consumeMessageHookList != null && !this.consumeMessageHookList.isEmpty(); } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java index 724cf54c813..97b7e62850d 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java @@ -287,6 +287,12 @@ private CompletableFuture asyncSendMessage(ChannelHandlerContex MessageAccessor.putProperty(msgInner, MessageConst.PROPERTY_CLUSTER, clusterName); msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties())); + LogicalQueueContext logicalQueueContext = super.buildLogicalQueueContext(msgInner.getTopic(), msgInner.getQueueId(), response); + CompletableFuture future = logicalQueueContext.hookBeforePut(ctx, requestHeader, request, response); + if (future != null) { + return future; + } + CompletableFuture putMessageResult = null; Map origProps = MessageDecoder.string2messageProperties(requestHeader.getProperties()); String transFlag = origProps.get(MessageConst.PROPERTY_TRANSACTION_PREPARED); @@ -296,12 +302,14 @@ private CompletableFuture asyncSendMessage(ChannelHandlerContex response.setRemark( "the broker[" + this.brokerController.getBrokerConfig().getBrokerIP1() + "] sending transaction message is forbidden"); + logicalQueueContext.hookAfterPut(null); return CompletableFuture.completedFuture(response); } putMessageResult = this.brokerController.getTransactionalMessageService().asyncPrepareMessage(msgInner); } else { putMessageResult = this.brokerController.getMessageStore().asyncPutMessage(msgInner); } + logicalQueueContext.hookAfterPut(putMessageResult); return handlePutMessageResultFuture(putMessageResult, response, request, msgInner, responseHeader, mqtraceContext, ctx, queueIdInt); } @@ -362,82 +370,6 @@ private boolean handleRetryAndDLQ(SendMessageRequestHeader requestHeader, Remoti return true; } - private RemotingCommand sendMessage(final ChannelHandlerContext ctx, - final RemotingCommand request, - final SendMessageContext sendMessageContext, - final SendMessageRequestHeader requestHeader) throws RemotingCommandException { - - final RemotingCommand response = RemotingCommand.createResponseCommand(SendMessageResponseHeader.class); - final SendMessageResponseHeader responseHeader = (SendMessageResponseHeader)response.readCustomHeader(); - - response.setOpaque(request.getOpaque()); - - response.addExtField(MessageConst.PROPERTY_MSG_REGION, this.brokerController.getBrokerConfig().getRegionId()); - response.addExtField(MessageConst.PROPERTY_TRACE_SWITCH, String.valueOf(this.brokerController.getBrokerConfig().isTraceOn())); - - log.debug("receive SendMessage request command, {}", request); - - final long startTimstamp = this.brokerController.getBrokerConfig().getStartAcceptSendRequestTimeStamp(); - if (this.brokerController.getMessageStore().now() < startTimstamp) { - response.setCode(ResponseCode.SYSTEM_ERROR); - response.setRemark(String.format("broker unable to service, until %s", UtilAll.timeMillisToHumanString2(startTimstamp))); - return response; - } - - response.setCode(-1); - super.msgCheck(ctx, requestHeader, response); - if (response.getCode() != -1) { - return response; - } - - final byte[] body = request.getBody(); - - int queueIdInt = requestHeader.getQueueId(); - TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(requestHeader.getTopic()); - - if (queueIdInt < 0) { - queueIdInt = Math.abs(this.random.nextInt() % 99999999) % topicConfig.getWriteQueueNums(); - } - - MessageExtBrokerInner msgInner = new MessageExtBrokerInner(); - msgInner.setTopic(requestHeader.getTopic()); - msgInner.setQueueId(queueIdInt); - - if (!handleRetryAndDLQ(requestHeader, response, request, msgInner, topicConfig)) { - return response; - } - - msgInner.setBody(body); - msgInner.setFlag(requestHeader.getFlag()); - MessageAccessor.setProperties(msgInner, MessageDecoder.string2messageProperties(requestHeader.getProperties())); - msgInner.setBornTimestamp(requestHeader.getBornTimestamp()); - msgInner.setBornHost(ctx.channel().remoteAddress()); - msgInner.setStoreHost(this.getStoreHost()); - msgInner.setReconsumeTimes(requestHeader.getReconsumeTimes() == null ? 0 : requestHeader.getReconsumeTimes()); - String clusterName = this.brokerController.getBrokerConfig().getBrokerClusterName(); - MessageAccessor.putProperty(msgInner, MessageConst.PROPERTY_CLUSTER, clusterName); - msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties())); - PutMessageResult putMessageResult = null; - Map oriProps = MessageDecoder.string2messageProperties(requestHeader.getProperties()); - String traFlag = oriProps.get(MessageConst.PROPERTY_TRANSACTION_PREPARED); - if (traFlag != null && Boolean.parseBoolean(traFlag) - && !(msgInner.getReconsumeTimes() > 0 && msgInner.getDelayTimeLevel() > 0)) { //For client under version 4.6.1 - if (this.brokerController.getBrokerConfig().isRejectTransactionMessage()) { - response.setCode(ResponseCode.NO_PERMISSION); - response.setRemark( - "the broker[" + this.brokerController.getBrokerConfig().getBrokerIP1() - + "] sending transaction message is forbidden"); - return response; - } - putMessageResult = this.brokerController.getTransactionalMessageService().prepareMessage(msgInner); - } else { - putMessageResult = this.brokerController.getMessageStore().putMessage(msgInner); - } - - return handlePutMessageResult(putMessageResult, response, request, msgInner, responseHeader, sendMessageContext, ctx, queueIdInt); - - } - private RemotingCommand handlePutMessageResult(PutMessageResult putMessageResult, RemotingCommand response, RemotingCommand request, MessageExt msg, SendMessageResponseHeader responseHeader, SendMessageContext sendMessageContext, ChannelHandlerContext ctx, @@ -587,7 +519,16 @@ private CompletableFuture asyncSendBatchMessage(ChannelHandlerC String clusterName = this.brokerController.getBrokerConfig().getBrokerClusterName(); MessageAccessor.putProperty(messageExtBatch, MessageConst.PROPERTY_CLUSTER, clusterName); + LogicalQueueContext logicalQueueContext = super.buildLogicalQueueContext(messageExtBatch.getTopic(), messageExtBatch.getQueueId(), response); + CompletableFuture future = logicalQueueContext.hookBeforePut(ctx, requestHeader, request, response); + if (future != null) { + return future; + } + CompletableFuture putMessageResult = this.brokerController.getMessageStore().asyncPutMessages(messageExtBatch); + + logicalQueueContext.hookAfterPut(putMessageResult); + return handlePutMessageResultFuture(putMessageResult, response, request, messageExtBatch, responseHeader, mqtraceContext, ctx, queueIdInt); } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java b/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java index 1f2bb4d8ba4..bf690057c6c 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java @@ -17,16 +17,20 @@ package org.apache.rocketmq.broker.topic; import java.util.Iterator; +import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.Objects; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; +import java.util.stream.Collectors; import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.broker.BrokerPathConfigHelper; +import org.apache.rocketmq.broker.domain.LogicalQueuesInfoInBroker; import org.apache.rocketmq.common.ConfigManager; import org.apache.rocketmq.common.DataVersion; import org.apache.rocketmq.common.MixAll; @@ -35,10 +39,15 @@ import org.apache.rocketmq.common.constant.PermName; import org.apache.rocketmq.common.protocol.body.KVTable; import org.apache.rocketmq.common.protocol.body.TopicConfigSerializeWrapper; +import org.apache.rocketmq.common.protocol.route.LogicalQueueRouteData; import org.apache.rocketmq.common.sysflag.TopicSysFlag; import org.apache.rocketmq.common.topic.TopicValidator; import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.logging.InternalLoggerFactory; +import org.apache.rocketmq.srvutil.ConcurrentHashMapUtil; +import org.apache.rocketmq.store.CleanFilesHook; +import org.apache.rocketmq.store.DefaultMessageStore; +import org.apache.rocketmq.store.MessageStore; public class TopicConfigManager extends ConfigManager { private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); @@ -52,8 +61,19 @@ public class TopicConfigManager extends ConfigManager { private final DataVersion dataVersion = new DataVersion(); private transient BrokerController brokerController; - public TopicConfigManager() { - } + private final ConcurrentMap logicalQueuesInfoTable = new ConcurrentHashMap<>(); + private final CleanFilesHook logicalQueueCleanHook = new CleanFilesHook() { + @Override public void execute(DefaultMessageStore defaultMessageStore, long deleteCount) { + if (deleteCount == 0) { + return; + } + TopicConfigManager.this.logicalQueueClean(); + } + + @Override public String getName() { + return TopicConfigManager.class.getSimpleName() + ".logicalQueueCleanHook"; + } + }; public TopicConfigManager(BrokerController brokerController) { this.brokerController = brokerController; @@ -362,7 +382,7 @@ public void updateTopicConfig(final TopicConfig topicConfig) { this.dataVersion.nextVersion(); - this.persist(); + this.persist(topicConfig.getTopicName(), topicConfig); } public void updateOrderTopicConfig(final KVTable orderKVTableFromNs) { @@ -421,6 +441,8 @@ public void deleteTopicConfig(final String topic) { public TopicConfigSerializeWrapper buildTopicConfigSerializeWrapper() { TopicConfigSerializeWrapper topicConfigSerializeWrapper = new TopicConfigSerializeWrapper(); topicConfigSerializeWrapper.setTopicConfigTable(this.topicConfigTable); + String brokerName = this.brokerController.getBrokerConfig().getBrokerName(); + topicConfigSerializeWrapper.setLogicalQueuesInfoMap(this.logicalQueuesInfoTable.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, e -> new LogicalQueuesInfoInBroker(e.getValue(), data -> Objects.equals(data.getBrokerName(), brokerName))))); topicConfigSerializeWrapper.setDataVersion(this.dataVersion); return topicConfigSerializeWrapper; } @@ -452,6 +474,7 @@ public void decode(String jsonString) { public String encode(final boolean prettyFormat) { TopicConfigSerializeWrapper topicConfigSerializeWrapper = new TopicConfigSerializeWrapper(); topicConfigSerializeWrapper.setTopicConfigTable(this.topicConfigTable); + topicConfigSerializeWrapper.setLogicalQueuesInfoMap(this.logicalQueuesInfoTable.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, e -> new LogicalQueuesInfoInBroker(e.getValue())))); topicConfigSerializeWrapper.setDataVersion(this.dataVersion); return topicConfigSerializeWrapper.toJson(prettyFormat); } @@ -471,4 +494,92 @@ public DataVersion getDataVersion() { public ConcurrentMap getTopicConfigTable() { return topicConfigTable; } + + public LogicalQueuesInfoInBroker selectLogicalQueuesInfo(String topicName) { + return this.logicalQueuesInfoTable.get(topicName); + } + + public LogicalQueuesInfoInBroker getOrCreateLogicalQueuesInfo(String topicName) { + return ConcurrentHashMapUtil.computeIfAbsent(this.logicalQueuesInfoTable, topicName, ignored -> new LogicalQueuesInfoInBroker()); + } + + public boolean replaceTopicConfig(String topic, TopicConfig oldTopicConfig, TopicConfig newTopicConfig) { + boolean ok = this.topicConfigTable.replace(topic, oldTopicConfig, newTopicConfig); + if (ok) { + this.dataVersion.nextVersion(); + persist(topic, newTopicConfig); + } + return ok; + } + + public CleanFilesHook getLogicalQueueCleanHook() { + return logicalQueueCleanHook; + } + + void logicalQueueClean() { + String brokerName = this.brokerController.getBrokerConfig().getBrokerName(); + MessageStore messageStore = this.brokerController.getMessageStore(); + for (Entry entry : this.logicalQueuesInfoTable.entrySet()) { + String topic = entry.getKey(); + LogicalQueuesInfoInBroker logicalQueuesInfo = entry.getValue(); + Lock readLock = logicalQueuesInfo.readLock(); + Lock writeLock = logicalQueuesInfo.writeLock(); + boolean changed = false; + readLock.lock(); + try { + for (List list : logicalQueuesInfo.values()) { + while (!list.isEmpty()) { + LogicalQueueRouteData logicalQueueRouteData = list.get(0); + String brokerBelongs; + if (brokerName.equals(logicalQueueRouteData.getBrokerName())) { + if (logicalQueueRouteData.isWritable()) { + break; + } + boolean canRemove = logicalQueueRouteData.isExpired() || logicalQueueRouteData.getMessagesCount() == 0; + if (!canRemove) { + // do not use getMinOffsetInQueue method, since it is using ConsumeQueue data, but not CommitLog, CQ data is not accurate after CommitLog cleaning. + long commitLogOffset = messageStore.getCommitLogOffsetInQueue(topic, logicalQueueRouteData.getQueueId(), logicalQueueRouteData.getOffsetMax() - 1); + canRemove = commitLogOffset == 0 || messageStore.getMinPhyOffset() > commitLogOffset; + } + if (!canRemove) { + break; + } + brokerBelongs = "self"; + } else { + brokerBelongs = "other"; + } + readLock.unlock(); + writeLock.lock(); + try { + list.remove(0); + } finally { + readLock.lock(); + writeLock.unlock(); + } + log.info("logicalQueueClean remove {} broker {}", brokerBelongs, logicalQueueRouteData); + changed = true; + } + } + if (changed) { + logicalQueuesInfo = new LogicalQueuesInfoInBroker(logicalQueuesInfo); + } + } finally { + readLock.unlock(); + } + if (changed) { + this.dataVersion.nextVersion(); + this.persist(topic, logicalQueuesInfo); + this.brokerController.registerIncrementBrokerData(this.selectTopicConfig(topic), this.dataVersion); + log.info("registerIncrementBrokerData because logicalQueueClean: {}", topic); + } + } + } + + public void deleteQueueRouteData(String topic) { + if (this.logicalQueuesInfoTable.remove(topic) != null) { + log.info("delete queueRouteData config OK, topic: {}", topic); + this.dataVersion.nextVersion(); + persist(topic, (LogicalQueuesInfoInBroker) null); + } + } } diff --git a/broker/src/test/java/org/apache/rocketmq/broker/BrokerOuterAPITest.java b/broker/src/test/java/org/apache/rocketmq/broker/BrokerOuterAPITest.java index 9ea1eeee31b..339ed110c85 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/BrokerOuterAPITest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/BrokerOuterAPITest.java @@ -22,19 +22,26 @@ import com.google.common.collect.Lists; import io.netty.channel.ChannelHandlerContext; import java.lang.reflect.Field; +import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.concurrent.TimeUnit; import org.apache.rocketmq.broker.out.BrokerOuterAPI; import org.apache.rocketmq.common.BrokerConfig; +import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.namesrv.RegisterBrokerResult; +import org.apache.rocketmq.common.protocol.RequestCode; import org.apache.rocketmq.common.protocol.ResponseCode; +import org.apache.rocketmq.common.protocol.body.ClusterInfo; import org.apache.rocketmq.common.protocol.body.TopicConfigSerializeWrapper; import org.apache.rocketmq.common.protocol.header.namesrv.QueryDataVersionResponseHeader; import org.apache.rocketmq.common.protocol.header.namesrv.RegisterBrokerResponseHeader; +import org.apache.rocketmq.common.protocol.route.BrokerData; import org.apache.rocketmq.remoting.netty.NettyClientConfig; import org.apache.rocketmq.remoting.netty.NettyRemotingClient; import org.apache.rocketmq.remoting.netty.NettyServerConfig; import org.apache.rocketmq.remoting.protocol.RemotingCommand; +import org.apache.rocketmq.remoting.protocol.RemotingSerializable; import org.apache.rocketmq.store.MessageStore; import org.apache.rocketmq.store.config.MessageStoreConfig; import org.junit.Test; @@ -50,6 +57,8 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.argThat; +import static org.mockito.ArgumentMatchers.isNull; import static org.mockito.Mockito.when; @RunWith(MockitoJUnitRunner.class) @@ -182,6 +191,26 @@ public RemotingCommand answer(InvocationOnMock invocation) throws Throwable { assertEquals(2, registerBrokerResultList.size()); } + @Test + public void testGetBrokerClusterInfo() throws Exception { + init(); + brokerOuterAPI.start(); + + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + + ClusterInfo want = new ClusterInfo(); + want.setBrokerAddrTable(new HashMap<>(Collections.singletonMap("key", new BrokerData("cluster", "broker", new HashMap<>(Collections.singletonMap(MixAll.MASTER_ID, "127.0.0.1:10911")))))); + response.setBody(RemotingSerializable.encode(want)); + + when(nettyRemotingClient.invokeSync(isNull(), argThat(argument -> argument.getCode() == RequestCode.GET_BROKER_CLUSTER_INFO), anyLong())).thenReturn(response); + ClusterInfo got = brokerOuterAPI.getBrokerClusterInfo(); + + assertEquals(want, got); + } + + private RemotingCommand buildResponse(Boolean changed) { final RemotingCommand response = RemotingCommand.createResponseCommand(QueryDataVersionResponseHeader.class); final QueryDataVersionResponseHeader responseHeader = (QueryDataVersionResponseHeader) response.readCustomHeader(); diff --git a/broker/src/test/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessorTest.java b/broker/src/test/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessorTest.java index 2764761d39f..d6cc8f9bd8a 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessorTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessorTest.java @@ -16,26 +16,54 @@ */ package org.apache.rocketmq.broker.processor; +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.TypeReference; import com.google.common.collect.Sets; +import io.netty.channel.Channel; import io.netty.channel.ChannelHandlerContext; +import java.net.InetSocketAddress; +import java.net.UnknownHostException; +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.LongAdder; import org.apache.rocketmq.broker.BrokerController; +import org.apache.rocketmq.broker.domain.LogicalQueuesInfoInBroker; +import org.apache.rocketmq.broker.topic.TopicConfigManager; import org.apache.rocketmq.common.BrokerConfig; import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.TopicConfig; import org.apache.rocketmq.common.TopicFilterType; +import org.apache.rocketmq.common.TopicQueueId; import org.apache.rocketmq.common.constant.PermName; import org.apache.rocketmq.common.message.MessageAccessor; import org.apache.rocketmq.common.message.MessageConst; import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.common.protocol.RequestCode; import org.apache.rocketmq.common.protocol.ResponseCode; +import org.apache.rocketmq.common.protocol.body.CreateMessageQueueForLogicalQueueRequestBody; +import org.apache.rocketmq.common.protocol.body.MigrateLogicalQueueBody; +import org.apache.rocketmq.common.protocol.body.ReuseTopicLogicalQueueRequestBody; +import org.apache.rocketmq.common.protocol.body.SealTopicLogicalQueueRequestBody; +import org.apache.rocketmq.common.protocol.body.UpdateTopicLogicalQueueMappingRequestBody; import org.apache.rocketmq.common.protocol.header.CreateTopicRequestHeader; +import org.apache.rocketmq.common.protocol.header.DeleteTopicLogicalQueueRequestHeader; import org.apache.rocketmq.common.protocol.header.DeleteTopicRequestHeader; +import org.apache.rocketmq.common.protocol.header.GetTopicConfigRequestHeader; +import org.apache.rocketmq.common.protocol.header.QueryTopicLogicalQueueMappingRequestHeader; import org.apache.rocketmq.common.protocol.header.ResumeCheckHalfMessageRequestHeader; +import org.apache.rocketmq.common.protocol.route.LogicalQueueRouteData; +import org.apache.rocketmq.common.protocol.route.MessageQueueRouteState; import org.apache.rocketmq.common.topic.TopicValidator; import org.apache.rocketmq.remoting.exception.RemotingCommandException; import org.apache.rocketmq.remoting.netty.NettyClientConfig; import org.apache.rocketmq.remoting.netty.NettyServerConfig; import org.apache.rocketmq.remoting.protocol.RemotingCommand; +import org.apache.rocketmq.remoting.protocol.RemotingSerializable; import org.apache.rocketmq.store.AppendMessageResult; import org.apache.rocketmq.store.AppendMessageStatus; import org.apache.rocketmq.store.MappedFile; @@ -45,6 +73,7 @@ import org.apache.rocketmq.store.PutMessageStatus; import org.apache.rocketmq.store.SelectMappedBufferResult; import org.apache.rocketmq.store.config.MessageStoreConfig; +import org.assertj.core.util.Lists; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -52,12 +81,13 @@ import org.mockito.Spy; import org.mockito.junit.MockitoJUnitRunner; -import java.net.UnknownHostException; -import java.nio.ByteBuffer; -import java.util.Set; - import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @RunWith(MockitoJUnitRunner.class) @@ -68,32 +98,52 @@ public class AdminBrokerProcessorTest { @Mock private ChannelHandlerContext handlerContext; + @Mock + private Channel channel; + @Spy private BrokerController - brokerController = new BrokerController(new BrokerConfig(), new NettyServerConfig(), new NettyClientConfig(), - new MessageStoreConfig()); + brokerController = new BrokerController(new BrokerConfig(), new NettyServerConfig(), new NettyClientConfig(), + new MessageStoreConfig()); @Mock private MessageStore messageStore; + @Mock + private SendMessageProcessor sendMessageProcessor; + + @Mock + private ConcurrentMap inFlyWritingCouterMap; + private Set systemTopicSet; + private String topic; @Before - public void init() { + public void init() throws Exception { brokerController.setMessageStore(messageStore); + + doReturn(sendMessageProcessor).when(brokerController).getSendMessageProcessor(); + when(sendMessageProcessor.getInFlyWritingCounterMap()).thenReturn(inFlyWritingCouterMap); + adminBrokerProcessor = new AdminBrokerProcessor(brokerController); systemTopicSet = Sets.newHashSet( - TopicValidator.RMQ_SYS_SELF_TEST_TOPIC, - TopicValidator.RMQ_SYS_BENCHMARK_TOPIC, - TopicValidator.RMQ_SYS_SCHEDULE_TOPIC, - TopicValidator.RMQ_SYS_OFFSET_MOVED_EVENT, - TopicValidator.AUTO_CREATE_TOPIC_KEY_TOPIC, - this.brokerController.getBrokerConfig().getBrokerClusterName(), - this.brokerController.getBrokerConfig().getBrokerClusterName() + "_" + MixAll.REPLY_TOPIC_POSTFIX); + TopicValidator.RMQ_SYS_SELF_TEST_TOPIC, + TopicValidator.RMQ_SYS_BENCHMARK_TOPIC, + TopicValidator.RMQ_SYS_SCHEDULE_TOPIC, + TopicValidator.RMQ_SYS_OFFSET_MOVED_EVENT, + TopicValidator.AUTO_CREATE_TOPIC_KEY_TOPIC, + this.brokerController.getBrokerConfig().getBrokerClusterName(), + this.brokerController.getBrokerConfig().getBrokerClusterName() + "_" + MixAll.REPLY_TOPIC_POSTFIX); if (this.brokerController.getBrokerConfig().isTraceTopicEnable()) { systemTopicSet.add(this.brokerController.getBrokerConfig().getMsgTraceTopicName()); } + when(handlerContext.channel()).thenReturn(channel); + when(channel.remoteAddress()).thenReturn(new InetSocketAddress("127.0.0.1", 12345)); + + topic = "FooBar" + System.nanoTime(); + TopicConfigManager topicConfigManager = brokerController.getTopicConfigManager(); + topicConfigManager.updateTopicConfig(new TopicConfig(topic)); } @Test @@ -101,7 +151,7 @@ public void testProcessRequest_success() throws RemotingCommandException, Unknow RemotingCommand request = createResumeCheckHalfMessageCommand(); when(messageStore.selectOneMessageByOffset(any(Long.class))).thenReturn(createSelectMappedBufferResult()); when(messageStore.putMessage(any(MessageExtBrokerInner.class))).thenReturn(new PutMessageResult - (PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK))); + (PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK))); RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request); assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); } @@ -111,7 +161,7 @@ public void testProcessRequest_fail() throws RemotingCommandException, UnknownHo RemotingCommand request = createResumeCheckHalfMessageCommand(); when(messageStore.selectOneMessageByOffset(any(Long.class))).thenReturn(createSelectMappedBufferResult()); when(messageStore.putMessage(any(MessageExtBrokerInner.class))).thenReturn(new PutMessageResult - (PutMessageStatus.UNKNOWN_ERROR, new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR))); + (PutMessageStatus.UNKNOWN_ERROR, new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR))); RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request); assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR); } @@ -155,6 +205,292 @@ public void testDeleteTopic() throws Exception { assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); } + @Test + public void testGetTopicConfig() throws Exception { + String topic = "foobar"; + brokerController.getTopicConfigManager().updateTopicConfig(new TopicConfig(topic)); + + { + GetTopicConfigRequestHeader requestHeader = new GetTopicConfigRequestHeader(); + requestHeader.setTopic(topic); + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_TOPIC_CONFIG, requestHeader); + request.makeCustomHeaderToNet(); + RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request); + assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); + assertThat(response.getBody()).isNotEmpty(); + } + { + GetTopicConfigRequestHeader requestHeader = new GetTopicConfigRequestHeader(); + requestHeader.setTopic("aaaaaaa"); + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_TOPIC_CONFIG, requestHeader); + request.makeCustomHeaderToNet(); + RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request); + assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR); + assertThat(response.getRemark()).contains("No topic in this broker."); + } + } + + @Test + public void testUpdateTopicLogicalQueueMapping() throws Exception { + int queueId = 0; + int logicalQueueIndex = 0; + + TopicConfigManager topicConfigManager = brokerController.getTopicConfigManager(); + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.UPDATE_TOPIC_LOGICAL_QUEUE_MAPPING, null); + UpdateTopicLogicalQueueMappingRequestBody requestBody = new UpdateTopicLogicalQueueMappingRequestBody(); + requestBody.setTopic(topic); + requestBody.setQueueId(queueId); + requestBody.setLogicalQueueIdx(logicalQueueIndex); + request.setBody(requestBody.encode()); + RemotingCommand response; + response = adminBrokerProcessor.processRequest(handlerContext, request); + assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); + assertThat(topicConfigManager.getOrCreateLogicalQueuesInfo(topic).get(logicalQueueIndex).get(0)).isEqualTo(new LogicalQueueRouteData(logicalQueueIndex, 0L, new MessageQueue(topic, brokerController.getBrokerConfig().getBrokerName(), queueId), MessageQueueRouteState.Normal, 0L, -1, -1, -1, brokerController.getBrokerAddr())); + + // delete + requestBody.setLogicalQueueIdx(-1); + request.setBody(requestBody.encode()); + response = adminBrokerProcessor.processRequest(handlerContext, request); + assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); + assertThat(topicConfigManager.getOrCreateLogicalQueuesInfo(topic).get(logicalQueueIndex)).isEmpty(); + verify(inFlyWritingCouterMap).remove(new TopicQueueId(topic, queueId)); + } + + @Test + public void testDeleteTopicLogicalQueueMapping() throws Exception { + int queueId = 0; + int logicalQueueIndex = 0; + TopicConfigManager topicConfigManager = brokerController.getTopicConfigManager(); + LogicalQueuesInfoInBroker logicalQueuesInfo = topicConfigManager.getOrCreateLogicalQueuesInfo(topic); + logicalQueuesInfo.put(logicalQueueIndex, Lists.newArrayList(new LogicalQueueRouteData(logicalQueueIndex, 0L, new MessageQueue(topic, brokerController.getBrokerConfig().getBrokerName(), queueId), MessageQueueRouteState.Normal, 0L, -1, -1, -1, brokerController.getBrokerAddr()))); + + DeleteTopicLogicalQueueRequestHeader requestHeader = new DeleteTopicLogicalQueueRequestHeader(); + requestHeader.setTopic(topic); + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.DELETE_TOPIC_LOGICAL_QUEUE_MAPPING, requestHeader); + request.makeCustomHeaderToNet(); + RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request); + assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR); + assertThat(response.getRemark()).isEqualTo("still 1 message queues"); + + logicalQueuesInfo.remove(logicalQueueIndex); + response = adminBrokerProcessor.processRequest(handlerContext, request); + assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); + assertThat(topicConfigManager.selectLogicalQueuesInfo(topic)).isNull(); + } + + @Test + public void testQueryTopicLogicalQueueMapping() throws Exception { + int queueId = 0; + int logicalQueueIndex = 0; + TopicConfigManager topicConfigManager = brokerController.getTopicConfigManager(); + LogicalQueuesInfoInBroker logicalQueuesInfo = topicConfigManager.getOrCreateLogicalQueuesInfo(topic); + logicalQueuesInfo.put(logicalQueueIndex, Lists.newArrayList(new LogicalQueueRouteData(logicalQueueIndex, 0L, new MessageQueue(topic, brokerController.getBrokerConfig().getBrokerName(), queueId), MessageQueueRouteState.Normal, 0L, -1, -1, -1, brokerController.getBrokerAddr()))); + + QueryTopicLogicalQueueMappingRequestHeader requestHeader = new QueryTopicLogicalQueueMappingRequestHeader(); + requestHeader.setTopic(topic); + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.QUERY_TOPIC_LOGICAL_QUEUE_MAPPING, requestHeader); + request.makeCustomHeaderToNet(); + RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request); + assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); + Map> m = JSON.parseObject(response.getBody(), new TypeReference>>() { + }.getType()); + assertThat(m.get(logicalQueueIndex)).isEqualTo(logicalQueuesInfo.get(logicalQueueIndex)); + } + + @Test + public void testSealTopicLogicalQueue() throws Exception { + int queueId = 0; + int logicalQueueIndex = 0; + TopicConfigManager topicConfigManager = brokerController.getTopicConfigManager(); + LogicalQueuesInfoInBroker logicalQueuesInfo = topicConfigManager.getOrCreateLogicalQueuesInfo(topic); + MessageQueue mq = new MessageQueue(topic, brokerController.getBrokerConfig().getBrokerName(), queueId); + logicalQueuesInfo.put(logicalQueueIndex, Lists.newArrayList(new LogicalQueueRouteData(logicalQueueIndex, 0L, mq, MessageQueueRouteState.Normal, 0L, -1, -1, -1, brokerController.getBrokerAddr()))); + + when(messageStore.getMaxOffsetInQueue(eq(topic), eq(queueId), anyBoolean())).thenReturn(100L); + when(messageStore.getMinOffsetInQueue(eq(topic), eq(queueId))).thenReturn(0L); + when(messageStore.getMinPhyOffset()).thenReturn(1000L); + when(messageStore.getCommitLogOffsetInQueue(eq(topic), eq(queueId), eq(0L))).thenReturn(2000L); + when(messageStore.getCommitLogOffsetInQueue(eq(topic), eq(queueId), eq(99L))).thenReturn(3000L); + MessageExt firstMsg = mock(MessageExt.class); + when(firstMsg.getStoreTimestamp()).thenReturn(200L); + when(messageStore.lookMessageByOffset(eq(2000L))).thenReturn(firstMsg); + MessageExt lastMsg = mock(MessageExt.class); + when(lastMsg.getStoreTimestamp()).thenReturn(300L); + when(messageStore.lookMessageByOffset(eq(3000L))).thenReturn(lastMsg); + + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.SEAL_TOPIC_LOGICAL_QUEUE, null); + SealTopicLogicalQueueRequestBody requestBody = new SealTopicLogicalQueueRequestBody(); + requestBody.setTopic(topic); + requestBody.setQueueId(queueId); + requestBody.setLogicalQueueIndex(logicalQueueIndex); + request.setBody(requestBody.encode()); + RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request); + assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); + LogicalQueueRouteData wantLogicalQueueRouteData = new LogicalQueueRouteData(logicalQueueIndex, 0L, mq, MessageQueueRouteState.ReadOnly, 0, 100, 200, 300, brokerController.getBrokerAddr()); + assertThat(logicalQueuesInfo.get(logicalQueueIndex).get(0)).isEqualTo(wantLogicalQueueRouteData); + assertThat((LogicalQueueRouteData) JSON.parseObject(response.getBody(), LogicalQueueRouteData.class)).isEqualTo(wantLogicalQueueRouteData); + + // expired + logicalQueuesInfo.put(logicalQueueIndex, Lists.newArrayList(new LogicalQueueRouteData(logicalQueueIndex, 0L, mq, MessageQueueRouteState.Normal, 0L, -1, -1, -1, brokerController.getBrokerAddr()))); + when(messageStore.getMinPhyOffset()).thenReturn(10000L); + response = adminBrokerProcessor.processRequest(handlerContext, request); + assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); + wantLogicalQueueRouteData = new LogicalQueueRouteData(logicalQueueIndex, 0L, mq, MessageQueueRouteState.Expired, 0, 100, 0, 0, brokerController.getBrokerAddr()); + assertThat(logicalQueuesInfo.get(logicalQueueIndex).get(0)).isEqualTo(wantLogicalQueueRouteData); + assertThat((LogicalQueueRouteData) JSON.parseObject(response.getBody(), LogicalQueueRouteData.class)).isEqualTo(wantLogicalQueueRouteData); + + // expired and empty + logicalQueuesInfo.put(logicalQueueIndex, Lists.newArrayList(new LogicalQueueRouteData(logicalQueueIndex, 0L, mq, MessageQueueRouteState.Normal, 0L, -1, -1, -1, brokerController.getBrokerAddr()))); + when(messageStore.getMinOffsetInQueue(eq(topic), eq(queueId))).thenReturn(100L); + response = adminBrokerProcessor.processRequest(handlerContext, request); + assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); + wantLogicalQueueRouteData = new LogicalQueueRouteData(logicalQueueIndex, 0L, mq, MessageQueueRouteState.Expired, 0, 100, 0, 0, brokerController.getBrokerAddr()); + assertThat(logicalQueuesInfo.get(logicalQueueIndex).get(0)).isEqualTo(wantLogicalQueueRouteData); + assertThat((LogicalQueueRouteData) JSON.parseObject(response.getBody(), LogicalQueueRouteData.class)).isEqualTo(wantLogicalQueueRouteData); + } + + @Test + public void testReuseTopicLogicalQueue() throws Exception { + int queueId = 0; + int logicalQueueIndex = 0; + TopicConfigManager topicConfigManager = brokerController.getTopicConfigManager(); + LogicalQueuesInfoInBroker logicalQueuesInfo = topicConfigManager.getOrCreateLogicalQueuesInfo(topic); + MessageQueue mq = new MessageQueue(topic, brokerController.getBrokerConfig().getBrokerName(), queueId); + LogicalQueueRouteData logicalQueueRouteData = new LogicalQueueRouteData(logicalQueueIndex, 500L, mq, MessageQueueRouteState.Expired, 100L, 200L, 300L, 400L, brokerController.getBrokerAddr()); + logicalQueuesInfo.put(logicalQueueIndex, Lists.newArrayList(logicalQueueRouteData)); + LogicalQueueRouteData wantData0 = new LogicalQueueRouteData(logicalQueueRouteData); + + when(messageStore.getMaxOffsetInQueue(eq(topic), eq(queueId), anyBoolean())).thenReturn(600L); + + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.REUSE_TOPIC_LOGICAL_QUEUE, null); + ReuseTopicLogicalQueueRequestBody requestBody = new ReuseTopicLogicalQueueRequestBody(); + requestBody.setTopic(topic); + requestBody.setQueueId(queueId); + requestBody.setLogicalQueueIndex(logicalQueueIndex); + requestBody.setMessageQueueRouteState(MessageQueueRouteState.WriteOnly); + request.setBody(requestBody.encode()); + + RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request); + assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); + LogicalQueueRouteData wantData1 = new LogicalQueueRouteData(logicalQueueIndex, -1L, mq, MessageQueueRouteState.WriteOnly, 600L, -1, -1, -1, brokerController.getBrokerAddr()); + assertThat((LogicalQueueRouteData) JSON.parseObject(response.getBody(), LogicalQueueRouteData.class)).isEqualTo(wantData1); + assertThat(logicalQueuesInfo.get(logicalQueueIndex)).isEqualTo(Arrays.asList(wantData0, wantData1)); + verify(inFlyWritingCouterMap).remove(new TopicQueueId(topic, queueId)); + } + + @Test + public void testCreateMessageQueueForLogicalQueue() throws Exception { + int logicalQueueIndex = 0; + TopicConfigManager topicConfigManager = brokerController.getTopicConfigManager(); + TopicConfig topicConfig = topicConfigManager.selectTopicConfig(topic); + topicConfig.setWriteQueueNums(0); + topicConfig.setReadQueueNums(0); + int queueId = 0; + assertThat(topicConfigManager.selectLogicalQueuesInfo(topic)).isNull(); + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CREATE_MESSAGE_QUEUE_FOR_LOGICAL_QUEUE, null); + CreateMessageQueueForLogicalQueueRequestBody requestBody = new CreateMessageQueueForLogicalQueueRequestBody(); + requestBody.setTopic(topic); + requestBody.setLogicalQueueIndex(logicalQueueIndex); + requestBody.setMessageQueueStatus(MessageQueueRouteState.WriteOnly); + request.setBody(requestBody.encode()); + + RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request); + assertThat(response.getCode()).withFailMessage("remark: %s", response.getRemark()).isEqualTo(ResponseCode.SUCCESS); + LogicalQueueRouteData wantLogicalQueueRouteData = new LogicalQueueRouteData(logicalQueueIndex, -1L, new MessageQueue(topic, brokerController.getBrokerConfig().getBrokerName(), queueId), MessageQueueRouteState.WriteOnly, 0L, -1, -1, -1, brokerController.getBrokerAddr()); + assertThat((LogicalQueueRouteData) JSON.parseObject(response.getBody(), LogicalQueueRouteData.class)).isEqualTo(wantLogicalQueueRouteData); + assertThat(topicConfigManager.selectLogicalQueuesInfo(topic).get(logicalQueueIndex).get(0)).isEqualTo(wantLogicalQueueRouteData); + } + + @Test + public void testMigrateTopicLogicalQueuePrepare() throws Exception { + int queueId = 0; + int logicalQueueIndex = 0; + TopicConfigManager topicConfigManager = brokerController.getTopicConfigManager(); + LogicalQueuesInfoInBroker logicalQueuesInfo = topicConfigManager.getOrCreateLogicalQueuesInfo(topic); + MessageQueue mq = new MessageQueue(topic, brokerController.getBrokerConfig().getBrokerName(), queueId); + LogicalQueueRouteData fromQueueRouteData = new LogicalQueueRouteData(logicalQueueIndex, 500L, mq, MessageQueueRouteState.Normal, 10L, -1L, -1L, -1L, brokerController.getBrokerAddr()); + logicalQueuesInfo.put(logicalQueueIndex, Lists.newArrayList(new LogicalQueueRouteData(fromQueueRouteData))); + + when(messageStore.getMaxOffsetInQueue(eq(topic), eq(queueId), anyBoolean())).thenReturn(100L); + when(messageStore.getMinOffsetInQueue(eq(topic), eq(queueId))).thenReturn(10L); + when(messageStore.getMinPhyOffset()).thenReturn(1000L); + when(messageStore.getCommitLogOffsetInQueue(eq(topic), eq(queueId), eq(10L))).thenReturn(2000L); + when(messageStore.getCommitLogOffsetInQueue(eq(topic), eq(queueId), eq(99L))).thenReturn(3000L); + MessageExt firstMsg = mock(MessageExt.class); + when(firstMsg.getStoreTimestamp()).thenReturn(200L); + when(messageStore.lookMessageByOffset(eq(2000L))).thenReturn(firstMsg); + MessageExt lastMsg = mock(MessageExt.class); + when(lastMsg.getStoreTimestamp()).thenReturn(300L); + when(messageStore.lookMessageByOffset(eq(3000L))).thenReturn(lastMsg); + + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.MIGRATE_TOPIC_LOGICAL_QUEUE_PREPARE, null); + MigrateLogicalQueueBody requestBody = new MigrateLogicalQueueBody(); + requestBody.setFromQueueRouteData(fromQueueRouteData); + LogicalQueueRouteData toQueueRouteData = new LogicalQueueRouteData(); + toQueueRouteData.setMessageQueue(new MessageQueue(topic, "toBroker", 1)); + requestBody.setToQueueRouteData(toQueueRouteData); + request.setBody(requestBody.encode()); + + RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request); + assertThat(response.getCode()).withFailMessage("remark: %s", response.getRemark()).isEqualTo(ResponseCode.SUCCESS); + fromQueueRouteData.setState(MessageQueueRouteState.ReadOnly); + fromQueueRouteData.setOffsetMax(100L); + fromQueueRouteData.setFirstMsgTimeMillis(200L); + fromQueueRouteData.setLastMsgTimeMillis(300L); + toQueueRouteData.setLogicalQueueDelta(590L); + MigrateLogicalQueueBody responseBody = RemotingSerializable.decode(response.getBody(), MigrateLogicalQueueBody.class); + assertThat(responseBody.getFromQueueRouteData()).isEqualTo(fromQueueRouteData); + assertThat(responseBody.getToQueueRouteData()).isEqualTo(toQueueRouteData); + assertThat(logicalQueuesInfo.get(logicalQueueIndex)).isEqualTo(Lists.newArrayList(fromQueueRouteData, toQueueRouteData)); + } + + @Test + public void testMigrateTopicLogicalQueueCommit() throws Exception { + int queueId = 0; + int logicalQueueIndex = 0; + TopicConfigManager topicConfigManager = brokerController.getTopicConfigManager(); + LogicalQueuesInfoInBroker logicalQueuesInfo = topicConfigManager.getOrCreateLogicalQueuesInfo(topic); + MessageQueue mq = new MessageQueue(topic, brokerController.getBrokerConfig().getBrokerName(), queueId); + LogicalQueueRouteData fromQueueRouteData = new LogicalQueueRouteData(); + fromQueueRouteData.setMessageQueue(new MessageQueue(topic, "fromBroker", 0)); + LogicalQueueRouteData toQueueRouteData = new LogicalQueueRouteData(logicalQueueIndex, 500L, mq, MessageQueueRouteState.Normal, 500L, -1L, -1L, -1L, brokerController.getBrokerAddr()); + logicalQueuesInfo.put(logicalQueueIndex, Lists.newArrayList(new LogicalQueueRouteData(toQueueRouteData))); + + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.MIGRATE_TOPIC_LOGICAL_QUEUE_COMMIT, null); + MigrateLogicalQueueBody requestBody = new MigrateLogicalQueueBody(); + requestBody.setFromQueueRouteData(fromQueueRouteData); + requestBody.setToQueueRouteData(toQueueRouteData); + request.setBody(requestBody.encode()); + + RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request); + assertThat(response.getCode()).withFailMessage("remark: %s", response.getRemark()).isEqualTo(ResponseCode.SUCCESS); + MigrateLogicalQueueBody responseBody = RemotingSerializable.decode(response.getBody(), MigrateLogicalQueueBody.class); + assertThat(responseBody.getFromQueueRouteData()).isEqualTo(fromQueueRouteData); + assertThat(responseBody.getToQueueRouteData()).isEqualTo(toQueueRouteData); + assertThat(logicalQueuesInfo.get(logicalQueueIndex)).isEqualTo(Lists.newArrayList(toQueueRouteData)); + } + + @Test + public void testMigrateTopicLogicalQueueNotify() throws Exception { + int queueId = 0; + int logicalQueueIndex = 0; + TopicConfigManager topicConfigManager = brokerController.getTopicConfigManager(); + LogicalQueuesInfoInBroker logicalQueuesInfo = topicConfigManager.getOrCreateLogicalQueuesInfo(topic); + LogicalQueueRouteData fromQueueRouteData = new LogicalQueueRouteData(logicalQueueIndex, 100L, new MessageQueue(topic, "fromBroker", queueId), MessageQueueRouteState.ReadOnly, 10L, 410L, 200L, 300L, brokerController.getBrokerAddr()); + LogicalQueueRouteData toQueueRouteData = new LogicalQueueRouteData(logicalQueueIndex, 500L, new MessageQueue(topic, "toBroker", queueId), MessageQueueRouteState.Normal, 500L, -1L, -1L, -1L, brokerController.getBrokerAddr()); + logicalQueuesInfo.put(logicalQueueIndex, Lists.newArrayList(new LogicalQueueRouteData(fromQueueRouteData))); + + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.MIGRATE_TOPIC_LOGICAL_QUEUE_NOTIFY, null); + MigrateLogicalQueueBody requestBody = new MigrateLogicalQueueBody(); + requestBody.setFromQueueRouteData(fromQueueRouteData); + requestBody.setToQueueRouteData(toQueueRouteData); + request.setBody(requestBody.encode()); + + RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request); + assertThat(response.getCode()).withFailMessage("remark: %s", response.getRemark()).isEqualTo(ResponseCode.SUCCESS); + assertThat(logicalQueuesInfo.get(logicalQueueIndex)).isEqualTo(Lists.newArrayList(fromQueueRouteData, toQueueRouteData)); + } + private RemotingCommand buildCreateTopicRequest(String topic) { CreateTopicRequestHeader requestHeader = new CreateTopicRequestHeader(); requestHeader.setTopic(topic); diff --git a/broker/src/test/java/org/apache/rocketmq/broker/processor/PullMessageProcessorTest.java b/broker/src/test/java/org/apache/rocketmq/broker/processor/PullMessageProcessorTest.java index c96f708e854..b6377553d7a 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/processor/PullMessageProcessorTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/processor/PullMessageProcessorTest.java @@ -20,24 +20,31 @@ import io.netty.channel.ChannelHandlerContext; import java.net.InetSocketAddress; import java.util.ArrayList; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Set; import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.broker.client.ClientChannelInfo; +import org.apache.rocketmq.broker.domain.LogicalQueuesInfoInBroker; import org.apache.rocketmq.broker.filter.ExpressionMessageFilter; import org.apache.rocketmq.broker.mqtrace.ConsumeMessageContext; import org.apache.rocketmq.broker.mqtrace.ConsumeMessageHook; import org.apache.rocketmq.common.BrokerConfig; import org.apache.rocketmq.common.TopicConfig; import org.apache.rocketmq.common.consumer.ConsumeFromWhere; +import org.apache.rocketmq.common.message.MessageConst; +import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.common.protocol.RequestCode; import org.apache.rocketmq.common.protocol.ResponseCode; import org.apache.rocketmq.common.protocol.header.PullMessageRequestHeader; +import org.apache.rocketmq.common.protocol.header.PullMessageResponseHeader; import org.apache.rocketmq.common.protocol.heartbeat.ConsumeType; import org.apache.rocketmq.common.protocol.heartbeat.ConsumerData; import org.apache.rocketmq.common.protocol.heartbeat.MessageModel; import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData; +import org.apache.rocketmq.common.protocol.route.LogicalQueueRouteData; +import org.apache.rocketmq.common.protocol.route.MessageQueueRouteState; import org.apache.rocketmq.remoting.exception.RemotingCommandException; import org.apache.rocketmq.remoting.netty.NettyClientConfig; import org.apache.rocketmq.remoting.netty.NettyServerConfig; @@ -46,6 +53,7 @@ import org.apache.rocketmq.store.GetMessageStatus; import org.apache.rocketmq.store.MessageStore; import org.apache.rocketmq.store.config.MessageStoreConfig; +import org.assertj.core.util.Lists; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -53,11 +61,15 @@ import org.mockito.Spy; import org.mockito.junit.MockitoJUnitRunner; +import static java.util.Optional.ofNullable; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.intThat; +import static org.mockito.Mockito.lenient; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -92,6 +104,7 @@ public void init() { consumerData.getConsumeFromWhere(), consumerData.getSubscriptionDataSet(), false); + brokerController.getTopicConfigManager().updateTopicConfig(new TopicConfig(topic)); } @Test @@ -192,6 +205,94 @@ public void testProcessRequest_NoMsgInQueue() throws RemotingCommandException { assertThat(response.getCode()).isEqualTo(ResponseCode.PULL_OFFSET_MOVED); } + @Test + public void testProcessRequest_LogicalQueue() throws Exception { + String brokerName = brokerController.getBrokerConfig().getBrokerName(); + int queueId = 1; + + GetMessageResult getMessageResult = createGetMessageResult(); + when(messageStore.getMessage(anyString(), eq(topic), eq(queueId), eq(456L), anyInt(), any(ExpressionMessageFilter.class))).thenReturn(getMessageResult); + when(messageStore.getMaxOffsetInQueue(eq(topic), eq(queueId))).thenReturn(2000L); + when(messageStore.getMinPhyOffset()).thenReturn(0L); + + LogicalQueuesInfoInBroker logicalQueuesInfo = brokerController.getTopicConfigManager().getOrCreateLogicalQueuesInfo(topic); + LogicalQueueRouteData queueRouteData1 = new LogicalQueueRouteData(0, 0, new MessageQueue(topic, brokerName, queueId), MessageQueueRouteState.Normal, 0, -1, -1, -1, brokerController.getBrokerAddr()); + logicalQueuesInfo.put(0, Lists.newArrayList(queueRouteData1)); + logicalQueuesInfo.updateQueueRouteDataByQueueId(queueRouteData1.getQueueId(), queueRouteData1); + + // normal + { + final RemotingCommand request = createPullMsgCommand(RequestCode.PULL_MESSAGE); + RemotingCommand response = pullMessageProcessor.processRequest(handlerContext, request); + assertThat(response).isNotNull(); + assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); + } + // write only + queueRouteData1.setState(MessageQueueRouteState.WriteOnly); + { + final RemotingCommand request = createPullMsgCommand(RequestCode.PULL_MESSAGE); + RemotingCommand response = pullMessageProcessor.processRequest(handlerContext, request); + assertThat(response).isNotNull(); + assertThat(response.getCode()).isEqualTo(ResponseCode.PULL_NOT_FOUND); + } + // no message and redirect + queueRouteData1.setState(MessageQueueRouteState.ReadOnly); + queueRouteData1.setOffsetMax(460); + queueRouteData1.setFirstMsgTimeMillis(100); + queueRouteData1.setLastMsgTimeMillis(200); + LogicalQueueRouteData queueRouteData2 = new LogicalQueueRouteData(0, 460, new MessageQueue(topic, "broker2", 1), MessageQueueRouteState.Normal, 0, -1, -1, -1, brokerController.getBrokerAddr()); + logicalQueuesInfo.get(0).add(queueRouteData2); + getMessageResult.setStatus(GetMessageStatus.OFFSET_FOUND_NULL); + when(messageStore.getCommitLogOffsetInQueue(eq(topic), eq(queueId), eq(460L - 1L))).thenReturn(1000L); + { + final RemotingCommand request = createPullMsgCommand(RequestCode.PULL_MESSAGE); + RemotingCommand response = pullMessageProcessor.processRequest(handlerContext, request); + assertThat(response).isNotNull(); + assertThat(response.getCode()).isEqualTo(ResponseCode.PULL_NOT_FOUND); + assertThat(response.getExtFields()).containsKey(MessageConst.PROPERTY_REDIRECT); + } + // same message queue has two routes + queueRouteData2.setState(MessageQueueRouteState.ReadOnly); + queueRouteData2.setOffsetMax(50); + queueRouteData2.setFirstMsgTimeMillis(300); + queueRouteData2.setLastMsgTimeMillis(400); + LogicalQueueRouteData queueRouteData3 = new LogicalQueueRouteData(0, 510, new MessageQueue(topic, queueRouteData2.getBrokerName(), queueId), MessageQueueRouteState.Normal, 460, -1, -1, -1, queueRouteData1.getBrokerAddr()); + logicalQueuesInfo.get(0).add(queueRouteData3); + logicalQueuesInfo.updateQueueRouteDataByQueueId(queueRouteData3.getQueueId(), queueRouteData3); + { + GetMessageResult getMessageResult2 = createGetMessageResult(); + getMessageResult2.setStatus(GetMessageStatus.FOUND); + getMessageResult2.setNextBeginOffset(460); + when(messageStore.getMessage(anyString(), eq(queueRouteData1.getTopic()), eq(queueRouteData1.getQueueId()), eq(456L), eq(4), any(ExpressionMessageFilter.class))).thenReturn(getMessageResult2); + } + { + GetMessageResult getMessageResult2 = createGetMessageResult(); + getMessageResult2.setStatus(GetMessageStatus.FOUND); + getMessageResult2.setNextBeginOffset(470); + lenient().when(messageStore.getMessage(anyString(), eq(queueRouteData1.getTopic()), eq(queueRouteData1.getQueueId()), eq(456L), intThat(i -> i > 4), any(ExpressionMessageFilter.class))).thenReturn(getMessageResult2); + } + { + final RemotingCommand request = createPullMsgCommand(RequestCode.PULL_MESSAGE); + RemotingCommand response = pullMessageProcessor.processRequest(handlerContext, request); + assertThat(response).isNotNull(); + assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); + assertThat(ofNullable(response.getExtFields()).orElse(new HashMap<>())).doesNotContainKey(MessageConst.PROPERTY_REDIRECT); + PullMessageResponseHeader header = (PullMessageResponseHeader) response.readCustomHeader(); + assertThat(header.getNextBeginOffset()).isEqualTo(460); + } + { + when(messageStore.getMinPhyOffset()).thenReturn(100000L); + + final RemotingCommand request = createPullMsgCommand(RequestCode.PULL_MESSAGE); + RemotingCommand response = pullMessageProcessor.processRequest(handlerContext, request); + assertThat(response).isNotNull(); + assertThat(response.getCode()).isEqualTo(ResponseCode.PULL_RETRY_IMMEDIATELY); + assertThat(ofNullable(response.getExtFields()).orElse(new HashMap<>())).containsKey(MessageConst.PROPERTY_REDIRECT); + PullMessageResponseHeader header = (PullMessageResponseHeader) response.readCustomHeader(); + assertThat(header.getNextBeginOffset()).isEqualTo(460); + } + } + private RemotingCommand createPullMsgCommand(int requestCode) { PullMessageRequestHeader requestHeader = new PullMessageRequestHeader(); requestHeader.setCommitOffset(123L); diff --git a/broker/src/test/java/org/apache/rocketmq/broker/processor/SendMessageProcessorTest.java b/broker/src/test/java/org/apache/rocketmq/broker/processor/SendMessageProcessorTest.java index b9344e90ed6..e768469f7aa 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/processor/SendMessageProcessorTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/processor/SendMessageProcessorTest.java @@ -18,18 +18,30 @@ import io.netty.channel.Channel; import io.netty.channel.ChannelHandlerContext; +import java.net.InetSocketAddress; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; import org.apache.rocketmq.broker.BrokerController; +import org.apache.rocketmq.broker.domain.LogicalQueuesInfoInBroker; import org.apache.rocketmq.broker.mqtrace.SendMessageContext; import org.apache.rocketmq.broker.mqtrace.SendMessageHook; import org.apache.rocketmq.broker.transaction.TransactionalMessageService; import org.apache.rocketmq.common.BrokerConfig; +import org.apache.rocketmq.common.TopicConfig; +import org.apache.rocketmq.common.constant.PermName; import org.apache.rocketmq.common.message.MessageConst; import org.apache.rocketmq.common.message.MessageDecoder; import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.common.protocol.RequestCode; import org.apache.rocketmq.common.protocol.ResponseCode; import org.apache.rocketmq.common.protocol.header.ConsumerSendMsgBackRequestHeader; import org.apache.rocketmq.common.protocol.header.SendMessageRequestHeader; +import org.apache.rocketmq.common.protocol.route.LogicalQueueRouteData; +import org.apache.rocketmq.common.protocol.route.MessageQueueRouteState; import org.apache.rocketmq.common.sysflag.MessageSysFlag; import org.apache.rocketmq.common.topic.TopicValidator; import org.apache.rocketmq.remoting.exception.RemotingCommandException; @@ -43,6 +55,7 @@ import org.apache.rocketmq.store.PutMessageResult; import org.apache.rocketmq.store.PutMessageStatus; import org.apache.rocketmq.store.config.MessageStoreConfig; +import org.assertj.core.util.Lists; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -52,12 +65,6 @@ import org.mockito.junit.MockitoJUnitRunner; import org.mockito.stubbing.Answer; -import java.net.InetSocketAddress; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.concurrent.CompletableFuture; - import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyLong; @@ -90,6 +97,8 @@ public void init() { when(handlerContext.channel()).thenReturn(mockChannel); when(messageStore.lookMessageByOffset(anyLong())).thenReturn(new MessageExt()); sendMessageProcessor = new SendMessageProcessor(brokerController); + + brokerController.getTopicConfigManager().updateTopicConfig(new TopicConfig(topic, 8, 8, PermName.PERM_WRITE|PermName.PERM_READ)); } @Test @@ -220,6 +229,62 @@ public Object answer(InvocationOnMock invocation) throws Throwable { assertThat(response[0].getCode()).isEqualTo(ResponseCode.SUCCESS); } + + @Test + public void testProcessRequest_LogicalQueue() throws Exception { + when(messageStore.asyncPutMessage(any(MessageExtBrokerInner.class))) + .thenReturn(CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK)))); + + LogicalQueuesInfoInBroker logicalQueuesInfo = brokerController.getTopicConfigManager().getOrCreateLogicalQueuesInfo(topic); + LogicalQueueRouteData queueRouteData1 = new LogicalQueueRouteData(0, 0, new MessageQueue(topic, brokerController.getBrokerConfig().getBrokerName(), 1), MessageQueueRouteState.Normal, 0, -1, -1, -1, brokerController.getBrokerAddr()); + logicalQueuesInfo.put(0, Lists.newArrayList(queueRouteData1)); + logicalQueuesInfo.updateQueueRouteDataByQueueId(queueRouteData1.getQueueId(), queueRouteData1); + + SendMessageRequestHeader requestHeader = createSendMsgRequestHeader(); + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.SEND_MESSAGE, requestHeader); + request.setBody(new byte[] {'a'}); + request.makeCustomHeaderToNet(); + + // normal + RemotingCommand responseToReturn; + { + CompletableFuture responseFuture = new CompletableFuture<>(); + doAnswer(invocation -> { + responseFuture.complete(invocation.getArgument(0)); + return null; + }).when(handlerContext).writeAndFlush(any(Object.class)); + + responseToReturn = sendMessageProcessor.processRequest(handlerContext, request); + if (responseToReturn == null) { + responseToReturn = responseFuture.get(3, TimeUnit.SECONDS); + } + } + + assertThat(responseToReturn.getCode()).isEqualTo(ResponseCode.SUCCESS); + assertThat(responseToReturn.getOpaque()).isEqualTo(request.getOpaque()); + + // read only + queueRouteData1.setState(MessageQueueRouteState.ReadOnly); + responseToReturn = sendMessageProcessor.processRequest(handlerContext, request); + assertThat(responseToReturn.getCode()).isEqualTo(ResponseCode.NO_PERMISSION); + assertThat(responseToReturn.getRemark()).contains("not writable"); + + // read only and forward + logicalQueuesInfo.get(0).add(new LogicalQueueRouteData(0, 100, new MessageQueue(topic, "broker2", 1), MessageQueueRouteState.Normal, 0, -1, -1, -1, brokerController.getBrokerAddr())); + + responseToReturn = sendMessageProcessor.processRequest(handlerContext, request); + assertThat(responseToReturn.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR); + assertThat(responseToReturn.getRemark()).contains("forward error"); + + // read only and redirect + requestHeader = (SendMessageRequestHeader) request.readCustomHeader(); + requestHeader.setSysFlag(MessageSysFlag.LOGICAL_QUEUE_FLAG); + request.makeCustomHeaderToNet(); + responseToReturn = sendMessageProcessor.processRequest(handlerContext, request); + assertThat(responseToReturn.getCode()).isEqualTo(ResponseCode.NO_PERMISSION); + assertThat(responseToReturn.getExtFields()).containsKey(MessageConst.PROPERTY_REDIRECT); + } + private RemotingCommand createSendTransactionMsgCommand(int requestCode) { SendMessageRequestHeader header = createSendMsgRequestHeader(); int sysFlag = header.getSysFlag(); diff --git a/broker/src/test/java/org/apache/rocketmq/broker/topic/TopicConfigManagerTest.java b/broker/src/test/java/org/apache/rocketmq/broker/topic/TopicConfigManagerTest.java new file mode 100644 index 00000000000..f504965fada --- /dev/null +++ b/broker/src/test/java/org/apache/rocketmq/broker/topic/TopicConfigManagerTest.java @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.broker.topic; + +import com.google.common.collect.Lists; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import org.apache.rocketmq.broker.BrokerController; +import org.apache.rocketmq.broker.domain.LogicalQueuesInfoInBroker; +import org.apache.rocketmq.common.BrokerConfig; +import org.apache.rocketmq.common.DataVersion; +import org.apache.rocketmq.common.TopicConfig; +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.common.protocol.route.LogicalQueueRouteData; +import org.apache.rocketmq.common.protocol.route.MessageQueueRouteState; +import org.apache.rocketmq.store.DefaultMessageStore; +import org.apache.rocketmq.store.config.MessageStoreConfig; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.ArgumentMatchers; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class TopicConfigManagerTest { + @Mock + private DefaultMessageStore messageStore; + @Mock + private BrokerController brokerController; + + private TopicConfigManager topicConfigManager; + + private static final String topic = "FooBar"; + private static final String broker1Name = "broker1"; + private static final String broker1Addr = "127.0.0.1:12345"; + private static final int queueId1 = 1; + private static final String broker2Name = "broker2"; + private static final String broker2Addr = "127.0.0.2:12345"; + private static final int queueId2 = 2; + + @Before + public void before() { + BrokerConfig brokerConfig = new BrokerConfig(); + brokerConfig.setBrokerName(broker1Name); + when(brokerController.getBrokerConfig()).thenReturn(brokerConfig); + + when(brokerController.getMessageStore()).thenReturn(messageStore); + + MessageStoreConfig messageStoreConfig = new MessageStoreConfig(); + messageStoreConfig.setStorePathRootDir(System.getProperty("java.io.tmpdir")); + when(brokerController.getMessageStoreConfig()).thenReturn(messageStoreConfig); + + this.topicConfigManager = new TopicConfigManager(brokerController); + this.topicConfigManager.getTopicConfigTable().put(topic, new TopicConfig(topic)); + } + + @After + public void after() throws Exception { + if (topicConfigManager != null) { + Files.deleteIfExists(Paths.get(topicConfigManager.configFilePath())); + } + } + + @Test + public void logicalQueueCleanTest() { + LogicalQueuesInfoInBroker info = this.topicConfigManager.getOrCreateLogicalQueuesInfo(topic); + topicConfigManager.logicalQueueClean(); + assertThat(info).isEmpty(); + + final int logicalQueueIndex = 0; + LogicalQueueRouteData queueRouteData1 = new LogicalQueueRouteData(logicalQueueIndex, 0, new MessageQueue(topic, broker1Name, queueId1), MessageQueueRouteState.Normal, 0, -1, -1, -1, broker1Addr); + List l = Lists.newArrayList(new LogicalQueueRouteData(queueRouteData1)); + info.put(logicalQueueIndex, l); + + topicConfigManager.logicalQueueClean(); + assertThat(info.get(logicalQueueIndex)).isEqualTo(Collections.singletonList(queueRouteData1)); + verify(messageStore, never()).getCommitLogOffsetInQueue(eq(topic), eq(queueId1), anyLong()); + verify(messageStore, never()).getMinPhyOffset(); + verify(brokerController, never()).registerIncrementBrokerData(ArgumentMatchers.argThat(arg -> topic.equals(arg.getTopicName())), any(DataVersion.class)); + + LogicalQueueRouteData queueRouteData2 = new LogicalQueueRouteData(logicalQueueIndex, 100, new MessageQueue(topic, broker2Name, queueId2), MessageQueueRouteState.Normal, 0, -1, -1, -1, broker2Addr); + l.add(new LogicalQueueRouteData(queueRouteData2)); + queueRouteData1 = l.get(0); + queueRouteData1.setState(MessageQueueRouteState.ReadOnly); + queueRouteData1.setOffsetMax(100); + queueRouteData1.setFirstMsgTimeMillis(200); + queueRouteData1.setLastMsgTimeMillis(300); + queueRouteData1 = new LogicalQueueRouteData(queueRouteData1); + LogicalQueueRouteData queueRouteData3 = new LogicalQueueRouteData(logicalQueueIndex, 200, new MessageQueue(topic, broker1Name, queueId1), MessageQueueRouteState.Normal, 100, -1, -1, -1, broker1Addr); + l.add(new LogicalQueueRouteData(queueRouteData3)); + queueRouteData2 = l.get(1); + queueRouteData2.setState(MessageQueueRouteState.ReadOnly); + queueRouteData2.setOffsetMax(100); + queueRouteData2.setFirstMsgTimeMillis(400); + queueRouteData2.setLastMsgTimeMillis(500); + queueRouteData2 = new LogicalQueueRouteData(queueRouteData2); + when(messageStore.getCommitLogOffsetInQueue(eq(topic), eq(queueId1), eq(queueRouteData1.getOffsetMax() - 1))).thenReturn(1000L); + when(messageStore.getMinPhyOffset()).thenReturn(0L); + topicConfigManager.logicalQueueClean(); + assertThat(info.get(logicalQueueIndex)).isEqualTo(Arrays.asList(queueRouteData1, queueRouteData2, queueRouteData3)); + verify(messageStore).getCommitLogOffsetInQueue(eq(topic), eq(queueId1), eq(queueRouteData1.getOffsetMax() - 1)); + verify(messageStore).getMinPhyOffset(); + verify(brokerController, never()).registerIncrementBrokerData(ArgumentMatchers.argThat(arg -> topic.equals(arg.getTopicName())), any(DataVersion.class)); + + when(messageStore.getMinPhyOffset()).thenReturn(2000L); + topicConfigManager.logicalQueueClean(); + assertThat(info.get(logicalQueueIndex)).isEqualTo(Collections.singletonList(queueRouteData3)); + verify(brokerController).registerIncrementBrokerData(ArgumentMatchers.argThat(arg -> topic.equals(arg.getTopicName())), any(DataVersion.class)); + } +} \ No newline at end of file diff --git a/srvutil/src/main/java/org/apache/rocketmq/srvutil/ConcurrentHashMapUtil.java b/srvutil/src/main/java/org/apache/rocketmq/srvutil/ConcurrentHashMapUtil.java new file mode 100644 index 00000000000..cc98eb5bcb5 --- /dev/null +++ b/srvutil/src/main/java/org/apache/rocketmq/srvutil/ConcurrentHashMapUtil.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.srvutil; + +import java.util.concurrent.ConcurrentMap; +import java.util.function.Function; + +public class ConcurrentHashMapUtil { + private static final boolean IS_JDK8; + + static { + // Java 8 or lower: 1.6.0_23, 1.7.0, 1.7.0_80, 1.8.0_211 + // Java 9 or higher: 9.0.1, 11.0.4, 12, 12.0.1 + IS_JDK8 = System.getProperty("java.version").startsWith("1.8."); + } + + private ConcurrentHashMapUtil() { + } + + /** + * A temporary workaround for Java 8 specific performance issue JDK-8161372 .
Use implementation of + * ConcurrentMap.computeIfAbsent instead. + * + * @see https://bugs.openjdk.java.net/browse/JDK-8161372 + */ + public static V computeIfAbsent(ConcurrentMap map, K key, Function func) { + if (IS_JDK8) { + V v, newValue; + return ((v = map.get(key)) == null && + (newValue = func.apply(key)) != null && + (v = map.putIfAbsent(key, newValue)) == null) ? newValue : v; + } else { + return map.computeIfAbsent(key, func); + } + } +} diff --git a/store/src/main/java/org/apache/rocketmq/store/CleanFilesHook.java b/store/src/main/java/org/apache/rocketmq/store/CleanFilesHook.java new file mode 100644 index 00000000000..ef6e22a8f88 --- /dev/null +++ b/store/src/main/java/org/apache/rocketmq/store/CleanFilesHook.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.store; + +public interface CleanFilesHook { + void execute(DefaultMessageStore defaultMessageStore, long deleteCount); + + String getName(); +} diff --git a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java index cce6481b8da..0492aa95e79 100644 --- a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java +++ b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java @@ -140,7 +140,17 @@ public int deleteExpiredFile( final long intervalForcibly, final boolean cleanImmediately ) { - return this.mappedFileQueue.deleteExpiredFileByTime(expiredTime, deleteFilesInterval, intervalForcibly, cleanImmediately); + return deleteExpiredFile(expiredTime, deleteFilesInterval, intervalForcibly, cleanImmediately, 0); + } + + public int deleteExpiredFile( + final long expiredTime, + final int deleteFilesInterval, + final long intervalForcibly, + final boolean cleanImmediately, + final int deleteFileBatchMax + ) { + return this.mappedFileQueue.deleteExpiredFileByTime(expiredTime, deleteFilesInterval, intervalForcibly, cleanImmediately, deleteFileBatchMax); } /** diff --git a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java index c25a1147328..b11eb49047c 100644 --- a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java @@ -28,12 +28,14 @@ import java.util.HashMap; import java.util.Iterator; import java.util.LinkedList; +import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; @@ -118,6 +120,8 @@ public class DefaultMessageStore implements MessageStore { private final ScheduledExecutorService diskCheckScheduledExecutorService = Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("DiskCheckScheduledThread")); + private final List cleanFilesHooks = new CopyOnWriteArrayList<>(); + public DefaultMessageStore(final MessageStoreConfig messageStoreConfig, final BrokerStatsManager brokerStatsManager, final MessageArrivingListener messageArrivingListener, final BrokerConfig brokerConfig) throws IOException { this.messageArrivingListener = messageArrivingListener; @@ -720,10 +724,20 @@ public GetMessageResult getMessage(final String group, final String topic, final } public long getMaxOffsetInQueue(String topic, int queueId) { - ConsumeQueue logic = this.findConsumeQueue(topic, queueId); - if (logic != null) { - long offset = logic.getMaxOffsetInQueue(); - return offset; + return getMaxOffsetInQueue(topic, queueId, true); + } + + public long getMaxOffsetInQueue(String topic, int queueId, boolean committed) { + if (committed) { + ConsumeQueue logic = this.findConsumeQueue(topic, queueId); + if (logic != null) { + return logic.getMaxOffsetInQueue(); + } + } else { + Long offset = this.commitLog.getTopicQueueTable().get(topic + "-" + queueId); + if (offset != null) { + return offset; + } } return 0; @@ -1301,12 +1315,23 @@ private void createTempFile() throws IOException { log.info(fileName + (result ? " create OK" : " already exists")); } + public void registerCleanFileHook(CleanFilesHook hook) { + this.cleanFilesHooks.add(hook); + } + private void addScheduleTask() { this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { @Override public void run() { - DefaultMessageStore.this.cleanFilesPeriodically(); + long deleteCount = DefaultMessageStore.this.cleanFilesPeriodically(); + DefaultMessageStore.this.cleanFilesHooks.forEach(hook -> { + try { + hook.execute(DefaultMessageStore.this, deleteCount); + } catch (Throwable t) { + log.error("execute CleanFilesHook[{}] error", hook.getName(), t); + } + }); } }, 1000 * 60, this.messageStoreConfig.getCleanResourceInterval(), TimeUnit.MILLISECONDS); @@ -1351,9 +1376,11 @@ public void run() { }, 1000L, 10000L, TimeUnit.MILLISECONDS); } - private void cleanFilesPeriodically() { - this.cleanCommitLogService.run(); - this.cleanConsumeQueueService.run(); + private long cleanFilesPeriodically() { + long deleteCount = 0L; + deleteCount += this.cleanCommitLogService.run(); + deleteCount += this.cleanConsumeQueueService.run(); + return deleteCount; } private void checkSelf() { @@ -1611,17 +1638,19 @@ public void excuteDeleteFilesManualy() { DefaultMessageStore.log.info("executeDeleteFilesManually was invoked"); } - public void run() { + public long run() { + int deleteCount = 0; try { - this.deleteExpiredFiles(); + deleteCount = this.deleteExpiredFiles(); this.redeleteHangedFile(); } catch (Throwable e) { DefaultMessageStore.log.warn(this.getServiceName() + " service has exception. ", e); } + return deleteCount; } - private void deleteExpiredFiles() { + private int deleteExpiredFiles() { int deleteCount = 0; long fileReservedTime = DefaultMessageStore.this.getMessageStoreConfig().getFileReservedTime(); int deletePhysicFilesInterval = DefaultMessageStore.this.getMessageStoreConfig().getDeleteCommitLogFilesInterval(); @@ -1654,6 +1683,7 @@ private void deleteExpiredFiles() { log.warn("disk space will be full soon, but delete file failed."); } } + return deleteCount; } private void redeleteHangedFile() { @@ -1775,17 +1805,20 @@ public boolean isSpaceFull() { class CleanConsumeQueueService { private long lastPhysicalMinOffset = 0; - public void run() { + public long run() { + long deleteCount = 0; try { - this.deleteExpiredFiles(); + deleteCount = this.deleteExpiredFiles(); } catch (Throwable e) { DefaultMessageStore.log.warn(this.getServiceName() + " service has exception. ", e); } + return deleteCount; } - private void deleteExpiredFiles() { + private long deleteExpiredFiles() { int deleteLogicsFilesInterval = DefaultMessageStore.this.getMessageStoreConfig().getDeleteConsumeQueueFilesInterval(); + long deleteCountSum = 0L; long minOffset = DefaultMessageStore.this.commitLog.getMinOffset(); if (minOffset > this.lastPhysicalMinOffset) { this.lastPhysicalMinOffset = minOffset; @@ -1795,7 +1828,7 @@ private void deleteExpiredFiles() { for (ConcurrentMap maps : tables.values()) { for (ConsumeQueue logic : maps.values()) { int deleteCount = logic.deleteExpiredFile(minOffset); - + deleteCountSum += deleteCount; if (deleteCount > 0 && deleteLogicsFilesInterval > 0) { try { Thread.sleep(deleteLogicsFilesInterval); @@ -1807,6 +1840,7 @@ private void deleteExpiredFiles() { DefaultMessageStore.this.indexService.deleteExpiredFile(minOffset); } + return deleteCountSum; } public String getServiceName() { diff --git a/store/src/main/java/org/apache/rocketmq/store/MappedFileQueue.java b/store/src/main/java/org/apache/rocketmq/store/MappedFileQueue.java index cc145921cef..ac7894d2dc2 100644 --- a/store/src/main/java/org/apache/rocketmq/store/MappedFileQueue.java +++ b/store/src/main/java/org/apache/rocketmq/store/MappedFileQueue.java @@ -56,8 +56,8 @@ public MappedFileQueue(final String storePath, int mappedFileSize, } public void checkSelf() { - - if (!this.mappedFiles.isEmpty()) { + List mappedFiles = new ArrayList<>(this.mappedFiles); + if (!mappedFiles.isEmpty()) { Iterator iterator = mappedFiles.iterator(); MappedFile pre = null; while (iterator.hasNext()) { @@ -238,21 +238,8 @@ public MappedFile getLastMappedFile(final long startOffset) { } public MappedFile getLastMappedFile() { - MappedFile mappedFileLast = null; - - while (!this.mappedFiles.isEmpty()) { - try { - mappedFileLast = this.mappedFiles.get(this.mappedFiles.size() - 1); - break; - } catch (IndexOutOfBoundsException e) { - //continue; - } catch (Exception e) { - log.error("getLastMappedFile has exception.", e); - break; - } - } - - return mappedFileLast; + MappedFile[] mappedFiles = this.mappedFiles.toArray(new MappedFile[0]); + return mappedFiles.length == 0 ? null : mappedFiles[mappedFiles.length - 1]; } public boolean resetOffset(long offset) { @@ -336,7 +323,11 @@ public void deleteLastMappedFile() { public int deleteExpiredFileByTime(final long expiredTime, final int deleteFilesInterval, final long intervalForcibly, - final boolean cleanImmediately) { + final boolean cleanImmediately, + int deleteFileBatchMax) { + if (deleteFileBatchMax == 0) { + deleteFileBatchMax = DELETE_FILES_BATCH_MAX; + } Object[] mfs = this.copyMappedFiles(0); if (null == mfs) @@ -354,7 +345,7 @@ public int deleteExpiredFileByTime(final long expiredTime, files.add(mappedFile); deleteCount++; - if (files.size() >= DELETE_FILES_BATCH_MAX) { + if (files.size() >= deleteFileBatchMax) { break; } diff --git a/store/src/main/java/org/apache/rocketmq/store/MessageStore.java b/store/src/main/java/org/apache/rocketmq/store/MessageStore.java index 0cea607677d..74fa4f45286 100644 --- a/store/src/main/java/org/apache/rocketmq/store/MessageStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/MessageStore.java @@ -115,6 +115,16 @@ GetMessageResult getMessage(final String group, final String topic, final int qu */ long getMaxOffsetInQueue(final String topic, final int queueId); + /** + * Get maximum offset of the topic queue. + * + * @param topic Topic name. + * @param queueId Queue ID. + * @param committed If only count committed + * @return Maximum offset at present. + */ + long getMaxOffsetInQueue(final String topic, final int queueId, final boolean committed); + /** * Get the minimum offset of the topic queue. * diff --git a/store/src/test/java/org/apache/rocketmq/store/MappedFileQueueTest.java b/store/src/test/java/org/apache/rocketmq/store/MappedFileQueueTest.java index 8f76051d1f8..0a736f92799 100644 --- a/store/src/test/java/org/apache/rocketmq/store/MappedFileQueueTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/MappedFileQueueTest.java @@ -225,7 +225,7 @@ public void testDeleteExpiredFileByTime() throws Exception { mappedFile.getFile().setLastModified(System.currentTimeMillis() - expiredTime * 2); } } - mappedFileQueue.deleteExpiredFileByTime(expiredTime, 0, 0, false); + mappedFileQueue.deleteExpiredFileByTime(expiredTime, 0, 0, false, Integer.MAX_VALUE); assertThat(mappedFileQueue.getMappedFiles().size()).isEqualTo(45); } From acbc0be04bf0d3552a891cab3cd3198628582ce3 Mon Sep 17 00:00:00 2001 From: chenzlalvin Date: Wed, 23 Jun 2021 17:32:09 +0800 Subject: [PATCH 3/5] [RIP-21] submodule namesrv --- namesrv/pom.xml | 5 + .../ClusterTestRequestProcessor.java | 2 +- .../processor/DefaultRequestProcessor.java | 23 +++- .../namesrv/routeinfo/RouteInfoManager.java | 67 ++++++++++- .../DefaultRequestProcessorTest.java | 109 +++++++++++++++++- 5 files changed, 194 insertions(+), 12 deletions(-) diff --git a/namesrv/pom.xml b/namesrv/pom.xml index 01ef5b60fe9..42da143f03a 100644 --- a/namesrv/pom.xml +++ b/namesrv/pom.xml @@ -48,5 +48,10 @@ org.slf4j slf4j-api + + com.google.guava + guava + test + diff --git a/namesrv/src/main/java/org/apache/rocketmq/namesrv/processor/ClusterTestRequestProcessor.java b/namesrv/src/main/java/org/apache/rocketmq/namesrv/processor/ClusterTestRequestProcessor.java index a58a3b97fc2..dd152888aa7 100644 --- a/namesrv/src/main/java/org/apache/rocketmq/namesrv/processor/ClusterTestRequestProcessor.java +++ b/namesrv/src/main/java/org/apache/rocketmq/namesrv/processor/ClusterTestRequestProcessor.java @@ -56,7 +56,7 @@ public RemotingCommand getRouteInfoByTopic(ChannelHandlerContext ctx, final GetRouteInfoRequestHeader requestHeader = (GetRouteInfoRequestHeader) request.decodeCommandCustomHeader(GetRouteInfoRequestHeader.class); - TopicRouteData topicRouteData = this.namesrvController.getRouteInfoManager().pickupTopicRouteData(requestHeader.getTopic()); + TopicRouteData topicRouteData = this.namesrvController.getRouteInfoManager().pickupTopicRouteData(requestHeader.getTopic(), false); if (topicRouteData != null) { String orderTopicConf = this.namesrvController.getKvConfigManager().getKVConfig(NamesrvUtil.NAMESPACE_ORDER_TOPIC_CONFIG, diff --git a/namesrv/src/main/java/org/apache/rocketmq/namesrv/processor/DefaultRequestProcessor.java b/namesrv/src/main/java/org/apache/rocketmq/namesrv/processor/DefaultRequestProcessor.java index f8bc55e7aab..11cfcd2dcbe 100644 --- a/namesrv/src/main/java/org/apache/rocketmq/namesrv/processor/DefaultRequestProcessor.java +++ b/namesrv/src/main/java/org/apache/rocketmq/namesrv/processor/DefaultRequestProcessor.java @@ -19,6 +19,7 @@ import io.netty.channel.ChannelHandlerContext; import java.io.UnsupportedEncodingException; import java.util.Properties; +import java.util.Set; import java.util.concurrent.atomic.AtomicLong; import org.apache.rocketmq.common.DataVersion; import org.apache.rocketmq.common.MQVersion; @@ -27,8 +28,6 @@ import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.help.FAQUrl; -import org.apache.rocketmq.logging.InternalLogger; -import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.common.namesrv.NamesrvUtil; import org.apache.rocketmq.common.namesrv.RegisterBrokerResult; import org.apache.rocketmq.common.protocol.RequestCode; @@ -50,7 +49,11 @@ import org.apache.rocketmq.common.protocol.header.namesrv.UnRegisterBrokerRequestHeader; import org.apache.rocketmq.common.protocol.header.namesrv.WipeWritePermOfBrokerRequestHeader; import org.apache.rocketmq.common.protocol.header.namesrv.WipeWritePermOfBrokerResponseHeader; -import org.apache.rocketmq.common.protocol.route.TopicRouteData; +import org.apache.rocketmq.common.protocol.route.LogicalQueuesInfoUnordered; +import org.apache.rocketmq.common.protocol.route.TopicRouteDataNameSrv; +import org.apache.rocketmq.common.sysflag.MessageSysFlag; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.namesrv.NamesrvController; import org.apache.rocketmq.remoting.common.RemotingHelper; import org.apache.rocketmq.remoting.exception.RemotingCommandException; @@ -346,7 +349,9 @@ public RemotingCommand getRouteInfoByTopic(ChannelHandlerContext ctx, final GetRouteInfoRequestHeader requestHeader = (GetRouteInfoRequestHeader) request.decodeCommandCustomHeader(GetRouteInfoRequestHeader.class); - TopicRouteData topicRouteData = this.namesrvController.getRouteInfoManager().pickupTopicRouteData(requestHeader.getTopic()); + boolean includeLogicalQueuesInfo = (requestHeader.getSysFlag() & MessageSysFlag.LOGICAL_QUEUE_FLAG) > 0; + + TopicRouteDataNameSrv topicRouteData = this.namesrvController.getRouteInfoManager().pickupTopicRouteData(requestHeader.getTopic(), includeLogicalQueuesInfo); if (topicRouteData != null) { if (this.namesrvController.getNamesrvConfig().isOrderMessageEnable()) { @@ -356,6 +361,16 @@ public RemotingCommand getRouteInfoByTopic(ChannelHandlerContext ctx, topicRouteData.setOrderTopicConf(orderTopicConf); } + Set logicalQueueIdsFilter = requestHeader.getLogicalQueueIdsFilter(); + if (logicalQueueIdsFilter != null) { + LogicalQueuesInfoUnordered logicalQueuesInfo = topicRouteData.getLogicalQueuesInfoUnordered(); + if (logicalQueuesInfo != null) { + LogicalQueuesInfoUnordered filtered = new LogicalQueuesInfoUnordered(logicalQueueIdsFilter.size()); + logicalQueueIdsFilter.forEach(integer -> filtered.put(integer, logicalQueuesInfo.get(integer))); + topicRouteData.setLogicalQueuesInfoUnordered(filtered); + } + } + byte[] content = topicRouteData.encode(); response.setBody(content); response.setCode(ResponseCode.SUCCESS); diff --git a/namesrv/src/main/java/org/apache/rocketmq/namesrv/routeinfo/RouteInfoManager.java b/namesrv/src/main/java/org/apache/rocketmq/namesrv/routeinfo/RouteInfoManager.java index edef87ce2d7..476e92bc338 100644 --- a/namesrv/src/main/java/org/apache/rocketmq/namesrv/routeinfo/RouteInfoManager.java +++ b/namesrv/src/main/java/org/apache/rocketmq/namesrv/routeinfo/RouteInfoManager.java @@ -17,6 +17,7 @@ package org.apache.rocketmq.namesrv.routeinfo; import io.netty.channel.Channel; +import java.util.Collection; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -24,26 +25,32 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.Objects; import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.stream.Collectors; import org.apache.rocketmq.common.DataVersion; import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.TopicConfig; import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.constant.PermName; -import org.apache.rocketmq.logging.InternalLogger; -import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.common.namesrv.RegisterBrokerResult; import org.apache.rocketmq.common.protocol.body.ClusterInfo; import org.apache.rocketmq.common.protocol.body.TopicConfigSerializeWrapper; import org.apache.rocketmq.common.protocol.body.TopicList; import org.apache.rocketmq.common.protocol.route.BrokerData; +import org.apache.rocketmq.common.protocol.route.LogicalQueuesInfo; +import org.apache.rocketmq.common.protocol.route.LogicalQueuesInfoUnordered; import org.apache.rocketmq.common.protocol.route.QueueData; -import org.apache.rocketmq.common.protocol.route.TopicRouteData; +import org.apache.rocketmq.common.protocol.route.TopicRouteDataNameSrv; import org.apache.rocketmq.common.sysflag.TopicSysFlag; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.remoting.common.RemotingUtil; +import org.apache.rocketmq.srvutil.ConcurrentHashMapUtil; public class RouteInfoManager { private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.NAMESRV_LOGGER_NAME); @@ -54,6 +61,7 @@ public class RouteInfoManager { private final HashMap> clusterAddrTable; private final HashMap brokerLiveTable; private final HashMap/* Filter Server */> filterServerTable; + private final ConcurrentMap logicalQueuesInfoTable; public RouteInfoManager() { this.topicQueueTable = new HashMap>(1024); @@ -61,6 +69,7 @@ public RouteInfoManager() { this.clusterAddrTable = new HashMap>(32); this.brokerLiveTable = new HashMap(256); this.filterServerTable = new HashMap>(256); + this.logicalQueuesInfoTable = new ConcurrentHashMap<>(1024); } public byte[] getAllClusterInfo() { @@ -148,18 +157,28 @@ public RegisterBrokerResult registerBroker( || registerFirst) { ConcurrentMap tcTable = topicConfigWrapper.getTopicConfigTable(); + Map logicalQueuesInfoMap = topicConfigWrapper.getLogicalQueuesInfoMap(); if (tcTable != null) { for (Map.Entry entry : tcTable.entrySet()) { this.createAndUpdateQueueData(brokerName, entry.getValue()); } } + if (logicalQueuesInfoMap != null) { + long startTime = System.nanoTime(); + for (Map.Entry entry : logicalQueuesInfoMap.entrySet()) { + String topicName = entry.getKey(); + LogicalQueuesInfoUnordered logicalQueuesInfo = ConcurrentHashMapUtil.computeIfAbsent(this.logicalQueuesInfoTable, topicName, ignore -> new LogicalQueuesInfoUnordered()); + mergeLogicalQueuesInfo(brokerName, topicName, logicalQueuesInfo, entry.getValue()); + } + log.debug("mergeQueueRouteDataTable topic={} time={}ns", System.nanoTime() - startTime); + } } } BrokerLiveInfo prevBrokerLiveInfo = this.brokerLiveTable.put(brokerAddr, new BrokerLiveInfo( System.currentTimeMillis(), - topicConfigWrapper.getDataVersion(), + topicConfigWrapper != null ? topicConfigWrapper.getDataVersion() : new DataVersion(), channel, haServerAddr)); if (null == prevBrokerLiveInfo) { @@ -371,8 +390,12 @@ private void removeTopicByBrokerName(final String brokerName) { } } - public TopicRouteData pickupTopicRouteData(final String topic) { - TopicRouteData topicRouteData = new TopicRouteData(); + public TopicRouteDataNameSrv pickupTopicRouteData(final String topic) { + return pickupTopicRouteData(topic, false); + } + + public TopicRouteDataNameSrv pickupTopicRouteData(final String topic, boolean includeLogicalQueuesInfo) { + TopicRouteDataNameSrv topicRouteData = new TopicRouteDataNameSrv(); boolean foundQueueData = false; boolean foundBrokerData = false; Set brokerNameSet = new HashSet(); @@ -420,6 +443,10 @@ public TopicRouteData pickupTopicRouteData(final String topic) { log.debug("pickupTopicRouteData {} {}", topic, topicRouteData); if (foundBrokerData && foundQueueData) { + if (includeLogicalQueuesInfo) { + topicRouteData.setLogicalQueuesInfoUnordered(logicalQueuesInfoTable.get(topic)); + } + return topicRouteData; } @@ -750,6 +777,34 @@ public byte[] getHasUnitSubUnUnitTopicList() { return topicList.encode(); } + + private static void mergeLogicalQueuesInfo(String brokerName, String topicName, + LogicalQueuesInfoUnordered logicalQueuesInfoInNamesrv, + LogicalQueuesInfo logicalQueuesInfoFromBroker) { + Set newKeys = logicalQueuesInfoFromBroker.values() + .stream() + .flatMap(Collection::stream) + .filter(v -> Objects.equals(brokerName, v.getBrokerName())) + .map(v -> new LogicalQueuesInfoUnordered.Key(null, v.getQueueId(), v.getOffsetDelta())) + .collect(Collectors.toSet()); + logicalQueuesInfoInNamesrv.values().forEach(m -> + m.values().removeIf(queueRouteData -> + Objects.equals(brokerName, queueRouteData.getBrokerName()) && + !newKeys.contains(new LogicalQueuesInfoUnordered.Key(null, queueRouteData.getQueueId(), queueRouteData.getOffsetDelta())))); + logicalQueuesInfoFromBroker.forEach((logicalQueueId, queueRouteDataListFromBroker) -> { + if (logicalQueueId == null) { + log.warn("queueRouteDataTable topic {} contains null logicalQueueId: {}", topicName, logicalQueuesInfoFromBroker); + return; + } + queueRouteDataListFromBroker.stream() + .filter(queueRouteDataFromBroker -> Objects.equals(brokerName, queueRouteDataFromBroker.getBrokerName())) + .forEach(queueRouteDataFromBroker -> + ConcurrentHashMapUtil.computeIfAbsent(logicalQueuesInfoInNamesrv, logicalQueueId, ignored -> new ConcurrentHashMap<>(queueRouteDataListFromBroker.size())) + .put(new LogicalQueuesInfoUnordered.Key(brokerName, queueRouteDataFromBroker.getQueueId(), queueRouteDataFromBroker.getOffsetDelta()), + queueRouteDataFromBroker) + ); + }); + } } class BrokerLiveInfo { diff --git a/namesrv/src/test/java/org/apache/rocketmq/namesrv/processor/DefaultRequestProcessorTest.java b/namesrv/src/test/java/org/apache/rocketmq/namesrv/processor/DefaultRequestProcessorTest.java index d4a2f66f99f..ff477a3e012 100644 --- a/namesrv/src/test/java/org/apache/rocketmq/namesrv/processor/DefaultRequestProcessorTest.java +++ b/namesrv/src/test/java/org/apache/rocketmq/namesrv/processor/DefaultRequestProcessorTest.java @@ -16,16 +16,22 @@ */ package org.apache.rocketmq.namesrv.processor; +import com.alibaba.fastjson.JSON; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; import io.netty.channel.Channel; import io.netty.channel.ChannelHandlerContext; import java.lang.reflect.Field; import java.lang.reflect.Modifier; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import org.apache.rocketmq.common.DataVersion; +import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.TopicConfig; -import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.common.namesrv.NamesrvConfig; import org.apache.rocketmq.common.namesrv.RegisterBrokerResult; import org.apache.rocketmq.common.protocol.RequestCode; @@ -34,14 +40,23 @@ import org.apache.rocketmq.common.protocol.header.namesrv.DeleteKVConfigRequestHeader; import org.apache.rocketmq.common.protocol.header.namesrv.GetKVConfigRequestHeader; import org.apache.rocketmq.common.protocol.header.namesrv.GetKVConfigResponseHeader; +import org.apache.rocketmq.common.protocol.header.namesrv.GetRouteInfoRequestHeader; import org.apache.rocketmq.common.protocol.header.namesrv.PutKVConfigRequestHeader; import org.apache.rocketmq.common.protocol.header.namesrv.RegisterBrokerRequestHeader; import org.apache.rocketmq.common.protocol.route.BrokerData; +import org.apache.rocketmq.common.protocol.route.LogicalQueueRouteData; +import org.apache.rocketmq.common.protocol.route.LogicalQueuesInfo; +import org.apache.rocketmq.common.protocol.route.LogicalQueuesInfoUnordered; +import org.apache.rocketmq.common.protocol.route.MessageQueueRouteState; +import org.apache.rocketmq.common.protocol.route.TopicRouteDataNameSrv; +import org.apache.rocketmq.common.sysflag.MessageSysFlag; +import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.namesrv.NamesrvController; import org.apache.rocketmq.namesrv.routeinfo.RouteInfoManager; import org.apache.rocketmq.remoting.exception.RemotingCommandException; import org.apache.rocketmq.remoting.netty.NettyServerConfig; import org.apache.rocketmq.remoting.protocol.RemotingCommand; +import org.apache.rocketmq.remoting.protocol.RemotingSerializable; import org.assertj.core.util.Maps; import org.junit.Before; import org.junit.Test; @@ -184,6 +199,98 @@ public void testProcessRequest_RegisterBroker() throws RemotingCommandException, .contains(new HashMap.SimpleEntry("broker", broker)); } + @Test + public void testProcessRequest_RegisterBrokerLogicalQueue() throws Exception { + String cluster = "cluster"; + String broker1Name = "broker1"; + String broker1Addr = "10.10.1.1"; + String broker2Name = "broker2"; + String broker2Addr = "10.10.1.2"; + String topic = "foobar"; + + LogicalQueueRouteData queueRouteData1 = new LogicalQueueRouteData(0, 0, new MessageQueue(topic, broker1Name, 0), MessageQueueRouteState.ReadOnly, 0, 10, 100, 100, broker1Addr); + { + RegisterBrokerRequestHeader header = new RegisterBrokerRequestHeader(); + header.setBrokerName(broker1Name); + RemotingCommand request = RemotingCommand.createRequestCommand( + RequestCode.REGISTER_BROKER, header); + request.addExtField("brokerName", broker1Name); + request.addExtField("brokerAddr", broker1Addr); + request.addExtField("clusterName", cluster); + request.addExtField("haServerAddr", "10.10.2.1"); + request.addExtField("brokerId", String.valueOf(MixAll.MASTER_ID)); + TopicConfigSerializeWrapper topicConfigSerializeWrapper = new TopicConfigSerializeWrapper(); + topicConfigSerializeWrapper.setTopicConfigTable(new ConcurrentHashMap<>(Collections.singletonMap(topic, new TopicConfig(topic)))); + topicConfigSerializeWrapper.setLogicalQueuesInfoMap(Maps.newHashMap(topic, new LogicalQueuesInfo(Collections.singletonMap(0, Lists.newArrayList( + queueRouteData1 + ))))); + topicConfigSerializeWrapper.setDataVersion(new DataVersion()); + request.setBody(RemotingSerializable.encode(topicConfigSerializeWrapper)); + + ChannelHandlerContext ctx = mock(ChannelHandlerContext.class); + when(ctx.channel()).thenReturn(null); + + RemotingCommand response = defaultRequestProcessor.processRequest(ctx, request); + + assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); + assertThat(response.getRemark()).isNull(); + } + LogicalQueueRouteData queueRouteData2 = new LogicalQueueRouteData(0, 100, new MessageQueue(topic, broker2Name, 0), MessageQueueRouteState.Normal, 0, -1, -1, -1, broker2Addr); + LogicalQueueRouteData queueRouteData3 = new LogicalQueueRouteData(1, 100, new MessageQueue(topic, broker2Name, 0), MessageQueueRouteState.Normal, 0, -1, -1, -1, broker2Addr); + { + RegisterBrokerRequestHeader header = new RegisterBrokerRequestHeader(); + header.setBrokerName(broker2Name); + RemotingCommand request = RemotingCommand.createRequestCommand( + RequestCode.REGISTER_BROKER, header); + request.addExtField("brokerName", broker2Name); + request.addExtField("brokerAddr", broker2Addr); + request.addExtField("clusterName", cluster); + request.addExtField("haServerAddr", "10.10.2.1"); + request.addExtField("brokerId", String.valueOf(MixAll.MASTER_ID)); + TopicConfigSerializeWrapper topicConfigSerializeWrapper = new TopicConfigSerializeWrapper(); + topicConfigSerializeWrapper.setTopicConfigTable(new ConcurrentHashMap<>(Collections.singletonMap(topic, new TopicConfig(topic)))); + topicConfigSerializeWrapper.setLogicalQueuesInfoMap(Maps.newHashMap(topic, new LogicalQueuesInfo(ImmutableMap.of( + 0, Collections.singletonList(queueRouteData2), + 1, Collections.singletonList(queueRouteData3) + )))); + topicConfigSerializeWrapper.setDataVersion(new DataVersion()); + request.setBody(RemotingSerializable.encode(topicConfigSerializeWrapper)); + + ChannelHandlerContext ctx = mock(ChannelHandlerContext.class); + when(ctx.channel()).thenReturn(null); + + RemotingCommand response = defaultRequestProcessor.processRequest(ctx, request); + + assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); + assertThat(response.getRemark()).isNull(); + } + + { + GetRouteInfoRequestHeader header = new GetRouteInfoRequestHeader(); + header.setTopic(topic); + header.setSysFlag(MessageSysFlag.LOGICAL_QUEUE_FLAG); + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_ROUTEINFO_BY_TOPIC, header); + request.makeCustomHeaderToNet(); + + ChannelHandlerContext ctx = mock(ChannelHandlerContext.class); + when(ctx.channel()).thenReturn(null); + + RemotingCommand response = defaultRequestProcessor.processRequest(ctx, request); + + assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); + + TopicRouteDataNameSrv topicRouteDataNameSrv = JSON.parseObject(response.getBody(), TopicRouteDataNameSrv.class); + assertThat(topicRouteDataNameSrv).isNotNull(); + LogicalQueuesInfoUnordered logicalQueuesInfoUnordered = new LogicalQueuesInfoUnordered(); + logicalQueuesInfoUnordered.put(0, ImmutableMap.of( + new LogicalQueuesInfoUnordered.Key(queueRouteData1.getBrokerName(), queueRouteData1.getQueueId(), queueRouteData1.getOffsetDelta()), queueRouteData1, + new LogicalQueuesInfoUnordered.Key(queueRouteData2.getBrokerName(), queueRouteData2.getQueueId(), queueRouteData2.getOffsetDelta()), queueRouteData2 + )); + logicalQueuesInfoUnordered.put(1, ImmutableMap.of(new LogicalQueuesInfoUnordered.Key(queueRouteData3.getBrokerName(), queueRouteData3.getQueueId(), queueRouteData3.getOffsetDelta()), queueRouteData3)); + assertThat(topicRouteDataNameSrv.getLogicalQueuesInfoUnordered()).isEqualTo(logicalQueuesInfoUnordered); + } + } + @Test public void testProcessRequest_RegisterBrokerWithFilterServer() throws RemotingCommandException, NoSuchFieldException, IllegalAccessException { From f3990857499e71c986f08c9193eb9be90855eb12 Mon Sep 17 00:00:00 2001 From: chenzlalvin Date: Wed, 23 Jun 2021 17:32:18 +0800 Subject: [PATCH 4/5] [RIP-21] submodule tools --- distribution/conf/logback_tools.xml | 5 + tools/pom.xml | 4 + .../tools/admin/DefaultMQAdminExt.java | 55 +++- .../tools/admin/DefaultMQAdminExtImpl.java | 60 +++- .../rocketmq/tools/admin/MQAdminExt.java | 31 +- .../tools/command/MQAdminStartup.java | 11 + .../tools/command/SubCommandException.java | 4 + ...DeleteTopicLogicalQueueMappingCommand.java | 91 ++++++ .../MigrateTopicLogicalQueueCommand.java | 210 +++++++++++++ .../QueryTopicLogicalQueueMappingCommand.java | 123 ++++++++ ...UpdateTopicLogicalQueueMappingCommand.java | 159 ++++++++++ .../UpdateTopicLogicalQueueNumCommand.java | 285 ++++++++++++++++++ .../command/topic/UpdateTopicSubCommand.java | 20 ++ .../tools/admin/DefaultMQAdminExtTest.java | 75 ++++- 14 files changed, 1117 insertions(+), 16 deletions(-) create mode 100644 tools/src/main/java/org/apache/rocketmq/tools/command/logicalqueue/DeleteTopicLogicalQueueMappingCommand.java create mode 100644 tools/src/main/java/org/apache/rocketmq/tools/command/logicalqueue/MigrateTopicLogicalQueueCommand.java create mode 100644 tools/src/main/java/org/apache/rocketmq/tools/command/logicalqueue/QueryTopicLogicalQueueMappingCommand.java create mode 100644 tools/src/main/java/org/apache/rocketmq/tools/command/logicalqueue/UpdateTopicLogicalQueueMappingCommand.java create mode 100644 tools/src/main/java/org/apache/rocketmq/tools/command/logicalqueue/UpdateTopicLogicalQueueNumCommand.java diff --git a/distribution/conf/logback_tools.xml b/distribution/conf/logback_tools.xml index 28283ad1d1d..d4bd5c27714 100644 --- a/distribution/conf/logback_tools.xml +++ b/distribution/conf/logback_tools.xml @@ -66,6 +66,11 @@ + + + + + diff --git a/tools/pom.xml b/tools/pom.xml index abe8197de39..096d531fa75 100644 --- a/tools/pom.xml +++ b/tools/pom.xml @@ -64,5 +64,9 @@ org.yaml snakeyaml + + com.google.guava + guava + diff --git a/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExt.java b/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExt.java index c27c85c66de..1c20324d1fa 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExt.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExt.java @@ -43,12 +43,16 @@ import org.apache.rocketmq.common.protocol.body.ConsumerRunningInfo; import org.apache.rocketmq.common.protocol.body.GroupList; import org.apache.rocketmq.common.protocol.body.KVTable; +import org.apache.rocketmq.common.protocol.body.MigrateLogicalQueueBody; import org.apache.rocketmq.common.protocol.body.ProducerConnection; import org.apache.rocketmq.common.protocol.body.QueryConsumeQueueResponseBody; import org.apache.rocketmq.common.protocol.body.QueueTimeSpan; import org.apache.rocketmq.common.protocol.body.SubscriptionGroupWrapper; import org.apache.rocketmq.common.protocol.body.TopicConfigSerializeWrapper; import org.apache.rocketmq.common.protocol.body.TopicList; +import org.apache.rocketmq.common.protocol.route.LogicalQueueRouteData; +import org.apache.rocketmq.common.protocol.route.LogicalQueuesInfo; +import org.apache.rocketmq.common.protocol.route.MessageQueueRouteState; import org.apache.rocketmq.common.protocol.route.TopicRouteData; import org.apache.rocketmq.common.subscription.SubscriptionGroupConfig; import org.apache.rocketmq.common.topic.TopicValidator; @@ -209,7 +213,8 @@ public SubscriptionGroupConfig examineSubscriptionGroupConfig(String addr, Strin } @Override - public TopicConfig examineTopicConfig(String addr, String topic) { + public TopicConfig examineTopicConfig(String addr, + String topic) throws RemotingSendRequestException, RemotingConnectException, RemotingTimeoutException, MQBrokerException, InterruptedException { return defaultMQAdminExtImpl.examineTopicConfig(addr, topic); } @@ -578,4 +583,52 @@ public void setMessageRequestMode(final String brokerAddr, final String topic, f RemotingConnectException, MQClientException { this.defaultMQAdminExtImpl.setMessageRequestMode(brokerAddr, topic, consumerGroup, mode, popShareQueueNum, timeoutMillis); } + + @Override + public void updateTopicLogicalQueueMapping(String brokerAddr, String topic, int queueId, int logicalQueueIndex) throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException, MQBrokerException { + this.defaultMQAdminExtImpl.updateTopicLogicalQueueMapping(brokerAddr, topic, queueId, logicalQueueIndex); + } + + @Override + public LogicalQueuesInfo queryTopicLogicalQueueMapping(String brokerAddr, String topic) throws InterruptedException, RemotingConnectException, RemotingTimeoutException, RemotingSendRequestException, MQBrokerException { + return this.defaultMQAdminExtImpl.queryTopicLogicalQueueMapping(brokerAddr, topic); + } + + @Override + public void deleteTopicLogicalQueueMapping(String brokerAddr, String topic) throws InterruptedException, MQBrokerException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException { + this.defaultMQAdminExtImpl.deleteTopicLogicalQueueMapping(brokerAddr, topic); + } + + @Override + public LogicalQueueRouteData sealTopicLogicalQueue(String brokerAddr, LogicalQueueRouteData queueRouteData) throws InterruptedException, RemotingConnectException, RemotingTimeoutException, RemotingSendRequestException, MQBrokerException { + return this.defaultMQAdminExtImpl.sealTopicLogicalQueue(brokerAddr, queueRouteData); + } + + @Override public LogicalQueueRouteData reuseTopicLogicalQueue(String brokerAddr, String topic, int queueId, + int logicalQueueIdx, MessageQueueRouteState messageQueueRouteState) throws InterruptedException, MQBrokerException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException { + return this.defaultMQAdminExtImpl.reuseTopicLogicalQueue(brokerAddr, topic, queueId, logicalQueueIdx, messageQueueRouteState); + } + + @Override public LogicalQueueRouteData createMessageQueueForLogicalQueue(String brokerAddr, String topic, + int logicalQueueIdx, MessageQueueRouteState messageQueueStatus) throws InterruptedException, MQBrokerException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException { + return this.defaultMQAdminExtImpl.createMessageQueueForLogicalQueue(brokerAddr, topic, logicalQueueIdx, messageQueueStatus); + } + + @Override public MigrateLogicalQueueBody migrateTopicLogicalQueuePrepare( + LogicalQueueRouteData fromQueueRouteData, + LogicalQueueRouteData toQueueRouteData) throws InterruptedException, MQBrokerException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException { + return this.defaultMQAdminExtImpl.migrateTopicLogicalQueuePrepare(fromQueueRouteData, toQueueRouteData); + } + + @Override public MigrateLogicalQueueBody migrateTopicLogicalQueueCommit( + LogicalQueueRouteData fromQueueRouteData, + LogicalQueueRouteData toQueueRouteData) throws InterruptedException, MQBrokerException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException { + return this.defaultMQAdminExtImpl.migrateTopicLogicalQueueCommit(fromQueueRouteData, toQueueRouteData); + } + + @Override public void migrateTopicLogicalQueueNotify(String brokerAddr, + LogicalQueueRouteData fromQueueRouteData, + LogicalQueueRouteData toQueueRouteData) throws InterruptedException, MQBrokerException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException { + this.defaultMQAdminExtImpl.migrateTopicLogicalQueueNotify(brokerAddr, fromQueueRouteData, toQueueRouteData); + } } diff --git a/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtImpl.java b/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtImpl.java index 8ae68cdecbe..ac936561e96 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtImpl.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtImpl.java @@ -66,6 +66,7 @@ import org.apache.rocketmq.common.protocol.body.ConsumerRunningInfo; import org.apache.rocketmq.common.protocol.body.GroupList; import org.apache.rocketmq.common.protocol.body.KVTable; +import org.apache.rocketmq.common.protocol.body.MigrateLogicalQueueBody; import org.apache.rocketmq.common.protocol.body.ProducerConnection; import org.apache.rocketmq.common.protocol.body.QueryConsumeQueueResponseBody; import org.apache.rocketmq.common.protocol.body.QueueTimeSpan; @@ -75,6 +76,9 @@ import org.apache.rocketmq.common.protocol.header.UpdateConsumerOffsetRequestHeader; import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData; import org.apache.rocketmq.common.protocol.route.BrokerData; +import org.apache.rocketmq.common.protocol.route.LogicalQueueRouteData; +import org.apache.rocketmq.common.protocol.route.LogicalQueuesInfo; +import org.apache.rocketmq.common.protocol.route.MessageQueueRouteState; import org.apache.rocketmq.common.protocol.route.QueueData; import org.apache.rocketmq.common.protocol.route.TopicRouteData; import org.apache.rocketmq.common.subscription.SubscriptionGroupConfig; @@ -223,8 +227,8 @@ public SubscriptionGroupConfig examineSubscriptionGroupConfig(String addr, Strin } @Override - public TopicConfig examineTopicConfig(String addr, String topic) { - return null; + public TopicConfig examineTopicConfig(String addr, String topic) throws InterruptedException, MQBrokerException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException { + return this.mqClientInstance.getMQClientAPIImpl().getTopicConfig(addr, topic, timeoutMillis); } @Override @@ -325,6 +329,54 @@ public MessageExt viewMessage(String topic, return this.mqClientInstance.getMQAdminImpl().queryMessageByUniqKey(topic, msgId); } + @Override + public void updateTopicLogicalQueueMapping(String brokerAddr, String topic, int queueId, int logicalQueueIndex) throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException, MQBrokerException { + this.mqClientInstance.getMQClientAPIImpl().updateTopicLogicalQueue(brokerAddr, topic, queueId, logicalQueueIndex, timeoutMillis); + } + + @Override + public LogicalQueuesInfo queryTopicLogicalQueueMapping(String brokerAddr, String topic) throws InterruptedException, MQBrokerException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException { + return this.mqClientInstance.getMQClientAPIImpl().queryTopicLogicalQueue(brokerAddr, topic, timeoutMillis); + } + + @Override + public void deleteTopicLogicalQueueMapping(String brokerAddr, String topic) throws InterruptedException, RemotingConnectException, RemotingTimeoutException, RemotingSendRequestException, MQBrokerException { + this.mqClientInstance.getMQClientAPIImpl().deleteTopicLogicalQueueMapping(brokerAddr, topic, timeoutMillis); + } + + @Override + public LogicalQueueRouteData sealTopicLogicalQueue(String brokerAddr, LogicalQueueRouteData queueRouteData) throws InterruptedException, MQBrokerException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException { + return this.mqClientInstance.getMQClientAPIImpl().sealTopicLogicalQueue(brokerAddr, queueRouteData, timeoutMillis); + } + + @Override public LogicalQueueRouteData reuseTopicLogicalQueue(String brokerAddr, String topic, int queueId, + int logicalQueueIdx, MessageQueueRouteState messageQueueRouteState) throws InterruptedException, MQBrokerException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException { + return this.mqClientInstance.getMQClientAPIImpl().reuseTopicLogicalQueue(brokerAddr, topic, queueId, logicalQueueIdx, messageQueueRouteState, timeoutMillis); + } + + @Override public LogicalQueueRouteData createMessageQueueForLogicalQueue(String brokerAddr, String topic, + int logicalQueueIdx, MessageQueueRouteState messageQueueStatus) throws InterruptedException, MQBrokerException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException { + return this.mqClientInstance.getMQClientAPIImpl().createMessageQueueForLogicalQueue(brokerAddr, topic, logicalQueueIdx, messageQueueStatus, timeoutMillis); + } + + @Override public MigrateLogicalQueueBody migrateTopicLogicalQueuePrepare( + LogicalQueueRouteData fromQueueRouteData, + LogicalQueueRouteData toQueueRouteData) throws InterruptedException, MQBrokerException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException { + return this.mqClientInstance.getMQClientAPIImpl().migrateTopicLogicalQueuePrepare(fromQueueRouteData.getBrokerAddr(), fromQueueRouteData, toQueueRouteData, timeoutMillis); + } + + @Override public MigrateLogicalQueueBody migrateTopicLogicalQueueCommit( + LogicalQueueRouteData fromQueueRouteData, + LogicalQueueRouteData toQueueRouteData) throws InterruptedException, MQBrokerException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException { + return this.mqClientInstance.getMQClientAPIImpl().migrateTopicLogicalQueueCommit(toQueueRouteData.getBrokerAddr(), fromQueueRouteData, toQueueRouteData, timeoutMillis); + } + + @Override public void migrateTopicLogicalQueueNotify(String brokerAddr, + LogicalQueueRouteData fromQueueRouteData, + LogicalQueueRouteData toQueueRouteData) throws InterruptedException, RemotingConnectException, RemotingTimeoutException, RemotingSendRequestException, MQBrokerException { + this.mqClientInstance.getMQClientAPIImpl().migrateTopicLogicalQueueNotify(brokerAddr, fromQueueRouteData, toQueueRouteData, timeoutMillis); + } + @Override public ConsumerConnection examineConsumerConnectionInfo( String consumerGroup) throws InterruptedException, MQBrokerException, @@ -982,6 +1034,10 @@ public long maxOffset(MessageQueue mq) throws MQClientException { return this.mqClientInstance.getMQAdminImpl().maxOffset(mq); } + public long maxOffset(MessageQueue mq, boolean committed) throws MQClientException { + return this.mqClientInstance.getMQAdminImpl().maxOffset(mq, committed); + } + @Override public long minOffset(MessageQueue mq) throws MQClientException { return this.mqClientInstance.getMQAdminImpl().minOffset(mq); diff --git a/tools/src/main/java/org/apache/rocketmq/tools/admin/MQAdminExt.java b/tools/src/main/java/org/apache/rocketmq/tools/admin/MQAdminExt.java index 5ce6db18997..c1b42f5aeb4 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/admin/MQAdminExt.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/admin/MQAdminExt.java @@ -42,6 +42,7 @@ import org.apache.rocketmq.common.protocol.body.ConsumerRunningInfo; import org.apache.rocketmq.common.protocol.body.GroupList; import org.apache.rocketmq.common.protocol.body.KVTable; +import org.apache.rocketmq.common.protocol.body.MigrateLogicalQueueBody; import org.apache.rocketmq.common.protocol.body.ProducerConnection; import org.apache.rocketmq.common.protocol.body.QueryConsumeQueueResponseBody; import org.apache.rocketmq.common.protocol.body.QueueTimeSpan; @@ -49,6 +50,9 @@ import org.apache.rocketmq.common.protocol.body.TopicConfigSerializeWrapper; import org.apache.rocketmq.common.protocol.body.TopicList; import org.apache.rocketmq.common.protocol.route.TopicRouteData; +import org.apache.rocketmq.common.protocol.route.LogicalQueueRouteData; +import org.apache.rocketmq.common.protocol.route.LogicalQueuesInfo; +import org.apache.rocketmq.common.protocol.route.MessageQueueRouteState; import org.apache.rocketmq.common.subscription.SubscriptionGroupConfig; import org.apache.rocketmq.remoting.exception.RemotingCommandException; import org.apache.rocketmq.remoting.exception.RemotingConnectException; @@ -96,7 +100,8 @@ void createAndUpdateSubscriptionGroupConfig(final String addr, SubscriptionGroupConfig examineSubscriptionGroupConfig(final String addr, final String group); - TopicConfig examineTopicConfig(final String addr, final String topic); + TopicConfig examineTopicConfig(final String addr, + final String topic) throws InterruptedException, MQBrokerException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException; TopicStatsTable examineTopicStats( final String topic) throws RemotingException, MQClientException, InterruptedException, @@ -295,4 +300,28 @@ void setMessageRequestMode(final String brokerAddr, final String topic, final St MessageRequestMode mode, final int popWorkGroupSize, final long timeoutMillis) throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException, MQClientException; + + void updateTopicLogicalQueueMapping(String brokerAddr, String topic, int queueId, int logicalQueueIndex) throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException, MQBrokerException; + + LogicalQueuesInfo queryTopicLogicalQueueMapping(String brokerAddr, String topic) throws InterruptedException, MQBrokerException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException; + + void deleteTopicLogicalQueueMapping(String brokerAddr, String topic) throws InterruptedException, RemotingConnectException, RemotingTimeoutException, RemotingSendRequestException, MQBrokerException; + + LogicalQueueRouteData sealTopicLogicalQueue(String brokerAddr, LogicalQueueRouteData queueRouteData) throws InterruptedException, MQBrokerException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException; + + LogicalQueueRouteData reuseTopicLogicalQueue(String brokerAddr, String topic, int queueId, int logicalQueueIdx, + MessageQueueRouteState messageQueueRouteState) throws InterruptedException, MQBrokerException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException; + + LogicalQueueRouteData createMessageQueueForLogicalQueue(String brokerAddr, String topic, int logicalQueueIdx, + MessageQueueRouteState messageQueueStatus) throws InterruptedException, MQBrokerException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException; + + MigrateLogicalQueueBody migrateTopicLogicalQueuePrepare(LogicalQueueRouteData fromQueueRouteData, LogicalQueueRouteData toQueueRouteData) throws InterruptedException, MQBrokerException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException; + + MigrateLogicalQueueBody migrateTopicLogicalQueueCommit( + LogicalQueueRouteData fromQueueRouteData, + LogicalQueueRouteData toQueueRouteData) throws InterruptedException, MQBrokerException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException; + + void migrateTopicLogicalQueueNotify(String brokerAddr, + LogicalQueueRouteData fromQueueRouteData, + LogicalQueueRouteData toQueueRouteData) throws InterruptedException, RemotingConnectException, RemotingTimeoutException, RemotingSendRequestException, MQBrokerException; } diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java b/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java index 6e075f79595..3e989391f80 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java @@ -52,6 +52,11 @@ import org.apache.rocketmq.tools.command.consumer.SetConsumeModeSubCommand; import org.apache.rocketmq.tools.command.consumer.StartMonitoringSubCommand; import org.apache.rocketmq.tools.command.consumer.UpdateSubGroupSubCommand; +import org.apache.rocketmq.tools.command.logicalqueue.DeleteTopicLogicalQueueMappingCommand; +import org.apache.rocketmq.tools.command.logicalqueue.MigrateTopicLogicalQueueCommand; +import org.apache.rocketmq.tools.command.logicalqueue.QueryTopicLogicalQueueMappingCommand; +import org.apache.rocketmq.tools.command.logicalqueue.UpdateTopicLogicalQueueMappingCommand; +import org.apache.rocketmq.tools.command.logicalqueue.UpdateTopicLogicalQueueNumCommand; import org.apache.rocketmq.tools.command.message.CheckMsgSendRTCommand; import org.apache.rocketmq.tools.command.message.ConsumeMessageCommand; import org.apache.rocketmq.tools.command.message.PrintMessageByQueueCommand; @@ -215,6 +220,12 @@ public static void initCommand() { initCommand(new ClusterAclConfigVersionListSubCommand()); initCommand(new UpdateGlobalWhiteAddrSubCommand()); initCommand(new GetAccessConfigSubCommand()); + + initCommand(new UpdateTopicLogicalQueueMappingCommand()); + initCommand(new DeleteTopicLogicalQueueMappingCommand()); + initCommand(new QueryTopicLogicalQueueMappingCommand()); + initCommand(new MigrateTopicLogicalQueueCommand()); + initCommand(new UpdateTopicLogicalQueueNumCommand()); } private static void initLogback() throws JoranException { diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/SubCommandException.java b/tools/src/main/java/org/apache/rocketmq/tools/command/SubCommandException.java index fadd8532ef1..d907059c03a 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/SubCommandException.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/SubCommandException.java @@ -26,6 +26,10 @@ public SubCommandException(String msg) { super(msg); } + public SubCommandException(String format, Object... args) { + super(String.format(format, args)); + } + /** * @param msg Message. * @param cause Cause. diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/logicalqueue/DeleteTopicLogicalQueueMappingCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/logicalqueue/DeleteTopicLogicalQueueMappingCommand.java new file mode 100644 index 00000000000..77eb918b0ee --- /dev/null +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/logicalqueue/DeleteTopicLogicalQueueMappingCommand.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.tools.command.logicalqueue; + +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.remoting.RPCHook; +import org.apache.rocketmq.srvutil.ServerUtil; +import org.apache.rocketmq.tools.admin.DefaultMQAdminExt; +import org.apache.rocketmq.tools.command.CommandUtil; +import org.apache.rocketmq.tools.command.SubCommand; +import org.apache.rocketmq.tools.command.SubCommandException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class DeleteTopicLogicalQueueMappingCommand implements SubCommand { + private static final Logger log = LoggerFactory.getLogger(LoggerName.STDOUT_LOGGER_NAME); + + @Override public String commandName() { + return "deleteTopicLogicalQueueMapping"; + } + + @Override public String commandDesc() { + return "delete logical queue mapping info of a topic"; + } + + @Override public Options buildCommandlineOptions(Options options) { + Option opt; + + opt = new Option("t", "topic", true, "topic name."); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("c", "clusterName", true, "cluster name."); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("b", "brokerAddr", true, "broker addr."); + opt.setRequired(false); + options.addOption(opt); + + return options; + } + + @Override public void execute(CommandLine commandLine, Options options, + RPCHook rpcHook) throws SubCommandException { + DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook); + defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis())); + + try { + String topic = commandLine.getOptionValue("t").trim(); + List brokerAddrs; + if (commandLine.hasOption("b")) { + brokerAddrs = Collections.singletonList(commandLine.getOptionValue("c").trim()); + } else if (commandLine.hasOption("c")) { + String clusterName = commandLine.getOptionValue("c").trim(); + brokerAddrs = CommandUtil.fetchMasterAddrByClusterName(defaultMQAdminExt, clusterName).stream().sorted().collect(Collectors.toList()); + } else { + ServerUtil.printCommandLineHelp("mqadmin " + this.commandName(), options); + return; + } + for (String brokerAddr : brokerAddrs) { + log.info("deleteTopicLogicalQueueMapping {} {}", brokerAddr, topic); + defaultMQAdminExt.deleteTopicLogicalQueueMapping(brokerAddr, topic); + } + } catch (Exception e) { + throw new SubCommandException(this.getClass().getSimpleName() + " command failed", e); + } finally { + defaultMQAdminExt.shutdown(); + } + } +} diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/logicalqueue/MigrateTopicLogicalQueueCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/logicalqueue/MigrateTopicLogicalQueueCommand.java new file mode 100644 index 00000000000..5da8b0d4808 --- /dev/null +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/logicalqueue/MigrateTopicLogicalQueueCommand.java @@ -0,0 +1,210 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.tools.command.logicalqueue; + +import com.google.common.collect.Multimap; +import com.google.common.collect.Multimaps; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.rocketmq.client.exception.MQBrokerException; +import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.common.TopicConfig; +import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.common.protocol.body.MigrateLogicalQueueBody; +import org.apache.rocketmq.common.protocol.route.BrokerData; +import org.apache.rocketmq.common.protocol.route.LogicalQueueRouteData; +import org.apache.rocketmq.common.protocol.route.LogicalQueuesInfo; +import org.apache.rocketmq.common.protocol.route.MessageQueueRouteState; +import org.apache.rocketmq.common.protocol.route.TopicRouteData; +import org.apache.rocketmq.remoting.RPCHook; +import org.apache.rocketmq.remoting.exception.RemotingConnectException; +import org.apache.rocketmq.remoting.exception.RemotingException; +import org.apache.rocketmq.tools.admin.DefaultMQAdminExt; +import org.apache.rocketmq.tools.command.SubCommand; +import org.apache.rocketmq.tools.command.SubCommandException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static java.util.Optional.ofNullable; + +public class MigrateTopicLogicalQueueCommand implements SubCommand { + private static final Logger log = LoggerFactory.getLogger(LoggerName.STDOUT_LOGGER_NAME); + + @Override public String commandName() { + return "migrateTopicLogicalQueue"; + } + + @Override public String commandDesc() { + return "migrate a logical queue of a topic from one broker to another."; + } + + @Override public Options buildCommandlineOptions(Options options) { + Option opt; + + opt = new Option("t", "topic", true, "topic name."); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("i", "index", true, "logical queue index."); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("b", "brokerAddr", true, "new broker name"); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("fd", "forceDelta", true, "assume fromBroker down, force migrate"); + opt.setRequired(false); + options.addOption(opt); + + return options; + } + + public void execute(DefaultMQAdminExt mqAdminExt, String topic, int logicalQueueIdx, + String toBrokerName, + Long forceDelta) throws RemotingException, MQBrokerException, InterruptedException, SubCommandException, MQClientException { + TopicRouteData topicRouteInfo = mqAdminExt.examineTopicRouteInfo(topic); + LogicalQueuesInfo logicalQueuesInfo = topicRouteInfo.getLogicalQueuesInfo(); + if (logicalQueuesInfo == null) { + throw new SubCommandException("topic not enabled logical queue"); + } + List queueRouteDataList = logicalQueuesInfo.get(logicalQueueIdx); + if (queueRouteDataList == null) { + throw new SubCommandException("logical queue %d not exist", logicalQueueIdx); + } + Map brokerAddrTable = mqAdminExt.examineBrokerClusterInfo().getBrokerAddrTable(); + String toBrokerAddr = lookupBrokerMasterAddr(brokerAddrTable, toBrokerName); + if (toBrokerAddr == null) { + throw new SubCommandException("destination broker %s not found", toBrokerName); + } + LogicalQueueRouteData fromQueueRouteData = queueRouteDataList.stream().filter(LogicalQueueRouteData::isWritable).reduce((first, second) -> second).orElse(null); + if (fromQueueRouteData == null) { + throw new SubCommandException("logical queue %d not writable, can not migrate", logicalQueueIdx); + } + String fromBrokerName = fromQueueRouteData.getBrokerName(); + String fromBrokerAddr = ofNullable(lookupBrokerMasterAddr(brokerAddrTable, fromBrokerName)).orElse(fromQueueRouteData.getBrokerAddr()); + if (fromBrokerAddr == null) { + throw new SubCommandException("unexpected source broker name %s not found", fromBrokerName); + } + LogicalQueueRouteData toQueueRouteData; + RETRY: + while (true) { + TopicConfig topicConfig = mqAdminExt.examineTopicConfig(toBrokerAddr, topic); + LogicalQueuesInfo logicalQueuesInfoInBroker = ofNullable(mqAdminExt.queryTopicLogicalQueueMapping(toBrokerAddr, topic)).orElse(new LogicalQueuesInfo()); + toQueueRouteData = logicalQueuesInfoInBroker.getOrDefault(logicalQueueIdx, Collections.emptyList()).stream().filter(queueRouteData -> Objects.equals(toBrokerName, queueRouteData.getBrokerName()) && queueRouteData.isWriteOnly()).findFirst().orElse(null); + if (toQueueRouteData == null) { + Multimap m = Multimaps.index(logicalQueuesInfoInBroker.values().stream().flatMap(Collection::stream).filter(queueRouteData -> Objects.equals(toBrokerName, queueRouteData.getBrokerName())).iterator(), LogicalQueueRouteData::getQueueId); + for (int queueId = 0, writeQueueNums = topicConfig.getWriteQueueNums(); queueId < writeQueueNums; queueId++) { + if (m.get(queueId).stream().anyMatch(LogicalQueueRouteData::isWritable)) { + continue; + } + try { + toQueueRouteData = mqAdminExt.reuseTopicLogicalQueue(toBrokerAddr, topic, queueId, logicalQueueIdx, MessageQueueRouteState.WriteOnly); + log.info("reuseTopicLogicalQueue brokerName={} brokerAddr={} queueId={} logicalQueueIdx={} ok: {}", toBrokerName, toBrokerAddr, queueId, logicalQueueIdx, toQueueRouteData); + break; + } catch (MQBrokerException e) { + if ("queue writable".equals(e.getErrorMessage())) { + log.info("reuseTopicLogicalQueue brokerName={} brokerAddr={} queueId={} logicalQueueIdx={} writable, try again.", toBrokerName, toBrokerAddr, queueId, logicalQueueIdx); + continue RETRY; + } else { + throw e; + } + } + } + } + break; + } + if (toQueueRouteData == null) { + toQueueRouteData = mqAdminExt.createMessageQueueForLogicalQueue(toBrokerAddr, topic, logicalQueueIdx, MessageQueueRouteState.WriteOnly); + log.info("createMessageQueueForLogicalQueue brokerName={} brokerAddr={} logicalQueueIdx={} ok: {}", toBrokerName, toBrokerAddr, logicalQueueIdx, toQueueRouteData); + } + MigrateLogicalQueueBody migrateLogicalQueueBody; + if (forceDelta == null) { + try { + migrateLogicalQueueBody = mqAdminExt.migrateTopicLogicalQueuePrepare(fromQueueRouteData, toQueueRouteData); + } catch (RemotingConnectException e) { + throw new SubCommandException("migrateTopicLogicalQueuePrepare", e); + } + fromQueueRouteData = migrateLogicalQueueBody.getFromQueueRouteData(); + toQueueRouteData = migrateLogicalQueueBody.getToQueueRouteData(); + log.info("migrateTopicLogicalQueuePrepare from {} to {}", fromQueueRouteData, toQueueRouteData); + } else { + toQueueRouteData.setLogicalQueueDelta(forceDelta); + log.warn("migrateTopicLogicalQueuePrepare skip with forceDelta={}", forceDelta); + } + migrateLogicalQueueBody = mqAdminExt.migrateTopicLogicalQueueCommit(fromQueueRouteData, toQueueRouteData); + toQueueRouteData = migrateLogicalQueueBody.getToQueueRouteData(); + log.info("migrateTopicLogicalQueueCommit got: {}", toQueueRouteData); + if (forceDelta == null) { + try { + mqAdminExt.migrateTopicLogicalQueueNotify(fromBrokerAddr, fromQueueRouteData, toQueueRouteData); + } finally { + log.info("migrateTopicLogicalQueueNotify fromBroker {} {}", fromQueueRouteData.getBrokerName(), fromBrokerAddr); + } + } + Collection ignoreBrokerNames = Arrays.asList(fromBrokerName, toBrokerName); + Set brokerNames = queueRouteDataList.stream() + .map(LogicalQueueRouteData::getBrokerName) + .filter(v -> !ignoreBrokerNames.contains(v)) + .map(v -> lookupBrokerMasterAddr(brokerAddrTable, v)) + .collect(Collectors.toSet()); + int i = 1; + for (String brokerName : brokerNames) { + String brokerAddr = null; + try { + brokerAddr = lookupBrokerMasterAddr(brokerAddrTable, brokerName); + mqAdminExt.migrateTopicLogicalQueueNotify(brokerAddr, fromQueueRouteData, toQueueRouteData); + } finally { + log.info("migrateTopicLogicalQueueNotify otherBroker {}({}}) ({}/{})", brokerName, brokerAddr, i, brokerNames.size()); + } + } + } + + @Override public void execute(CommandLine commandLine, Options options, + RPCHook rpcHook) throws SubCommandException { + DefaultMQAdminExt mqAdminExt = new DefaultMQAdminExt(rpcHook); + mqAdminExt.setInstanceName(Long.toString(System.currentTimeMillis())); + + try { + String topic = commandLine.getOptionValue("t").trim(); + String newBrokerName = commandLine.getOptionValue("b").trim(); + int logicalQueueIdx = Integer.parseInt(commandLine.getOptionValue("i").trim()); + Long forceDelta = null; + if (commandLine.hasOption("fd")) { + forceDelta = Long.parseLong(commandLine.getOptionValue("fd").trim()); + } + execute(mqAdminExt, topic, logicalQueueIdx, newBrokerName, forceDelta); + } catch (Exception e) { + throw new SubCommandException(this.getClass().getSimpleName() + " command failed", e); + } finally { + mqAdminExt.shutdown(); + } + } + + private static String lookupBrokerMasterAddr(Map brokerAddrTable, String brokerName) { + return ofNullable(brokerAddrTable.get(brokerName)).map(BrokerData::selectBrokerAddr).orElse(null); + } +} diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/logicalqueue/QueryTopicLogicalQueueMappingCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/logicalqueue/QueryTopicLogicalQueueMappingCommand.java new file mode 100644 index 00000000000..6ce4bac35bc --- /dev/null +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/logicalqueue/QueryTopicLogicalQueueMappingCommand.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.tools.command.logicalqueue; + +import com.alibaba.fastjson.JSON; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSortedMap; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.SortedMap; +import java.util.stream.Collectors; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.common.protocol.route.LogicalQueueRouteData; +import org.apache.rocketmq.common.protocol.route.LogicalQueuesInfo; +import org.apache.rocketmq.remoting.RPCHook; +import org.apache.rocketmq.srvutil.ServerUtil; +import org.apache.rocketmq.tools.admin.DefaultMQAdminExt; +import org.apache.rocketmq.tools.command.CommandUtil; +import org.apache.rocketmq.tools.command.SubCommand; +import org.apache.rocketmq.tools.command.SubCommandException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class QueryTopicLogicalQueueMappingCommand implements SubCommand { + private static final Logger log = LoggerFactory.getLogger(LoggerName.STDOUT_LOGGER_NAME); + + @Override public String commandName() { + return "queryTopicLogicalQueueMapping"; + } + + @Override public String commandDesc() { + return "query logical queue mapping info of a topic"; + } + + @Override public Options buildCommandlineOptions(Options options) { + Option opt; + + opt = new Option("t", "topic", true, "topic name."); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("b", "brokerAddr", true, "broker address."); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("c", "clusterName", true, "cluster name."); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("m", "merge", false, "merge all brokers' result into one."); + opt.setRequired(false); + options.addOption(opt); + + return options; + } + + @Override public void execute(CommandLine commandLine, Options options, + RPCHook rpcHook) throws SubCommandException { + DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook); + defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis())); + + try { + String topic = commandLine.getOptionValue("t").trim(); + List brokerAddrs; + if (commandLine.hasOption("b")) { + brokerAddrs = Collections.singletonList(commandLine.getOptionValue("c").trim()); + } else if (commandLine.hasOption("c")) { + String clusterName = commandLine.getOptionValue("c").trim(); + brokerAddrs = CommandUtil.fetchMasterAddrByClusterName(defaultMQAdminExt, clusterName).stream().sorted().collect(Collectors.toList()); + } else { + ServerUtil.printCommandLineHelp("mqadmin " + this.commandName(), options); + return; + } + SortedMap>> result = Maps.newTreeMap(); + for (String brokerAddr : brokerAddrs) { + LogicalQueuesInfo one = defaultMQAdminExt.queryTopicLogicalQueueMapping(brokerAddr, topic); + result.put(brokerAddr, ImmutableSortedMap.copyOf(one)); + } + if (commandLine.hasOption("m")) { + SortedMap> mergedResultMap = Maps.newTreeMap(); + result.values().stream().flatMap(map -> map.values().stream()).flatMap(Collection::stream).forEach(queueRouteData -> { + List sortedQueueRouteDataList = mergedResultMap.computeIfAbsent(queueRouteData.getLogicalQueueIndex(), ignore -> Lists.newArrayList()); + int idx = Collections.binarySearch(sortedQueueRouteDataList, queueRouteData, + Comparator.comparingLong(LogicalQueueRouteData::getLogicalQueueDelta) + .thenComparing(LogicalQueueRouteData::getMessageQueue) + .thenComparingInt(LogicalQueueRouteData::getStateOrdinal)); + if (idx < 0) { + idx = -idx - 1; + } + sortedQueueRouteDataList.add(idx, queueRouteData); + }); + System.out.printf("%s%n", JSON.toJSONString(ImmutableList.copyOf(mergedResultMap.values()))); + } else { + System.out.printf("%s%n", JSON.toJSONString(result)); + } + } catch (Exception e) { + throw new SubCommandException(this.getClass().getSimpleName() + " command failed", e); + } finally { + defaultMQAdminExt.shutdown(); + } + } +} diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/logicalqueue/UpdateTopicLogicalQueueMappingCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/logicalqueue/UpdateTopicLogicalQueueMappingCommand.java new file mode 100644 index 00000000000..46276a31cde --- /dev/null +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/logicalqueue/UpdateTopicLogicalQueueMappingCommand.java @@ -0,0 +1,159 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.tools.command.logicalqueue; + +import com.google.common.collect.Maps; +import java.util.BitSet; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.rocketmq.client.exception.MQBrokerException; +import org.apache.rocketmq.common.TopicConfig; +import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.common.protocol.route.LogicalQueueRouteData; +import org.apache.rocketmq.remoting.RPCHook; +import org.apache.rocketmq.remoting.exception.RemotingConnectException; +import org.apache.rocketmq.remoting.exception.RemotingSendRequestException; +import org.apache.rocketmq.remoting.exception.RemotingTimeoutException; +import org.apache.rocketmq.srvutil.ServerUtil; +import org.apache.rocketmq.tools.admin.DefaultMQAdminExt; +import org.apache.rocketmq.tools.command.CommandUtil; +import org.apache.rocketmq.tools.command.SubCommand; +import org.apache.rocketmq.tools.command.SubCommandException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class UpdateTopicLogicalQueueMappingCommand implements SubCommand { + private static final Logger log = LoggerFactory.getLogger(LoggerName.STDOUT_LOGGER_NAME); + + @Override public String commandName() { + return "updateTopicLogicalQueueMapping"; + } + + @Override public String commandDesc() { + return "update logical queue mapping info of a topic"; + } + + @Override public Options buildCommandlineOptions(Options options) { + Option opt; + + opt = new Option("t", "topic", true, "topic name."); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("q", "queue", true, "message queue id."); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("i", "index", true, "logical queue index."); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("b", "broker", true, "broker addr."); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("c", "clusterName", true, "cluster name."); + opt.setRequired(false); + options.addOption(opt); + + return options; + } + + public void execute(DefaultMQAdminExt defaultMQAdminExt, String topic, Collection brokerAddrs) throws InterruptedException, MQBrokerException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException { + Map topicConfigMap = Maps.newHashMapWithExpectedSize(brokerAddrs.size()); + Map allocatedMessageQueueMap = Maps.newHashMap(); + BitSet allocatedLogicalQueueIndices = new BitSet(); + brokerAddrs = brokerAddrs.stream().sorted().collect(Collectors.toList()); + for (String brokerAddr : brokerAddrs) { + TopicConfig topicConfig = defaultMQAdminExt.examineTopicConfig(brokerAddr, topic); + if (topicConfig == null) { + log.warn("examineTopicConfig brokerAddr={} topic={} not exist, skip!", brokerAddr, topic); + continue; + } + topicConfigMap.put(brokerAddr, topicConfig); + + BitSet allocatedQueueIds = new BitSet(); + Optional.ofNullable(defaultMQAdminExt.queryTopicLogicalQueueMapping(brokerAddr, topic)) + .ifPresent(queueRouteData -> queueRouteData.forEach((idx, value) -> { + allocatedLogicalQueueIndices.set(idx); + value.stream().mapToInt(LogicalQueueRouteData::getQueueId).forEach(allocatedQueueIds::set); + })); + allocatedMessageQueueMap.put(brokerAddr, allocatedQueueIds); + } + int unallocatedLogicalQueueIdx = -1; + for (Map.Entry entry : topicConfigMap.entrySet()) { + String brokerAddr = entry.getKey(); + TopicConfig topicConfig = entry.getValue(); + int queueNums = Integer.max(topicConfig.getReadQueueNums(), topicConfig.getWriteQueueNums()); + BitSet allocatedQueueIds = allocatedMessageQueueMap.get(brokerAddr); + for (int unallocatedQueueId = allocatedQueueIds.nextClearBit(0); unallocatedQueueId < queueNums; unallocatedQueueId = allocatedQueueIds.nextClearBit(unallocatedQueueId + 1)) { + unallocatedLogicalQueueIdx = allocatedLogicalQueueIndices.nextClearBit(unallocatedLogicalQueueIdx + 1); + log.info("updateTopicLogicalQueueMapping brokerAddr={} topic={} queueId={} to {}", brokerAddr, topic, unallocatedQueueId, unallocatedLogicalQueueIdx); + defaultMQAdminExt.updateTopicLogicalQueueMapping(brokerAddr, topic, unallocatedQueueId, unallocatedLogicalQueueIdx); + allocatedQueueIds.set(unallocatedQueueId); + allocatedLogicalQueueIndices.set(unallocatedLogicalQueueIdx); + } + } + } + + @Override public void execute(CommandLine commandLine, Options options, + RPCHook rpcHook) throws SubCommandException { + DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook); + defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis())); + + try { + defaultMQAdminExt.start(); + String topic = commandLine.getOptionValue("t").trim(); + List brokerAddrs; + if (commandLine.hasOption("b")) { + String brokerAddr = commandLine.getOptionValue("b").trim(); + boolean hasQueueId = commandLine.hasOption("q"); + boolean hasLogicalQueueIndex = commandLine.hasOption("i"); + if (hasQueueId && hasLogicalQueueIndex) { + int queueId = Integer.parseInt(commandLine.getOptionValue("q").trim()); + int logicalQueueIndex = Integer.parseInt(commandLine.getOptionValue("i").trim()); + defaultMQAdminExt.updateTopicLogicalQueueMapping(brokerAddr, topic, queueId, logicalQueueIndex); + log.info("updateTopicLogicalQueueMapping brokerAddr={} topic={} queueId={} to {}", brokerAddr, topic, queueId, logicalQueueIndex); + return; + } else if (hasQueueId || hasLogicalQueueIndex) { + log.error("logicalQueueIndex and queueId must be specified together."); + return; + } else { + log.error("brokerAddr specified but no logicalQueueIndex and queueId found"); + return; + } + } else if (commandLine.hasOption("c")) { + String clusterName = commandLine.getOptionValue("c").trim(); + brokerAddrs = CommandUtil.fetchMasterAddrByClusterName(defaultMQAdminExt, clusterName).stream().sorted().collect(Collectors.toList()); + } else { + ServerUtil.printCommandLineHelp("mqadmin " + this.commandName(), options); + return; + } + this.execute(defaultMQAdminExt, topic, brokerAddrs); + } catch (Exception e) { + throw new SubCommandException(this.getClass().getSimpleName() + " command failed", e); + } finally { + defaultMQAdminExt.shutdown(); + } + } +} diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/logicalqueue/UpdateTopicLogicalQueueNumCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/logicalqueue/UpdateTopicLogicalQueueNumCommand.java new file mode 100644 index 00000000000..6b9dd2fdf82 --- /dev/null +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/logicalqueue/UpdateTopicLogicalQueueNumCommand.java @@ -0,0 +1,285 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.tools.command.logicalqueue; + +import com.google.common.collect.ListMultimap; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Multimaps; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.Iterator; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.NavigableMap; +import java.util.Objects; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.rocketmq.client.exception.MQBrokerException; +import org.apache.rocketmq.common.TopicConfig; +import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.common.protocol.route.LogicalQueueRouteData; +import org.apache.rocketmq.common.protocol.route.LogicalQueuesInfo; +import org.apache.rocketmq.common.protocol.route.MessageQueueRouteState; +import org.apache.rocketmq.remoting.RPCHook; +import org.apache.rocketmq.remoting.exception.RemotingConnectException; +import org.apache.rocketmq.remoting.exception.RemotingSendRequestException; +import org.apache.rocketmq.remoting.exception.RemotingTimeoutException; +import org.apache.rocketmq.tools.admin.DefaultMQAdminExt; +import org.apache.rocketmq.tools.command.CommandUtil; +import org.apache.rocketmq.tools.command.SubCommand; +import org.apache.rocketmq.tools.command.SubCommandException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class UpdateTopicLogicalQueueNumCommand implements SubCommand { + private static final Logger log = LoggerFactory.getLogger(LoggerName.STDOUT_LOGGER_NAME); + + @Override public String commandName() { + return "updateTopicLogicalQueueNum"; + } + + @Override public String commandDesc() { + return "change logical queue num (increase or decrease) of a topic."; + } + + @Override public Options buildCommandlineOptions(Options options) { + Option opt; + + opt = new Option("t", "topic", true, "topic name."); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("c", "clusterName", true, "cluster name."); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("n", "num", true, "logical queue num."); + opt.setRequired(true); + options.addOption(opt); + + return options; + } + + @Override public void execute(CommandLine commandLine, Options options, + RPCHook rpcHook) throws SubCommandException { + DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook); + defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis())); + try { + defaultMQAdminExt.start(); + String clusterName = commandLine.getOptionValue("c").trim(); + String topic = commandLine.getOptionValue("t").trim(); + int newLogicalQueueNum = Integer.parseUnsignedInt(commandLine.getOptionValue("n")); + execute(defaultMQAdminExt, clusterName, topic, newLogicalQueueNum); + } catch (Exception e) { + throw new SubCommandException(this.getClass().getSimpleName() + " command failed", e); + } finally { + defaultMQAdminExt.shutdown(); + } + } + + public void execute(DefaultMQAdminExt defaultMQAdminExt, String clusterName, String topic, + int newLogicalQueueNum) throws RemotingSendRequestException, RemotingConnectException, RemotingTimeoutException, MQBrokerException, InterruptedException, SubCommandException { + List brokerAddrs = CommandUtil.fetchMasterAddrByClusterName(defaultMQAdminExt, clusterName).stream().sorted().collect(Collectors.toList()); + Map topicConfigsByBrokerAddr = Maps.newHashMapWithExpectedSize(brokerAddrs.size()); + NavigableMap> allLogicalQueueMapByIndex = Maps.newTreeMap(); + Map allLogicalQueueMapByBroker = Maps.newHashMap(); + for (String brokerAddr : brokerAddrs) { + TopicConfig topicConfig = defaultMQAdminExt.examineTopicConfig(brokerAddr, topic); + if (topicConfig == null) { + log.info("examineTopicConfig brokerAddr={} topic={} not exist, skip!", brokerAddr, topic); + continue; + } + topicConfigsByBrokerAddr.put(brokerAddr, topicConfig); + + LogicalQueuesInfo logicalQueuesInfo = defaultMQAdminExt.queryTopicLogicalQueueMapping(brokerAddr, topic); + if (logicalQueuesInfo == null) { + throw new SubCommandException(String.format(Locale.ENGLISH, "broker=%s topic=%s logical queue not enabled", brokerAddr, topic)); + } + allLogicalQueueMapByBroker.put(brokerAddr, logicalQueuesInfo); + logicalQueuesInfo.values().stream().flatMap(Collection::stream).forEach(queueRouteData -> { + List sortedQueueRouteDataList = allLogicalQueueMapByIndex.computeIfAbsent(queueRouteData.getLogicalQueueIndex(), ignore -> Lists.newArrayListWithExpectedSize(1)); + int idx = Collections.binarySearch(sortedQueueRouteDataList, queueRouteData, + Comparator.comparingLong(LogicalQueueRouteData::getLogicalQueueDelta) + .thenComparing(LogicalQueueRouteData::getMessageQueue) + .thenComparingInt(LogicalQueueRouteData::getStateOrdinal)); + if (idx < 0) { + idx = -idx - 1; + } + sortedQueueRouteDataList.add(idx, queueRouteData); + }); + } + int oldLogicalQueueNum = (int) allLogicalQueueMapByIndex.values().stream().filter(queueRouteDataList -> queueRouteDataList.stream().anyMatch(LogicalQueueRouteData::isWritable)).count(); + if (oldLogicalQueueNum == newLogicalQueueNum) { + log.info("logical queue num not changed!"); + } else if (oldLogicalQueueNum < newLogicalQueueNum) { + increaseLogicalQueueNum(defaultMQAdminExt, allLogicalQueueMapByBroker, allLogicalQueueMapByIndex, topicConfigsByBrokerAddr, oldLogicalQueueNum, newLogicalQueueNum); + } else { + decreaseLogicalQueueNum(defaultMQAdminExt, allLogicalQueueMapByIndex, oldLogicalQueueNum, newLogicalQueueNum); + } + } + + private void increaseLogicalQueueNum(DefaultMQAdminExt defaultMQAdminExt, + Map allLogicalQueuesInfoMapByBroker, + NavigableMap> allLogicalQueueMapByIndex, + Map topicConfigsByBrokerAddr, int oldLogicalQueueNum, + int newLogicalQueueNum) throws InterruptedException, MQBrokerException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException { + int curLogicalQueueNum = oldLogicalQueueNum; + String topic = topicConfigsByBrokerAddr.values().stream().findAny().map(TopicConfig::getTopicName).get(); + // try use queue not be assigned as logical queue + for (Map.Entry e : topicConfigsByBrokerAddr.entrySet()) { + String brokerAddr = e.getKey(); + TopicConfig topicConfig = e.getValue(); + LogicalQueuesInfo logicalQueuesInfo = allLogicalQueuesInfoMapByBroker.getOrDefault(brokerAddr, new LogicalQueuesInfo()); + ListMultimap m = Multimaps.index(logicalQueuesInfo.values().stream().flatMap(Collection::stream).iterator(), LogicalQueueRouteData::getQueueId); + for (int queueId = 0, writeQueueNums = topicConfig.getWriteQueueNums(); queueId < writeQueueNums; queueId++) { + if (m.get(queueId).stream().anyMatch(LogicalQueueRouteData::isWritable)) { + continue; + } + int logicalQueueIdx = curLogicalQueueNum; + LogicalQueueRouteData queueRouteData; + try { + queueRouteData = defaultMQAdminExt.reuseTopicLogicalQueue(brokerAddr, topic, queueId, logicalQueueIdx, MessageQueueRouteState.Normal); + } finally { + log.info("updateTopicLogicalQueueMapping reuse expired message queue from sealing logical queue brokerAddr={} queueId={} logicalQueueIdx={}", brokerAddr, queueId, logicalQueueIdx); + } + curLogicalQueueNum++; + if (curLogicalQueueNum >= newLogicalQueueNum) { + return; + } + allLogicalQueueMapByIndex.computeIfAbsent(logicalQueueIdx, integer -> Lists.newArrayListWithExpectedSize(1)).add(queueRouteData); + logicalQueuesInfo.computeIfAbsent(logicalQueueIdx, integer -> Lists.newArrayListWithExpectedSize(1)).add(queueRouteData); + } + } + // try reuse still sealing logical queue + for (Map.Entry> entry : allLogicalQueueMapByIndex.entrySet()) { + List queueRouteDataList = entry.getValue(); + if (queueRouteDataList.size() == 0 || queueRouteDataList.stream().anyMatch(LogicalQueueRouteData::isWritable)) { + continue; + } + int logicalQueueIdx = entry.getKey(); + // this is a sealing logical queue + LogicalQueueRouteData queueRouteData = queueRouteDataList.get(queueRouteDataList.size() - 1); + String brokerAddr = queueRouteData.getBrokerAddr(); + List queueRouteDataListByBroker = allLogicalQueuesInfoMapByBroker.get(brokerAddr).get(logicalQueueIdx); + if (queueRouteData.isExpired()) { + int queueId = queueRouteData.getQueueId(); + try { + queueRouteData = defaultMQAdminExt.reuseTopicLogicalQueue(brokerAddr, topic, queueId, logicalQueueIdx, MessageQueueRouteState.Normal); + } finally { + log.info("updateTopicLogicalQueueMapping reuse expired message queue from sealing logical queue brokerAddr={} queueId={} logicalQueueIdx={}", brokerAddr, queueId, logicalQueueIdx); + } + queueRouteDataList.add(queueRouteData); + queueRouteDataListByBroker.add(queueRouteData); + } else { + // create a message queue in last broker + // not expired message queue can not be reused, since delta value will not be described by one `long` + int queueId = -1; + try { + queueRouteData = defaultMQAdminExt.createMessageQueueForLogicalQueue(brokerAddr, topic, logicalQueueIdx, MessageQueueRouteState.Normal); + queueId = queueRouteData.getQueueId(); + } finally { + log.info("updateTopicLogicalQueueMapping create message queue from sealing logical queue brokerAddr={} queueId={} logicalQueueIdx={}", brokerAddr, queueId, logicalQueueIdx); + } + queueRouteDataList.add(queueRouteData); + queueRouteDataListByBroker.add(queueRouteData); + topicConfigsByBrokerAddr.get(brokerAddr).setWriteQueueNums(queueId + 1); + } + curLogicalQueueNum++; + if (curLogicalQueueNum >= newLogicalQueueNum) { + return; + } + } + // try broker already with expired message queue + for (Map.Entry entry : allLogicalQueuesInfoMapByBroker.entrySet()) { + String brokerAddr = entry.getKey(); + for (Iterator it = entry.getValue().values().stream().flatMap(Collection::stream) + .filter(LogicalQueueRouteData::isExpired) + .sorted(Comparator.comparingInt(LogicalQueueRouteData::getLogicalQueueIndex).thenComparingInt(LogicalQueueRouteData::getQueueId)) + .limit(newLogicalQueueNum - curLogicalQueueNum) + .iterator(); it.hasNext(); ) { + LogicalQueueRouteData queueRouteData = it.next(); + try { + LogicalQueueRouteData result = defaultMQAdminExt.reuseTopicLogicalQueue(brokerAddr, topic, queueRouteData.getQueueId(), queueRouteData.getLogicalQueueIndex(), MessageQueueRouteState.Normal); + // modify in-place + queueRouteData.copyFrom(result); + } finally { + log.info("updateTopicLogicalQueueMapping reuse expired message queue from sealing logical queue brokerAddr={} queueId={} logicalQueueIdx={}", brokerAddr, queueRouteData.getQueueId(), queueRouteData.getLogicalQueueIndex()); + } + allLogicalQueueMapByIndex.get(queueRouteData.getLogicalQueueIndex()).stream() + .filter(LogicalQueueRouteData::isExpired) + .filter(v -> Objects.equals(brokerAddr, v.getBrokerAddr()) && queueRouteData.getQueueId() == v.getQueueId() && queueRouteData.getLogicalQueueIndex() == v.getLogicalQueueIndex()) + .forEach(v -> v.copyFrom(queueRouteData)); + curLogicalQueueNum++; + if (curLogicalQueueNum >= newLogicalQueueNum) { + return; + } + } + } + + // try broker with least amount message queue, if amount equal, random select + for (; curLogicalQueueNum < newLogicalQueueNum; curLogicalQueueNum++) { + Map.Entry entry = allLogicalQueuesInfoMapByBroker.entrySet().stream().min(Comparator.comparingInt(value -> value.getValue().values().stream().flatMapToInt(l -> IntStream.of(l.size())).sum())).get(); + String brokerAddr = entry.getKey(); + int logicalQueueIdx = curLogicalQueueNum; + int queueId = -1; + LogicalQueueRouteData queueRouteData; + try { + queueRouteData = defaultMQAdminExt.createMessageQueueForLogicalQueue(brokerAddr, topic, logicalQueueIdx, MessageQueueRouteState.Normal); + queueId = queueRouteData.getQueueId(); + } finally { + log.info("updateTopicLogicalQueueMapping create message queue from fresh brokerAddr={} queueId={} logicalQueueIdx={}", brokerAddr, queueId, logicalQueueIdx); + } + entry.getValue().put(logicalQueueIdx, Lists.newArrayList(queueRouteData)); + } + } + + private void decreaseLogicalQueueNum(DefaultMQAdminExt defaultMQAdminExt, + NavigableMap> allLogicalQueueMapByIndex, + int oldLogicalQueueNum, + int newLogicalQueueNum) throws InterruptedException, RemotingConnectException, RemotingTimeoutException, RemotingSendRequestException, MQBrokerException, SubCommandException { + // seal logical queue from greatest index + Map.Entry> maxActiveEntry = allLogicalQueueMapByIndex.lastEntry(); + int curLogicalQueueNum = oldLogicalQueueNum; + while (curLogicalQueueNum > newLogicalQueueNum) { + boolean anyQueueSealed = false; + for (LogicalQueueRouteData queueRouteData : maxActiveEntry.getValue()) { + if (queueRouteData.isWritable()) { + anyQueueSealed = true; + LogicalQueueRouteData resultQueueRouteData = queueRouteData; + try { + resultQueueRouteData = defaultMQAdminExt.sealTopicLogicalQueue(queueRouteData.getBrokerAddr(), queueRouteData); + } finally { + log.info("seal message queue: {}", resultQueueRouteData); + } + } + } + if (anyQueueSealed) { + curLogicalQueueNum--; + } + maxActiveEntry = allLogicalQueueMapByIndex.lowerEntry(maxActiveEntry.getKey()); + if (maxActiveEntry == null) { + throw new SubCommandException(String.format(Locale.ENGLISH, "oldLogicalQueueNum=%d newLogicalQueueNum=%d curLogicalQueueNum=%d but can not find lowerEntry, unexpected situation", oldLogicalQueueNum, newLogicalQueueNum, curLogicalQueueNum)); + } + } + } +} diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateTopicSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateTopicSubCommand.java index c33ae52dbd3..b7f5379d36e 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateTopicSubCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateTopicSubCommand.java @@ -29,6 +29,7 @@ import org.apache.rocketmq.tools.command.CommandUtil; import org.apache.rocketmq.tools.command.SubCommand; import org.apache.rocketmq.tools.command.SubCommandException; +import org.apache.rocketmq.tools.command.logicalqueue.UpdateTopicLogicalQueueMappingCommand; public class UpdateTopicSubCommand implements SubCommand { @@ -67,6 +68,10 @@ public Options buildCommandlineOptions(Options options) { opt.setRequired(false); options.addOption(opt); + opt = new Option("lq", "logicalQueue", true, "set logical queue nums"); + opt.setRequired(false); + options.addOption(opt); + opt = new Option("p", "perm", true, "set topic's permission(2|4|6), intro[2:W 4:R; 6:RW]"); opt.setRequired(false); options.addOption(opt); @@ -132,7 +137,17 @@ public void execute(final CommandLine commandLine, final Options options, } topicConfig.setOrder(isOrder); + boolean useLogicalQueue = false; + if (commandLine.hasOption("lq")) { + useLogicalQueue = Boolean.parseBoolean(commandLine.getOptionValue("lq").trim()); + } + if (commandLine.hasOption('b')) { + if (useLogicalQueue) { + System.out.printf("-lq and -b can not be used together.%n"); + return; + } + String addr = commandLine.getOptionValue('b').trim(); defaultMQAdminExt.start(); @@ -156,6 +171,7 @@ public void execute(final CommandLine commandLine, final Options options, Set masterSet = CommandUtil.fetchMasterAddrByClusterName(defaultMQAdminExt, clusterName); + for (String addr : masterSet) { defaultMQAdminExt.createAndUpdateTopicConfig(addr, topicConfig); System.out.printf("create topic to %s success.%n", addr); @@ -177,6 +193,10 @@ public void execute(final CommandLine commandLine, final Options options, } System.out.printf("%s", topicConfig); + + if (useLogicalQueue) { + new UpdateTopicLogicalQueueMappingCommand().execute(defaultMQAdminExt, topicConfig.getTopicName(), masterSet); + } return; } diff --git a/tools/src/test/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtTest.java b/tools/src/test/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtTest.java index 3146b178115..e76095480fb 100644 --- a/tools/src/test/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtTest.java +++ b/tools/src/test/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtTest.java @@ -34,6 +34,7 @@ import org.apache.rocketmq.client.impl.MQClientAPIImpl; import org.apache.rocketmq.client.impl.MQClientManager; import org.apache.rocketmq.client.impl.factory.MQClientInstance; +import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.admin.ConsumeStats; import org.apache.rocketmq.common.admin.OffsetWrapper; import org.apache.rocketmq.common.admin.TopicOffset; @@ -59,6 +60,9 @@ import org.apache.rocketmq.common.protocol.heartbeat.MessageModel; import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData; import org.apache.rocketmq.common.protocol.route.BrokerData; +import org.apache.rocketmq.common.protocol.route.LogicalQueueRouteData; +import org.apache.rocketmq.common.protocol.route.LogicalQueuesInfo; +import org.apache.rocketmq.common.protocol.route.MessageQueueRouteState; import org.apache.rocketmq.common.protocol.route.QueueData; import org.apache.rocketmq.common.protocol.route.TopicRouteData; import org.apache.rocketmq.common.subscription.SubscriptionGroupConfig; @@ -68,6 +72,8 @@ import org.apache.rocketmq.remoting.exception.RemotingSendRequestException; import org.apache.rocketmq.remoting.exception.RemotingTimeoutException; import org.apache.rocketmq.tools.admin.api.MessageTrack; +import org.assertj.core.util.Lists; +import org.assertj.core.util.Maps; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; @@ -75,14 +81,24 @@ import org.mockito.junit.MockitoJUnitRunner; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @RunWith(MockitoJUnitRunner.class) public class DefaultMQAdminExtTest { + private static final String broker1Addr = "127.0.0.1:10911"; + private static final String broker1Name = "default-broker"; + private static final String cluster = "default-cluster"; + private static final String broker2Name = "broker-test"; + private static final String broker2Addr = "127.0.0.2:10911"; + private static final String topic1 = "topic_one"; + private static final String topic2 = "topic_two"; private static DefaultMQAdminExt defaultMQAdminExt; private static DefaultMQAdminExtImpl defaultMQAdminExtImpl; private static MQClientInstance mqClientInstance = MQClientManager.getInstance().getOrCreateMQClientInstance(new ClientConfig()); @@ -115,33 +131,40 @@ public static void init() throws Exception { when(mQClientAPIImpl.getBrokerConfig(anyString(), anyLong())).thenReturn(properties); Set topicSet = new HashSet<>(); - topicSet.add("topic_one"); - topicSet.add("topic_two"); + topicSet.add(topic1); + topicSet.add(topic2); topicList.setTopicList(topicSet); when(mQClientAPIImpl.getTopicListFromNameServer(anyLong())).thenReturn(topicList); List brokerDatas = new ArrayList<>(); HashMap brokerAddrs = new HashMap<>(); - brokerAddrs.put(1234l, "127.0.0.1:10911"); + brokerAddrs.put(MixAll.MASTER_ID, broker1Addr); BrokerData brokerData = new BrokerData(); - brokerData.setCluster("default-cluster"); - brokerData.setBrokerName("default-broker"); + brokerData.setCluster(cluster); + brokerData.setBrokerName(broker1Name); brokerData.setBrokerAddrs(brokerAddrs); brokerDatas.add(brokerData); + brokerDatas.add(new BrokerData(cluster, broker2Name, (HashMap) Maps.newHashMap(MixAll.MASTER_ID, broker2Addr))); topicRouteData.setBrokerDatas(brokerDatas); topicRouteData.setQueueDatas(new ArrayList()); topicRouteData.setFilterServerTable(new HashMap>()); - when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(topicRouteData); + LogicalQueuesInfo logicalQueuesInfoinfo = new LogicalQueuesInfo(); + logicalQueuesInfoinfo.put(0, Lists.newArrayList( + new LogicalQueueRouteData(0, 0, new MessageQueue(topic1, broker1Name, 0), MessageQueueRouteState.ReadOnly, 0, 1000, 2000, 3000, broker1Addr), + new LogicalQueueRouteData(0, 1000, new MessageQueue(topic1, broker2Name, 0), MessageQueueRouteState.Normal, 0, -1, -1, -1, broker2Addr) + )); + topicRouteData.setLogicalQueuesInfo(logicalQueuesInfoinfo); + when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong(), anyBoolean(), any())).thenReturn(topicRouteData); HashMap result = new HashMap<>(); - result.put("id", "1234"); - result.put("brokerName", "default-broker"); + result.put("id", String.valueOf(MixAll.MASTER_ID)); + result.put("brokerName", broker1Name); kvTable.setTable(result); when(mQClientAPIImpl.getBrokerRuntimeInfo(anyString(), anyLong())).thenReturn(kvTable); HashMap brokerAddrTable = new HashMap<>(); - brokerAddrTable.put("default-broker", brokerData); - brokerAddrTable.put("broker-test", new BrokerData()); + brokerAddrTable.put(broker1Name, brokerData); + brokerAddrTable.put(broker2Name, new BrokerData()); clusterInfo.setBrokerAddrTable(brokerAddrTable); clusterInfo.setClusterAddrTable(new HashMap>()); when(mQClientAPIImpl.getBrokerClusterInfo(anyLong())).thenReturn(clusterInfo); @@ -251,7 +274,7 @@ public void testFetchAllTopicList() throws RemotingException, MQClientException, @Test public void testFetchBrokerRuntimeStats() throws InterruptedException, MQBrokerException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException { KVTable brokerStats = defaultMQAdminExt.fetchBrokerRuntimeStats("127.0.0.1:10911"); - assertThat(brokerStats.getTable().get("id")).isEqualTo("1234"); + assertThat(brokerStats.getTable().get("id")).isEqualTo(String.valueOf(MixAll.MASTER_ID)); assertThat(brokerStats.getTable().get("brokerName")).isEqualTo("default-broker"); } @@ -277,7 +300,7 @@ public void testExamineBrokerClusterInfo() throws InterruptedException, MQBroker @Test public void testExamineConsumeStats() throws InterruptedException, RemotingException, MQClientException, MQBrokerException { ConsumeStats consumeStats = defaultMQAdminExt.examineConsumeStats("default-consumer-group", "unit-test"); - assertThat(consumeStats.getConsumeTps()).isEqualTo(1234); + assertThat(consumeStats.getConsumeTps()).isGreaterThanOrEqualTo(1234); } @Test @@ -406,4 +429,32 @@ public void testGetAllSubscriptionGroup() throws InterruptedException, MQBrokerE assertThat(subscriptionGroupWrapper.getSubscriptionGroupTable().get("Consumer-group-one").getGroupName()).isEqualTo("Consumer-group-one"); assertThat(subscriptionGroupWrapper.getSubscriptionGroupTable().get("Consumer-group-one").isConsumeBroadcastEnable()).isTrue(); } + + @Test + public void testMaxOffset() throws Exception { + when(mQClientAPIImpl.getMaxOffset(anyString(), anyString(), anyInt(), anyBoolean(), anyBoolean(), anyLong())).thenReturn(100L); + + assertThat(defaultMQAdminExt.maxOffset(new MessageQueue(topic1, broker1Name, 0))).isEqualTo(100L); + } + + @Test + public void testSearchOffset() throws Exception { + when(mQClientAPIImpl.searchOffset(anyString(), anyString(), anyInt(), anyLong(), anyLong())).thenReturn(101L); + + assertThat(defaultMQAdminExt.searchOffset(new MessageQueue(topic1, broker1Name, 0), System.currentTimeMillis())).isEqualTo(101L); + } + + @Test + public void testMaxOffset_LogicalQueue() throws Exception { + when(mQClientAPIImpl.getMaxOffset(eq(broker2Addr), anyString(), anyInt(), anyBoolean(), anyBoolean(), anyLong())).thenReturn(10L); + + assertThat(defaultMQAdminExt.maxOffset(new MessageQueue(topic1, MixAll.LOGICAL_QUEUE_MOCK_BROKER_NAME, 0))).isEqualTo(1010L); + } + + @Test + public void testSearchOffset_LogicalQueue() throws Exception { + when(mQClientAPIImpl.searchOffset(eq(broker2Addr), anyString(), anyInt(), anyLong(), anyLong())).thenReturn(11L); + + assertThat(defaultMQAdminExt.searchOffset(new MessageQueue(topic1, MixAll.LOGICAL_QUEUE_MOCK_BROKER_NAME, 0), System.currentTimeMillis())).isEqualTo(1011L); + } } \ No newline at end of file From 0846c3c648d9e9d90f9f2d548cdf26c57f22a76f Mon Sep 17 00:00:00 2001 From: chenzlalvin Date: Wed, 23 Jun 2021 17:32:28 +0800 Subject: [PATCH 5/5] [RIP-21] submodule test --- .../apache/rocketmq/test/util/MQAdmin.java | 9 +- .../apache/rocketmq/test/base/BaseConf.java | 95 +- .../test/base/IntegrationTestBase.java | 4 +- .../rocketmq/test/smoke/LogicalQueueIT.java | 1170 +++++++++++++++++ 4 files changed, 1246 insertions(+), 32 deletions(-) create mode 100644 test/src/test/java/org/apache/rocketmq/test/smoke/LogicalQueueIT.java diff --git a/test/src/main/java/org/apache/rocketmq/test/util/MQAdmin.java b/test/src/main/java/org/apache/rocketmq/test/util/MQAdmin.java index 8863ee3e52d..8c0caa6a271 100644 --- a/test/src/main/java/org/apache/rocketmq/test/util/MQAdmin.java +++ b/test/src/main/java/org/apache/rocketmq/test/util/MQAdmin.java @@ -20,6 +20,7 @@ import java.util.HashMap; import java.util.Set; import java.util.UUID; +import java.util.concurrent.ForkJoinPool; import org.apache.log4j.Logger; import org.apache.rocketmq.common.admin.TopicStatsTable; import org.apache.rocketmq.common.protocol.body.ClusterInfo; @@ -60,7 +61,7 @@ public static boolean createTopic(String nameSrvAddr, String clusterName, String } } - mqAdminExt.shutdown(); + ForkJoinPool.commonPool().execute(mqAdminExt::shutdown); return createResult; } @@ -99,7 +100,7 @@ public static boolean createSub(String nameSrvAddr, String clusterName, String c createResult = false; e.printStackTrace(); } - mqAdminExt.shutdown(); + ForkJoinPool.commonPool().execute(mqAdminExt::shutdown); return createResult; } @@ -113,7 +114,7 @@ public static ClusterInfo getCluster(String nameSrvAddr) { } catch (Exception e) { e.printStackTrace(); } - mqAdminExt.shutdown(); + ForkJoinPool.commonPool().execute(mqAdminExt::shutdown); return clusterInfo; } @@ -159,7 +160,7 @@ public void getSubConnection(String nameSrvAddr, String clusterName, String cons createResult = false; e.printStackTrace(); } - mqAdminExt.shutdown(); + ForkJoinPool.commonPool().execute(mqAdminExt::shutdown); } } diff --git a/test/src/test/java/org/apache/rocketmq/test/base/BaseConf.java b/test/src/test/java/org/apache/rocketmq/test/base/BaseConf.java index c6a835fd67f..79469e1f3b0 100644 --- a/test/src/test/java/org/apache/rocketmq/test/base/BaseConf.java +++ b/test/src/test/java/org/apache/rocketmq/test/base/BaseConf.java @@ -17,13 +17,23 @@ package org.apache.rocketmq.test.base; +import com.google.common.collect.ImmutableList; import java.util.ArrayList; import java.util.List; - +import java.util.Map; +import java.util.concurrent.ForkJoinPool; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; +import java.util.stream.Collectors; import org.apache.log4j.Logger; import org.apache.rocketmq.broker.BrokerController; +import org.apache.rocketmq.client.consumer.MQPullConsumer; +import org.apache.rocketmq.client.consumer.MQPushConsumer; +import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.client.producer.MQProducer; import org.apache.rocketmq.client.producer.TransactionListener; import org.apache.rocketmq.common.MQVersion; +import org.apache.rocketmq.common.protocol.route.BrokerData; import org.apache.rocketmq.namesrv.NamesrvController; import org.apache.rocketmq.remoting.protocol.RemotingCommand; import org.apache.rocketmq.test.client.rmq.RMQAsyncSendProducer; @@ -36,21 +46,28 @@ import org.apache.rocketmq.test.listener.AbstractListener; import org.apache.rocketmq.test.util.MQAdmin; import org.apache.rocketmq.test.util.MQRandomUtils; +import org.apache.rocketmq.tools.admin.DefaultMQAdminExt; +import org.apache.rocketmq.tools.admin.MQAdminExt; + +import static org.awaitility.Awaitility.await; public class BaseConf { - public static String nsAddr; - protected static String broker1Name; - protected static String broker2Name; - protected static String clusterName; - protected static int brokerNum; - protected static int waitTime = 5; - protected static int consumeTime = 2 * 60 * 1000; - protected static NamesrvController namesrvController; - protected static BrokerController brokerController1; - protected static BrokerController brokerController2; - protected static List mqClients = new ArrayList(); - protected static boolean debug = false; - private static Logger log = Logger.getLogger(BaseConf.class); + public final static String nsAddr; + protected final static String broker1Name; + protected final static String broker2Name; + protected final static String clusterName; + protected final static int brokerNum; + protected final static int waitTime = 5; + protected final static int consumeTime = 2 * 60 * 1000; + protected final static int QUEUE_NUMBERS = 8; + protected final static NamesrvController namesrvController; + protected final static BrokerController brokerController1; + protected final static BrokerController brokerController2; + protected final static List brokerControllerList; + protected final static Map brokerControllerMap; + protected final static List mqClients = new ArrayList(); + protected final static boolean debug = false; + private final static Logger log = Logger.getLogger(BaseConf.class); static { System.setProperty(RemotingCommand.REMOTING_VERSION_KEY, Integer.toString(MQVersion.CURRENT_VERSION)); @@ -62,14 +79,32 @@ public class BaseConf { broker1Name = brokerController1.getBrokerConfig().getBrokerName(); broker2Name = brokerController2.getBrokerConfig().getBrokerName(); brokerNum = 2; + brokerControllerList = ImmutableList.of(brokerController1, brokerController2); + brokerControllerMap = brokerControllerList.stream().collect(Collectors.toMap(input -> input.getBrokerConfig().getBrokerName(), Function.identity())); } public BaseConf() { } + // This method can't be placed in the static block of BaseConf, which seems to lead to a strange dead lock. + public static void waitBrokerRegistered(final String nsAddr, final String clusterName) { + final DefaultMQAdminExt mqAdminExt = new DefaultMQAdminExt(500); + mqAdminExt.setNamesrvAddr(nsAddr); + try { + mqAdminExt.start(); + await().atMost(30, TimeUnit.SECONDS).until(() -> { + List brokerDatas = mqAdminExt.examineTopicRouteInfo(clusterName).getBrokerDatas(); + return brokerDatas.size() == brokerNum; + }); + } catch (MQClientException e) { + log.error("init failed, please check BaseConf"); + } + ForkJoinPool.commonPool().execute(mqAdminExt::shutdown); + } + public static String initTopic() { - String topic = MQRandomUtils.getRandomTopic(); + String topic = "tt-" + MQRandomUtils.getRandomTopic(); IntegrationTestBase.initTopic(topic, nsAddr, clusterName); return topic; @@ -157,18 +192,26 @@ public static RMQNormalConsumer getConsumer(String nsAddr, String consumerGroup, } public static void shutdown() { - try { - for (Object mqClient : mqClients) { - if (mqClient instanceof AbstractMQProducer) { - ((AbstractMQProducer) mqClient).shutdown(); + ImmutableList mqClients = ImmutableList.copyOf(BaseConf.mqClients); + BaseConf.mqClients.clear(); + shutdown(mqClients); + } - } else { - ((AbstractMQConsumer) mqClient).shutdown(); - } + public static void shutdown(List mqClients) { + mqClients.forEach(mqClient -> ForkJoinPool.commonPool().execute(() -> { + if (mqClient instanceof AbstractMQProducer) { + ((AbstractMQProducer) mqClient).shutdown(); + } else if (mqClient instanceof AbstractMQConsumer) { + ((AbstractMQConsumer) mqClient).shutdown(); + } else if (mqClient instanceof MQAdminExt) { + ((MQAdminExt) mqClient).shutdown(); + } else if (mqClient instanceof MQProducer) { + ((MQProducer) mqClient).shutdown(); + } else if (mqClient instanceof MQPullConsumer) { + ((MQPullConsumer) mqClient).shutdown(); + } else if (mqClient instanceof MQPushConsumer) { + ((MQPushConsumer) mqClient).shutdown(); } - } catch (Exception e) { - e.printStackTrace(); - } - + })); } } diff --git a/test/src/test/java/org/apache/rocketmq/test/base/IntegrationTestBase.java b/test/src/test/java/org/apache/rocketmq/test/base/IntegrationTestBase.java index 82420105e07..50dc8fc64ab 100644 --- a/test/src/test/java/org/apache/rocketmq/test/base/IntegrationTestBase.java +++ b/test/src/test/java/org/apache/rocketmq/test/base/IntegrationTestBase.java @@ -47,7 +47,7 @@ public class IntegrationTestBase { protected static final List BROKER_CONTROLLERS = new ArrayList<>(); protected static final List NAMESRV_CONTROLLERS = new ArrayList<>(); protected static int topicCreateTime = 30 * 1000; - protected static final int COMMIT_LOG_SIZE = 1024 * 1024 * 100; + public static volatile int COMMIT_LOG_SIZE = 1024 * 1024 * 100; protected static final int INDEX_NUM = 1000; private static final AtomicInteger port = new AtomicInteger(40000); @@ -183,7 +183,7 @@ public static boolean initTopic(String topic, String nsAddr, String clusterName, } public static boolean initTopic(String topic, String nsAddr, String clusterName) { - return initTopic(topic, nsAddr, clusterName, 8); + return initTopic(topic, nsAddr, clusterName, BaseConf.QUEUE_NUMBERS); } public static void deleteFile(File file) { diff --git a/test/src/test/java/org/apache/rocketmq/test/smoke/LogicalQueueIT.java b/test/src/test/java/org/apache/rocketmq/test/smoke/LogicalQueueIT.java new file mode 100644 index 00000000000..b1e9013f674 --- /dev/null +++ b/test/src/test/java/org/apache/rocketmq/test/smoke/LogicalQueueIT.java @@ -0,0 +1,1170 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.test.smoke; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Maps; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Queue; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ForkJoinPool; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import org.apache.commons.lang3.RandomStringUtils; +import org.apache.commons.lang3.reflect.FieldUtils; +import org.apache.rocketmq.broker.BrokerController; +import org.apache.rocketmq.client.consumer.DefaultMQPullConsumer; +import org.apache.rocketmq.client.consumer.PullCallback; +import org.apache.rocketmq.client.consumer.PullResult; +import org.apache.rocketmq.client.consumer.PullStatus; +import org.apache.rocketmq.client.exception.MQBrokerException; +import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.client.producer.DefaultMQProducer; +import org.apache.rocketmq.client.producer.SendCallback; +import org.apache.rocketmq.client.producer.SendResult; +import org.apache.rocketmq.client.producer.SendResultForLogicalQueue; +import org.apache.rocketmq.common.MQVersion; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.TopicConfig; +import org.apache.rocketmq.common.constant.PermName; +import org.apache.rocketmq.common.message.Message; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.common.protocol.ResponseCode; +import org.apache.rocketmq.common.protocol.route.LogicalQueueRouteData; +import org.apache.rocketmq.common.protocol.route.LogicalQueuesInfo; +import org.apache.rocketmq.common.protocol.route.MessageQueueRouteState; +import org.apache.rocketmq.common.protocol.route.TopicRouteData; +import org.apache.rocketmq.namesrv.NamesrvController; +import org.apache.rocketmq.remoting.exception.RemotingConnectException; +import org.apache.rocketmq.remoting.exception.RemotingException; +import org.apache.rocketmq.remoting.protocol.RemotingCommand; +import org.apache.rocketmq.store.CommitLog; +import org.apache.rocketmq.store.DefaultMessageStore; +import org.apache.rocketmq.store.MappedFileQueue; +import org.apache.rocketmq.test.base.BaseConf; +import org.apache.rocketmq.test.base.IntegrationTestBase; +import org.apache.rocketmq.test.util.MQRandomUtils; +import org.apache.rocketmq.tools.admin.DefaultMQAdminExt; +import org.apache.rocketmq.tools.admin.DefaultMQAdminExtImpl; +import org.apache.rocketmq.tools.command.logicalqueue.MigrateTopicLogicalQueueCommand; +import org.apache.rocketmq.tools.command.logicalqueue.UpdateTopicLogicalQueueMappingCommand; +import org.apache.rocketmq.tools.command.logicalqueue.UpdateTopicLogicalQueueNumCommand; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.FixMethodOrder; +import org.junit.Test; +import org.junit.runners.MethodSorters; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static java.util.Optional.ofNullable; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.awaitility.Awaitility.await; +import static org.awaitility.Awaitility.waitAtMost; + +@FixMethodOrder(MethodSorters.NAME_ASCENDING) +public class LogicalQueueIT { + private static final Logger logger = LoggerFactory.getLogger(LogicalQueueIT.class); + public static String nsAddr; + private static String broker1Name; + private static String broker2Name; + private static String clusterName; + private static int brokerNum; + private final static int QUEUE_NUMBERS = 8; + private static NamesrvController namesrvController; + private static BrokerController brokerController1; + private static BrokerController brokerController2; + private static Map brokerControllerMap; + private final static List mqClients = new ArrayList<>(); + + private static DefaultMQProducer producer; + private static DefaultMQPullConsumer consumer; + private static DefaultMQAdminExt mqAdminExt; + private static volatile String topic = null; + private static final String placeholderTopic = "placeholder"; + private static final int MSG_SENT_TIMES = 3; + private static final int COMMIT_LOG_FILE_SIZE = 512 * 1024; + + @BeforeClass + public static void beforeClass() throws Exception { + System.setProperty(RemotingCommand.REMOTING_VERSION_KEY, Integer.toString(MQVersion.CURRENT_VERSION)); + namesrvController = IntegrationTestBase.createAndStartNamesrv(); + nsAddr = "127.0.0.1:" + namesrvController.getNettyServerConfig().getListenPort(); + + int oldCommitLogSize = IntegrationTestBase.COMMIT_LOG_SIZE; + IntegrationTestBase.COMMIT_LOG_SIZE = COMMIT_LOG_FILE_SIZE; + brokerController1 = IntegrationTestBase.createAndStartBroker(nsAddr); + brokerController2 = IntegrationTestBase.createAndStartBroker(nsAddr); + IntegrationTestBase.COMMIT_LOG_SIZE = oldCommitLogSize; + + clusterName = brokerController1.getBrokerConfig().getBrokerClusterName(); + broker1Name = brokerController1.getBrokerConfig().getBrokerName(); + broker2Name = brokerController2.getBrokerConfig().getBrokerName(); + brokerNum = 2; + brokerControllerMap = ImmutableList.of(brokerController1, brokerController2).stream().collect(Collectors.toMap(input -> input.getBrokerConfig().getBrokerName(), Function.identity())); + + BaseConf.waitBrokerRegistered(nsAddr, clusterName); + + producer = new DefaultMQProducer(MQRandomUtils.getRandomConsumerGroup()); + mqClients.add(producer); + producer.setNamesrvAddr(nsAddr); + producer.setCompressMsgBodyOverHowmuch(Integer.MAX_VALUE); + producer.setSendMsgTimeout(1000); + producer.start(); + + consumer = new DefaultMQPullConsumer(BaseConf.initConsumerGroup()); + mqClients.add(consumer); + consumer.setNamesrvAddr(nsAddr); + consumer.setConsumerPullTimeoutMillis(1000); + consumer.start(); + + mqAdminExt = new DefaultMQAdminExt(1000); + mqClients.add(mqAdminExt); + mqAdminExt.setNamesrvAddr(nsAddr); + mqAdminExt.start(); + + mqAdminExt.createTopic(clusterName, placeholderTopic, 1); + } + + @AfterClass + public static void afterClass() { + BaseConf.shutdown(mqClients); + brokerControllerMap.forEach((s, brokerController) -> brokerController.shutdown()); + ofNullable(namesrvController).ifPresent(obj -> ForkJoinPool.commonPool().execute(obj::shutdown)); + } + + @Before + public void setUp() throws Exception { + topic = "tt-" + MQRandomUtils.getRandomTopic(); + logger.info("use topic: {}", topic); + mqAdminExt.createTopic(clusterName, topic, QUEUE_NUMBERS); + assertThat(mqAdminExt.examineTopicRouteInfo(topic).getBrokerDatas()).hasSize(brokerNum); + await().atMost(5, TimeUnit.SECONDS).until(() -> !mqAdminExt.examineTopicStats(topic).getOffsetTable().isEmpty()); + + consumer.setRegisterTopics(Collections.singleton(topic)); + // consumer.setMessageQueueListener & consumer.registerMessageQueueListener are useless in DefaultMQPullConsumer, they will never work, so do not need to test it + + new UpdateTopicLogicalQueueMappingCommand().execute(mqAdminExt, topic, brokerControllerMap.values().stream().map(BrokerController::getBrokerAddr).collect(Collectors.toSet())); + } + + private static String getCurrentMethodName() { + // 0: getStackTrace + // 1: getCurrentMethodName + // 2: __realMethod__ + return Thread.currentThread().getStackTrace()[2].getMethodName(); + } + + @Test + public void test001_SendPullSync() throws Exception { + String methodName = getCurrentMethodName(); + + List publishMessageQueues = producer.fetchPublishMessageQueues(topic); + assertThat(publishMessageQueues).hasSize(brokerNum * QUEUE_NUMBERS); + Set queueIds = IntStream.range(0, brokerNum * QUEUE_NUMBERS).boxed().collect(Collectors.toSet()); + for (MessageQueue messageQueue : publishMessageQueues) { + assertThat(messageQueue.getBrokerName()).isEqualTo(MixAll.LOGICAL_QUEUE_MOCK_BROKER_NAME); + assertThat(queueIds.remove(messageQueue.getQueueId())).isTrue(); + for (int i = 0; i < MSG_SENT_TIMES; i++) { + SendResult sendResult = producer.send(new Message(topic, String.format(Locale.ENGLISH, "%s-sync-%d-%d", methodName, messageQueue.getQueueId(), i).getBytes(StandardCharsets.UTF_8)), messageQueue); + assertThat(sendResult.getMessageQueue().getBrokerName()).isEqualTo(messageQueue.getBrokerName()); + assertThat(sendResult.getMessageQueue().getQueueId()).isEqualTo(messageQueue.getQueueId()); + } + } + assertThat(queueIds).isEmpty(); + + List subscribeMessageQueues = consumer.fetchSubscribeMessageQueues(topic).stream().sorted().collect(Collectors.toList()); + assertThat(subscribeMessageQueues).hasSize(brokerNum * QUEUE_NUMBERS); + subscribeMessageQueues.sort(Comparator.comparingInt(MessageQueue::getQueueId)); + queueIds.addAll(IntStream.range(0, brokerNum * QUEUE_NUMBERS).boxed().collect(Collectors.toSet())); + for (MessageQueue messageQueue : subscribeMessageQueues) { + assertThat(messageQueue.getBrokerName()).isEqualTo(MixAll.LOGICAL_QUEUE_MOCK_BROKER_NAME); + assertThat(queueIds.remove(messageQueue.getQueueId())).isTrue(); + long offset = mqAdminExt.minOffset(messageQueue); + PullResult pullResult = consumer.pull(messageQueue, "*", offset, 10); + assertThat(pullResult.getPullStatus()).isEqualTo(PullStatus.FOUND); + assertThat(pullResult.getMsgFoundList()).hasSize(MSG_SENT_TIMES); + offset = -1; + for (int i = 0; i < MSG_SENT_TIMES; i++) { + MessageExt msg = pullResult.getMsgFoundList().get(i); + assertThat(msg.getBrokerName()).isEqualTo(MixAll.LOGICAL_QUEUE_MOCK_BROKER_NAME); + assertThat(msg.getQueueId()).isEqualTo(messageQueue.getQueueId()); + assertThat(new String(msg.getBody(), StandardCharsets.UTF_8)).isEqualTo(String.format(Locale.ENGLISH, "%s-sync-%d-%d", methodName, messageQueue.getQueueId(), i)); + if (i > 0) { + assertThat(msg.getQueueOffset()).isEqualTo(offset + i); + } else { + offset = msg.getQueueOffset(); + } + } + assertThat(maxOffsetUncommitted(messageQueue)).isEqualTo(offset + MSG_SENT_TIMES); + } + assertThat(queueIds).isEmpty(); + } + + @Test + public void test002_SendPullAsync() throws Exception { + String methodName = getCurrentMethodName(); + + List publishMessageQueues = producer.fetchPublishMessageQueues(topic); + for (MessageQueue messageQueue : publishMessageQueues) { + for (int i = 0; i < MSG_SENT_TIMES; i++) { + CompletableFuture future = new CompletableFuture<>(); + producer.send(new Message(topic, String.format(Locale.ENGLISH, "%s-async-%d-%d", methodName, messageQueue.getQueueId(), i).getBytes(StandardCharsets.UTF_8)), messageQueue, new SendCallback() { + @Override public void onSuccess(SendResult sendResult) { + future.complete(sendResult); + } + + @Override public void onException(Throwable e) { + future.completeExceptionally(e); + } + }); + SendResult sendResult = future.get(); + assertThat(sendResult.getMessageQueue().getBrokerName()).isEqualTo(messageQueue.getBrokerName()); + assertThat(sendResult.getMessageQueue().getQueueId()).isEqualTo(messageQueue.getQueueId()); + } + } + + List subscribeMessageQueues = consumer.fetchSubscribeMessageQueues(topic).stream().sorted().collect(Collectors.toList()); + for (MessageQueue messageQueue : subscribeMessageQueues) { + long offset = mqAdminExt.minOffset(messageQueue); + CompletableFuture future = new CompletableFuture<>(); + consumer.pull(messageQueue, "*", offset, 10, new PullCallback() { + @Override public void onSuccess(PullResult pullResult) { + future.complete(pullResult); + } + + @Override public void onException(Throwable e) { + future.completeExceptionally(e); + } + }); + PullResult pullResult = future.get(); + assertThat(pullResult.getPullStatus()).isEqualTo(PullStatus.FOUND); + assertThat(pullResult.getMsgFoundList()).hasSize(MSG_SENT_TIMES); + offset = -1; + Iterator it = pullResult.getMsgFoundList().iterator(); + for (int i = 0; i < MSG_SENT_TIMES; i++) { + MessageExt msg = it.next(); + assertThat(msg.getBrokerName()).isEqualTo(MixAll.LOGICAL_QUEUE_MOCK_BROKER_NAME); + assertThat(msg.getQueueId()).isEqualTo(messageQueue.getQueueId()); + assertThat(new String(msg.getBody(), StandardCharsets.UTF_8)).isEqualTo(String.format(Locale.ENGLISH, "%s-async-%d-%d", methodName, messageQueue.getQueueId(), i)); + if (i > 0) { + assertThat(msg.getQueueOffset()).isEqualTo(offset + i); + } else { + offset = msg.getQueueOffset(); + } + } + } + } + + @Test + public void test003_MigrateOnceWithoutData() throws Exception { + final String methodName = getCurrentMethodName(); + + final int logicalQueueIdx = 1; + + TopicRouteData topicRouteInfo = mqAdminExt.examineTopicRouteInfo(topic); + List logicalQueueRouteDataList1 = topicRouteInfo.getLogicalQueuesInfo().get(logicalQueueIdx); + LogicalQueueRouteData lastLogicalQueueRouteData1 = logicalQueueRouteDataList1.get(logicalQueueRouteDataList1.size() - 1); + String newBrokerName; + if (lastLogicalQueueRouteData1.getBrokerName().equals(broker1Name)) { + newBrokerName = broker2Name; + } else { + newBrokerName = broker1Name; + } + + MessageQueue migratedMessageQueue = new MessageQueue(topic, MixAll.LOGICAL_QUEUE_MOCK_BROKER_NAME, logicalQueueIdx); + + new MigrateTopicLogicalQueueCommand().execute(mqAdminExt, topic, logicalQueueIdx, newBrokerName, null); + + topicRouteInfo = mqAdminExt.examineTopicRouteInfo(topic); + assertThat(topicRouteInfo.getLogicalQueuesInfo()).isNotNull(); + for (Map.Entry> entry : topicRouteInfo.getLogicalQueuesInfo().entrySet()) { + List logicalQueueRouteDataList2 = entry.getValue(); + if (entry.getKey() == logicalQueueIdx) { + assertThat(logicalQueueRouteDataList2).hasSize(logicalQueueRouteDataList1.size() + 1); + LogicalQueueRouteData lastLogicalQueueRouteData2 = logicalQueueRouteDataList2.get(logicalQueueRouteDataList2.size() - 2); + assertThat(lastLogicalQueueRouteData2.getMessageQueue()).isEqualTo(lastLogicalQueueRouteData1.getMessageQueue()); + assertThat(lastLogicalQueueRouteData2.getOffsetMax()).isGreaterThanOrEqualTo(0L); + assertThat(lastLogicalQueueRouteData2.getMessagesCount()).isEqualTo(0L); + assertThat(lastLogicalQueueRouteData2.isWritable()).isFalse(); + assertThat(lastLogicalQueueRouteData2.isReadable()).isFalse(); + assertThat(lastLogicalQueueRouteData2.isExpired()).isTrue(); + assertThat(lastLogicalQueueRouteData2.getLogicalQueueDelta()).isEqualTo(0L); + + LogicalQueueRouteData lastLogicalQueueRouteData3 = logicalQueueRouteDataList2.get(logicalQueueRouteDataList2.size() - 1); + assertThat(lastLogicalQueueRouteData3.getBrokerName()).isEqualTo(newBrokerName); + assertThat(lastLogicalQueueRouteData3.getOffsetMax()).isLessThan(0L); + assertThat(lastLogicalQueueRouteData3.isWritable()).isTrue(); + assertThat(lastLogicalQueueRouteData3.isReadable()).isTrue(); + assertThat(lastLogicalQueueRouteData3.isExpired()).isFalse(); + assertThat(lastLogicalQueueRouteData3.getLogicalQueueDelta()).isEqualTo(0L); + } else { + assertThat(logicalQueueRouteDataList2).hasSize(1); + LogicalQueueRouteData logicalQueueRouteData = logicalQueueRouteDataList2.get(0); + assertThat(logicalQueueRouteData.getOffsetMax()).isLessThan(0L); + assertThat(logicalQueueRouteData.isWritable()).isTrue(); + assertThat(logicalQueueRouteData.isReadable()).isTrue(); + assertThat(logicalQueueRouteData.isExpired()).isFalse(); + assertThat(logicalQueueRouteData.getLogicalQueueDelta()).isEqualTo(0L); + } + } + + List subscribeMessageQueues = consumer.fetchSubscribeMessageQueues(topic).stream().sorted().collect(Collectors.toList()); + assertThat(subscribeMessageQueues).hasSize(brokerNum * QUEUE_NUMBERS); + for (MessageQueue mq : subscribeMessageQueues) { + assertThat(mqAdminExt.minOffset(mq)).isEqualTo(0L); + } + + for (int i = 0; i < MSG_SENT_TIMES; i++) { + SendResult sendResult = producer.send(new Message(topic, String.format(Locale.ENGLISH, "%s-sync-%d-%d", methodName, migratedMessageQueue.getQueueId(), i).getBytes(StandardCharsets.UTF_8)), migratedMessageQueue); + assertThat(sendResult.getMessageQueue().getBrokerName()).isEqualTo(migratedMessageQueue.getBrokerName()); + assertThat(sendResult.getMessageQueue().getQueueId()).isEqualTo(migratedMessageQueue.getQueueId()); + SendResultForLogicalQueue sendResult2 = (SendResultForLogicalQueue) sendResult; + assertThat(sendResult2.getOrigBrokerName()).isEqualTo(newBrokerName); + assertThat(sendResult2.getOrigQueueId()).isEqualTo(QUEUE_NUMBERS); + } + + for (int i = 0; i < MSG_SENT_TIMES; i++) { + CompletableFuture future = new CompletableFuture<>(); + producer.send(new Message(topic, String.format(Locale.ENGLISH, "%s-async-%d-%d", methodName, migratedMessageQueue.getQueueId(), i).getBytes(StandardCharsets.UTF_8)), migratedMessageQueue, new SendCallback() { + @Override public void onSuccess(SendResult sendResult) { + future.complete(sendResult); + } + + @Override public void onException(Throwable e) { + future.completeExceptionally(e); + } + }); + SendResult sendResult = future.get(); + assertThat(sendResult.getMessageQueue().getBrokerName()).isEqualTo(migratedMessageQueue.getBrokerName()); + assertThat(sendResult.getMessageQueue().getQueueId()).isEqualTo(migratedMessageQueue.getQueueId()); + SendResultForLogicalQueue sendResult2 = (SendResultForLogicalQueue) sendResult; + assertThat(sendResult2.getOrigBrokerName()).isEqualTo(newBrokerName); + assertThat(sendResult2.getOrigQueueId()).isEqualTo(QUEUE_NUMBERS); + } + + assertThat(maxOffsetUncommitted(migratedMessageQueue)).isEqualTo(2 * MSG_SENT_TIMES); + + waitAtMost(5, TimeUnit.SECONDS).until(() -> mqAdminExt.maxOffset(migratedMessageQueue) == 2 * MSG_SENT_TIMES); + + PullResult pullResult = consumer.pull(migratedMessageQueue, "*", 0L, 2 * MSG_SENT_TIMES); + assertThat(pullResult.getPullStatus()).isEqualTo(PullStatus.FOUND); + assertThat(pullResult.getMinOffset()).isEqualTo(0); + assertThat(pullResult.getMaxOffset()).isEqualTo(2 * MSG_SENT_TIMES); + assertThat(pullResult.getNextBeginOffset()).isEqualTo(2 * MSG_SENT_TIMES); + List msgFoundList = pullResult.getMsgFoundList(); + assertThat(msgFoundList).hasSize(2 * MSG_SENT_TIMES); + Iterator it = pullResult.getMsgFoundList().iterator(); + long offset = 0L; + for (String prefix : new String[] {"sync", "async"}) { + for (int i = 0; i < MSG_SENT_TIMES; i++) { + MessageExt msg = it.next(); + assertThat(msg.getBrokerName()).isEqualTo(MixAll.LOGICAL_QUEUE_MOCK_BROKER_NAME); + assertThat(msg.getQueueId()).isEqualTo(migratedMessageQueue.getQueueId()); + assertThat(new String(msg.getBody(), StandardCharsets.UTF_8)).isEqualTo(String.format(Locale.ENGLISH, "%s-%s-%d-%d", methodName, prefix, migratedMessageQueue.getQueueId(), i)); + assertThat(msg.getQueueOffset()).isEqualTo(offset); + offset++; + } + } + + offset = pullResult.getNextBeginOffset(); + CompletableFuture future = new CompletableFuture<>(); + consumer.pull(migratedMessageQueue, "*", offset, 10, new PullCallback() { + @Override public void onSuccess(PullResult pullResult) { + future.complete(pullResult); + } + + @Override public void onException(Throwable e) { + future.completeExceptionally(e); + } + }); + pullResult = future.get(); + assertThat(pullResult.getPullStatus()).isEqualTo(PullStatus.NO_NEW_MSG); + assertThat(pullResult.getMinOffset()).isEqualTo(0); + assertThat(pullResult.getMaxOffset()).isEqualTo(2 * MSG_SENT_TIMES); + assertThat(pullResult.getNextBeginOffset()).isEqualTo(2 * MSG_SENT_TIMES); + assertThat(pullResult.getMsgFoundList()).isNull(); + } + + @Test + public void test004_MigrateOnceWithData() throws Exception { + final String methodName = getCurrentMethodName(); + + final int logicalQueueIdx = 1; + + TopicRouteData topicRouteInfo = mqAdminExt.examineTopicRouteInfo(topic); + List logicalQueueRouteDataList1 = topicRouteInfo.getLogicalQueuesInfo().get(logicalQueueIdx); + LogicalQueueRouteData lastLogicalQueueRouteData1 = logicalQueueRouteDataList1.get(logicalQueueRouteDataList1.size() - 1); + String newBrokerName; + if (lastLogicalQueueRouteData1.getBrokerName().equals(broker1Name)) { + newBrokerName = broker2Name; + } else { + newBrokerName = broker1Name; + } + + MessageQueue migratedMessageQueue = new MessageQueue(topic, MixAll.LOGICAL_QUEUE_MOCK_BROKER_NAME, logicalQueueIdx); + + for (int i = 0; i < MSG_SENT_TIMES; i++) { + SendResult sendResult = producer.send(new Message(topic, String.format(Locale.ENGLISH, "%s-sync-%d-%d", methodName, migratedMessageQueue.getQueueId(), i).getBytes(StandardCharsets.UTF_8)), migratedMessageQueue); + assertThat(sendResult.getMessageQueue().getBrokerName()).isEqualTo(migratedMessageQueue.getBrokerName()); + assertThat(sendResult.getMessageQueue().getQueueId()).isEqualTo(migratedMessageQueue.getQueueId()); + } + assertThat(maxOffsetUncommitted(migratedMessageQueue)).isEqualTo(MSG_SENT_TIMES); + + waitAtMost(5, TimeUnit.SECONDS).until(() -> mqAdminExt.maxOffset(migratedMessageQueue) == MSG_SENT_TIMES); + + { + long offset = 0L; + PullResult pullResult = consumer.pull(migratedMessageQueue, "*", offset, 2 * MSG_SENT_TIMES); + assertThat(pullResult.getPullStatus()).isEqualTo(PullStatus.FOUND); + assertThat(pullResult.getMinOffset()).isEqualTo(0); + assertThat(pullResult.getMaxOffset()).isEqualTo(MSG_SENT_TIMES); + assertThat(pullResult.getNextBeginOffset()).isEqualTo(MSG_SENT_TIMES); + List msgFoundList = pullResult.getMsgFoundList(); + assertThat(msgFoundList).hasSize(MSG_SENT_TIMES); + Iterator it = pullResult.getMsgFoundList().iterator(); + for (int i = 0; i < MSG_SENT_TIMES; i++) { + MessageExt msg = it.next(); + assertThat(msg.getBrokerName()).isEqualTo(MixAll.LOGICAL_QUEUE_MOCK_BROKER_NAME); + assertThat(msg.getQueueId()).isEqualTo(migratedMessageQueue.getQueueId()); + assertThat(new String(msg.getBody(), StandardCharsets.UTF_8)).isEqualTo(String.format(Locale.ENGLISH, "%s-sync-%d-%d", methodName, migratedMessageQueue.getQueueId(), i)); + assertThat(msg.getQueueOffset()).isEqualTo(offset); + offset++; + } + } + + new MigrateTopicLogicalQueueCommand().execute(mqAdminExt, topic, logicalQueueIdx, newBrokerName, null); + + topicRouteInfo = mqAdminExt.examineTopicRouteInfo(topic); + assertThat(topicRouteInfo.getLogicalQueuesInfo()).isNotNull(); + for (Map.Entry> entry : topicRouteInfo.getLogicalQueuesInfo().entrySet()) { + List logicalQueueRouteDataList2 = entry.getValue(); + if (entry.getKey() == logicalQueueIdx) { + assertThat(logicalQueueRouteDataList2).hasSize(logicalQueueRouteDataList1.size() + 1); + LogicalQueueRouteData lastLogicalQueueRouteData2 = logicalQueueRouteDataList2.get(logicalQueueRouteDataList2.size() - 2); + assertThat(lastLogicalQueueRouteData2.getMessageQueue()).isEqualTo(lastLogicalQueueRouteData1.getMessageQueue()); + assertThat(lastLogicalQueueRouteData2.getOffsetMax()).isGreaterThanOrEqualTo(0L); + assertThat(lastLogicalQueueRouteData2.getMessagesCount()).isEqualTo(MSG_SENT_TIMES); + assertThat(lastLogicalQueueRouteData2.isWritable()).isFalse(); + assertThat(lastLogicalQueueRouteData2.isReadable()).isTrue(); + assertThat(lastLogicalQueueRouteData2.isExpired()).isFalse(); + assertThat(lastLogicalQueueRouteData2.getLogicalQueueDelta()).isEqualTo(0L); + + LogicalQueueRouteData lastLogicalQueueRouteData3 = logicalQueueRouteDataList2.get(logicalQueueRouteDataList2.size() - 1); + assertThat(lastLogicalQueueRouteData3.getBrokerName()).isEqualTo(newBrokerName); + assertThat(lastLogicalQueueRouteData3.getOffsetMax()).isLessThan(0L); + assertThat(lastLogicalQueueRouteData3.isWritable()).isTrue(); + assertThat(lastLogicalQueueRouteData3.isReadable()).isTrue(); + assertThat(lastLogicalQueueRouteData3.isExpired()).isFalse(); + assertThat(lastLogicalQueueRouteData3.getLogicalQueueDelta()).isEqualTo(MSG_SENT_TIMES); + } else { + assertThat(logicalQueueRouteDataList2).hasSize(1); + LogicalQueueRouteData logicalQueueRouteData = logicalQueueRouteDataList2.get(0); + assertThat(logicalQueueRouteData.getOffsetMax()).isLessThan(0L); + assertThat(logicalQueueRouteData.isWritable()).isTrue(); + assertThat(logicalQueueRouteData.isReadable()).isTrue(); + assertThat(logicalQueueRouteData.isExpired()).isFalse(); + assertThat(logicalQueueRouteData.getLogicalQueueDelta()).isEqualTo(0L); + } + } + assertThat(migratedMessageQueue).isNotNull(); + + List subscribeMessageQueues = consumer.fetchSubscribeMessageQueues(topic).stream().sorted().collect(Collectors.toList()); + assertThat(subscribeMessageQueues).hasSize(brokerNum * QUEUE_NUMBERS); + for (MessageQueue mq : subscribeMessageQueues) { + assertThat(mqAdminExt.minOffset(mq)).isEqualTo(0L); + } + + for (int i = 0; i < MSG_SENT_TIMES; i++) { + CompletableFuture future = new CompletableFuture<>(); + producer.send(new Message(topic, String.format(Locale.ENGLISH, "%s-async-%d-%d", methodName, migratedMessageQueue.getQueueId(), i).getBytes(StandardCharsets.UTF_8)), migratedMessageQueue, new SendCallback() { + @Override public void onSuccess(SendResult sendResult) { + future.complete(sendResult); + } + + @Override public void onException(Throwable e) { + future.completeExceptionally(e); + } + }); + SendResult sendResult = future.get(); + assertThat(sendResult.getMessageQueue().getBrokerName()).isEqualTo(migratedMessageQueue.getBrokerName()); + assertThat(sendResult.getMessageQueue().getQueueId()).isEqualTo(migratedMessageQueue.getQueueId()); + SendResultForLogicalQueue sendResult2 = (SendResultForLogicalQueue) sendResult; + assertThat(sendResult2.getOrigBrokerName()).isEqualTo(newBrokerName); + assertThat(sendResult2.getOrigQueueId()).isEqualTo(QUEUE_NUMBERS); + } + + assertThat(maxOffsetUncommitted(migratedMessageQueue)).isEqualTo(2 * MSG_SENT_TIMES); + + waitAtMost(5, TimeUnit.SECONDS).until(() -> mqAdminExt.maxOffset(migratedMessageQueue) == 2 * MSG_SENT_TIMES); + + long offset = 0L; + PullResult pullResult = consumer.pull(migratedMessageQueue, "*", offset, 2 * MSG_SENT_TIMES); + assertThat(pullResult.getPullStatus()).isEqualTo(PullStatus.FOUND); + assertThat(pullResult.getMinOffset()).isEqualTo(0); + assertThat(pullResult.getMaxOffset()).isEqualTo(MSG_SENT_TIMES); + assertThat(pullResult.getNextBeginOffset()).isEqualTo(MSG_SENT_TIMES); + List msgFoundList = pullResult.getMsgFoundList(); + assertThat(msgFoundList).hasSize(MSG_SENT_TIMES); + Iterator it = pullResult.getMsgFoundList().iterator(); + for (int i = 0; i < MSG_SENT_TIMES; i++) { + MessageExt msg = it.next(); + assertThat(msg.getBrokerName()).isEqualTo(MixAll.LOGICAL_QUEUE_MOCK_BROKER_NAME); + assertThat(msg.getQueueId()).isEqualTo(migratedMessageQueue.getQueueId()); + assertThat(new String(msg.getBody(), StandardCharsets.UTF_8)).isEqualTo(String.format(Locale.ENGLISH, "%s-sync-%d-%d", methodName, migratedMessageQueue.getQueueId(), i)); + assertThat(msg.getQueueOffset()).isEqualTo(offset); + offset++; + } + + offset = pullResult.getNextBeginOffset(); + CompletableFuture pullResultFuture = new CompletableFuture<>(); + consumer.pull(migratedMessageQueue, "*", offset, 2 * MSG_SENT_TIMES, new PullCallback() { + @Override public void onSuccess(PullResult pullResult) { + pullResultFuture.complete(pullResult); + } + + @Override public void onException(Throwable e) { + pullResultFuture.completeExceptionally(e); + } + }); + pullResult = pullResultFuture.get(); + assertThat(pullResult.getPullStatus()).isEqualTo(PullStatus.FOUND); + assertThat(pullResult.getMinOffset()).isEqualTo(MSG_SENT_TIMES); + assertThat(pullResult.getMaxOffset()).isEqualTo(2 * MSG_SENT_TIMES); + assertThat(pullResult.getNextBeginOffset()).isEqualTo(2 * MSG_SENT_TIMES); + msgFoundList = pullResult.getMsgFoundList(); + assertThat(msgFoundList).hasSize(MSG_SENT_TIMES); + it = pullResult.getMsgFoundList().iterator(); + for (int i = 0; i < MSG_SENT_TIMES; i++) { + MessageExt msg = it.next(); + assertThat(msg.getBrokerName()).isEqualTo(MixAll.LOGICAL_QUEUE_MOCK_BROKER_NAME); + assertThat(msg.getQueueId()).isEqualTo(migratedMessageQueue.getQueueId()); + assertThat(new String(msg.getBody(), StandardCharsets.UTF_8)).isEqualTo(String.format(Locale.ENGLISH, "%s-async-%d-%d", methodName, migratedMessageQueue.getQueueId(), i)); + assertThat(msg.getQueueOffset()).isEqualTo(offset); + offset++; + } + + offset = pullResult.getNextBeginOffset(); + pullResult = consumer.pull(migratedMessageQueue, "*", offset, 10); + assertThat(pullResult.getPullStatus()).isEqualTo(PullStatus.NO_NEW_MSG); + assertThat(pullResult.getMinOffset()).isEqualTo(MSG_SENT_TIMES); + assertThat(pullResult.getMaxOffset()).isEqualTo(2 * MSG_SENT_TIMES); + assertThat(pullResult.getNextBeginOffset()).isEqualTo(2 * MSG_SENT_TIMES); + assertThat(pullResult.getMsgFoundList()).isNull(); + } + + @Test + public void test005_MigrateWithDataBackAndForth() throws Exception { + final String methodName = getCurrentMethodName(); + + final int logicalQueueIdx = 1; + + MessageQueue migratedMessageQueue = new MessageQueue(topic, MixAll.LOGICAL_QUEUE_MOCK_BROKER_NAME, logicalQueueIdx); + + BrokerController brokerController; + + TopicRouteData topicRouteInfo = mqAdminExt.examineTopicRouteInfo(topic); + LogicalQueueRouteData lastLogicalQueueRouteData; + { + List logicalQueueRouteDataList = topicRouteInfo.getLogicalQueuesInfo().get(logicalQueueIdx); + lastLogicalQueueRouteData = logicalQueueRouteDataList.get(logicalQueueRouteDataList.size() - 1); + } + final String fromBrokerName, toBrokerName, fromBrokerAddr, toBrokerAddr; + if (lastLogicalQueueRouteData.getBrokerName().equals(broker1Name)) { + fromBrokerName = broker1Name; + fromBrokerAddr = brokerController1.getBrokerAddr(); + toBrokerName = broker2Name; + toBrokerAddr = brokerController2.getBrokerAddr(); + } else { + fromBrokerName = broker2Name; + fromBrokerAddr = brokerController2.getBrokerAddr(); + toBrokerName = broker1Name; + toBrokerAddr = brokerController1.getBrokerAddr(); + } + + int msgIdx = 0; + + for (int i = 0; i < MSG_SENT_TIMES; i++) { + SendResult sendResult = producer.send(new Message(topic, String.format(Locale.ENGLISH, "%s-%d-%d", methodName, logicalQueueIdx, msgIdx++).getBytes(StandardCharsets.UTF_8)), migratedMessageQueue); + SendResultForLogicalQueue sendResult2 = (SendResultForLogicalQueue) sendResult; + assertThat(sendResult2.getOrigBrokerName()).isEqualTo(fromBrokerName); + assertThat(sendResult2.getOrigQueueId()).isEqualTo(logicalQueueIdx); + } + + rotateBrokerCommitLog(brokerControllerMap.get(fromBrokerName)); + + new MigrateTopicLogicalQueueCommand().execute(mqAdminExt, topic, logicalQueueIdx, toBrokerName, null); + + { + LogicalQueuesInfo info; + List logicalQueueRouteDataList; + info = mqAdminExt.queryTopicLogicalQueueMapping(fromBrokerAddr, topic); + logicalQueueRouteDataList = info.get(logicalQueueIdx); + assertThat(logicalQueueRouteDataList).hasSize(2); + info = mqAdminExt.queryTopicLogicalQueueMapping(toBrokerAddr, topic); + logicalQueueRouteDataList = info.get(logicalQueueIdx); + assertThat(logicalQueueRouteDataList).hasSize(1); + } + + for (int i = 0; i < MSG_SENT_TIMES; i++) { + SendResult sendResult = producer.send(new Message(topic, String.format(Locale.ENGLISH, "%s-%d-%d", methodName, logicalQueueIdx, msgIdx++).getBytes(StandardCharsets.UTF_8)), migratedMessageQueue); + SendResultForLogicalQueue sendResult2 = (SendResultForLogicalQueue) sendResult; + assertThat(sendResult2.getOrigBrokerName()).isEqualTo(toBrokerName); + assertThat(sendResult2.getOrigQueueId()).isEqualTo(QUEUE_NUMBERS); + } + + new MigrateTopicLogicalQueueCommand().execute(mqAdminExt, topic, logicalQueueIdx, fromBrokerName, null); + // now will reuse queue with a ReadOnly one + + { + LogicalQueuesInfo info; + List logicalQueueRouteDataList; + info = mqAdminExt.queryTopicLogicalQueueMapping(fromBrokerAddr, topic); + logicalQueueRouteDataList = info.get(logicalQueueIdx); + assertThat(logicalQueueRouteDataList).hasSize(3); + info = mqAdminExt.queryTopicLogicalQueueMapping(toBrokerAddr, topic); + logicalQueueRouteDataList = info.get(logicalQueueIdx); + assertThat(logicalQueueRouteDataList).hasSize(2); + } + + for (int i = 0; i < MSG_SENT_TIMES; i++) { + SendResult sendResult = producer.send(new Message(topic, String.format(Locale.ENGLISH, "%s-%d-%d", methodName, logicalQueueIdx, msgIdx++).getBytes(StandardCharsets.UTF_8)), migratedMessageQueue); + SendResultForLogicalQueue sendResult2 = (SendResultForLogicalQueue) sendResult; + assertThat(sendResult2.getOrigBrokerName()).isEqualTo(fromBrokerName); + assertThat(sendResult2.getOrigQueueId()).isEqualTo(logicalQueueIdx); + } + + LogicalQueueRouteData logicalQueueRouteData1; + LogicalQueueRouteData logicalQueueRouteData2; + { + List logicalQueueRouteDataList; + topicRouteInfo = mqAdminExt.examineTopicRouteInfo(topic); + logicalQueueRouteDataList = topicRouteInfo.getLogicalQueuesInfo().get(logicalQueueIdx); + assertThat(logicalQueueRouteDataList).hasSize(3); + logicalQueueRouteData1 = logicalQueueRouteDataList.get(0); + assertThat(logicalQueueRouteData1.getLogicalQueueDelta()).isEqualTo(0); + assertThat(logicalQueueRouteData1.isReadable()).isTrue(); + assertThat(logicalQueueRouteData1.isWritable()).isFalse(); + assertThat(logicalQueueRouteData1.isExpired()).isFalse(); + assertThat(logicalQueueRouteData1.isWriteOnly()).isFalse(); + assertThat(logicalQueueRouteData1.getBrokerName()).isEqualTo(fromBrokerName); + assertThat(logicalQueueRouteData1.getOffsetMax()).isGreaterThanOrEqualTo(0L); + assertThat(logicalQueueRouteData1.getMessagesCount()).isEqualTo(MSG_SENT_TIMES); + assertThat(logicalQueueRouteData1.getFirstMsgTimeMillis()).isGreaterThan(0L); + assertThat(logicalQueueRouteData1.getLastMsgTimeMillis()).isGreaterThan(0L); + logicalQueueRouteData2 = logicalQueueRouteDataList.get(1); + assertThat(logicalQueueRouteData2.getLogicalQueueDelta()).isEqualTo(MSG_SENT_TIMES); + assertThat(logicalQueueRouteData2.isReadable()).isTrue(); + assertThat(logicalQueueRouteData2.isWritable()).isFalse(); + assertThat(logicalQueueRouteData2.isExpired()).isFalse(); + assertThat(logicalQueueRouteData2.isWriteOnly()).isFalse(); + assertThat(logicalQueueRouteData2.getBrokerName()).isEqualTo(toBrokerName); + assertThat(logicalQueueRouteData2.getOffsetMax()).isGreaterThanOrEqualTo(0L); + assertThat(logicalQueueRouteData2.getMessagesCount()).isEqualTo(MSG_SENT_TIMES); + assertThat(logicalQueueRouteData2.getFirstMsgTimeMillis()).isGreaterThan(0L); + assertThat(logicalQueueRouteData2.getLastMsgTimeMillis()).isGreaterThan(0L); + LogicalQueueRouteData logicalQueueRouteData3 = logicalQueueRouteDataList.get(2); + assertThat(logicalQueueRouteData3.getLogicalQueueDelta()).isEqualTo(2 * MSG_SENT_TIMES); + assertThat(logicalQueueRouteData3.isReadable()).isTrue(); + assertThat(logicalQueueRouteData3.isWritable()).isTrue(); + assertThat(logicalQueueRouteData3.isExpired()).isFalse(); + assertThat(logicalQueueRouteData3.isWriteOnly()).isFalse(); + assertThat(logicalQueueRouteData3.getBrokerName()).isEqualTo(fromBrokerName); + assertThat(logicalQueueRouteData3.getOffsetMax()).isLessThan(0L); + } + + msgIdx = 0; + forLoop: + for (long offset = 0L; ; ) { + PullResult pullResult = consumer.pull(migratedMessageQueue, "*", offset, 3 * MSG_SENT_TIMES); + switch (pullResult.getPullStatus()) { + case NO_NEW_MSG: + assertThat(offset).isGreaterThanOrEqualTo(3L * MSG_SENT_TIMES); + break forLoop; + case OFFSET_ILLEGAL: + offset = pullResult.getNextBeginOffset(); + break; + default: + assertThat(pullResult.getPullStatus()).isEqualTo(PullStatus.FOUND); + assertThat(pullResult.getMsgFoundList()).isNotNull(); + assertThat(pullResult.getMsgFoundList()).hasSize(MSG_SENT_TIMES); + for (MessageExt msg : pullResult.getMsgFoundList()) { + assertThat(new String(msg.getBody(), StandardCharsets.UTF_8)).isEqualTo(String.format(Locale.ENGLISH, "%s-%d-%d", methodName, logicalQueueIdx, msgIdx)); + msgIdx++; + assertThat(msg.getQueueOffset()).isEqualTo(offset); + offset++; + } + offset = pullResult.getNextBeginOffset(); + break; + } + } + + waitAtMost(5, TimeUnit.SECONDS).until(() -> maxOffsetUncommitted(logicalQueueRouteData1.getMessageQueue()) == mqAdminExt.maxOffset(logicalQueueRouteData1.getMessageQueue())); + waitAtMost(5, TimeUnit.SECONDS).until(() -> maxOffsetUncommitted(logicalQueueRouteData2.getMessageQueue()) == mqAdminExt.maxOffset(logicalQueueRouteData2.getMessageQueue())); + + // now verify after commit log cleaned, toBroker's first queue route data will be expired too + brokerController = brokerControllerMap.get(logicalQueueRouteData2.getBrokerName()); + rotateBrokerCommitLog(brokerController); + deleteCommitLogFiles(brokerController, 1); + + { + topicRouteInfo = mqAdminExt.examineTopicRouteInfo(topic); + List logicalQueueRouteDataList = topicRouteInfo.getLogicalQueuesInfo().get(logicalQueueIdx); + assertThat(logicalQueueRouteDataList).hasSize(2); + assertThat(logicalQueueRouteDataList.get(0)).isEqualToIgnoringGivenFields(new LogicalQueueRouteData(logicalQueueIdx, 0, new MessageQueue(topic, fromBrokerName, logicalQueueIdx), MessageQueueRouteState.ReadOnly, 0, 3, -1, -1, fromBrokerAddr), "firstMsgTimeMillis", "lastMsgTimeMillis"); + assertThat(logicalQueueRouteDataList.get(1)).isEqualToComparingFieldByField(new LogicalQueueRouteData(logicalQueueIdx, 2 * MSG_SENT_TIMES, new MessageQueue(topic, fromBrokerName, logicalQueueIdx), MessageQueueRouteState.Normal, MSG_SENT_TIMES, -1, -1, -1, fromBrokerAddr)); + } + + // try pull again, since there is an expired queue route in the middle. + { + int msgCount = 0; + Queue wantMsgIdx = new LinkedList<>(); + wantMsgIdx.addAll(IntStream.range(0, MSG_SENT_TIMES).boxed().collect(Collectors.toList())); + wantMsgIdx.addAll(IntStream.range(2 * MSG_SENT_TIMES, 3 * MSG_SENT_TIMES).boxed().collect(Collectors.toList())); + forLoop: + for (long offset = mqAdminExt.minOffset(migratedMessageQueue); ; ) { + PullResult pullResult = consumer.pull(migratedMessageQueue, "*", offset, 3 * MSG_SENT_TIMES); + switch (pullResult.getPullStatus()) { + case NO_NEW_MSG: + assertThat(msgCount).as("offset=%d", offset).isEqualTo(2 * MSG_SENT_TIMES); + break forLoop; + case OFFSET_ILLEGAL: + offset = pullResult.getNextBeginOffset(); + break; + case FOUND: + msgCount += pullResult.getMsgFoundList().size(); + boolean first = true; + for (MessageExt msg : pullResult.getMsgFoundList()) { + assertThat(new String(msg.getBody(), StandardCharsets.UTF_8)).as("offset=%d", offset).isEqualTo(String.format(Locale.ENGLISH, "%s-%d-%d", methodName, logicalQueueIdx, wantMsgIdx.poll())); + if (first) { + assertThat(msg.getQueueOffset()).isGreaterThanOrEqualTo(offset); + first = false; + } else { + assertThat(msg.getQueueOffset()).isGreaterThan(offset); + } + offset = msg.getQueueOffset(); + } + offset = pullResult.getNextBeginOffset(); + break; + default: + Assert.fail(String.format(Locale.ENGLISH, "unexpected pull offset=%d status: %s", offset, pullResult)); + } + } + } + + // rotate first queue route to expired, and pull it + brokerController = brokerControllerMap.get(logicalQueueRouteData1.getBrokerName()); + rotateBrokerCommitLog(brokerController); + deleteCommitLogFiles(brokerController, 2); + + { + List logicalQueueRouteDataList; + topicRouteInfo = mqAdminExt.examineTopicRouteInfo(topic); + logicalQueueRouteDataList = topicRouteInfo.getLogicalQueuesInfo().get(logicalQueueIdx); + assertThat(logicalQueueRouteDataList).isEqualTo(Collections.singletonList(new LogicalQueueRouteData(logicalQueueIdx, 2 * MSG_SENT_TIMES, new MessageQueue(topic, fromBrokerName, logicalQueueIdx), MessageQueueRouteState.Normal, MSG_SENT_TIMES, -1, -1, -1, fromBrokerAddr))); + } + + { + int msgCount = 0; + Queue wantMsgIdx = new LinkedList<>(); + wantMsgIdx.addAll(IntStream.range(2 * MSG_SENT_TIMES, 3 * MSG_SENT_TIMES).boxed().collect(Collectors.toList())); + forLoop: + for (long offset = mqAdminExt.minOffset(migratedMessageQueue); ; ) { + PullResult pullResult = consumer.pull(migratedMessageQueue, "*", offset, 3 * MSG_SENT_TIMES); + switch (pullResult.getPullStatus()) { + case NO_NEW_MSG: + if (msgCount != MSG_SENT_TIMES) { + Assert.fail(String.format(Locale.ENGLISH, "want %d msg but got %d", MSG_SENT_TIMES, msgCount)); + } + break forLoop; + case OFFSET_ILLEGAL: + offset = pullResult.getNextBeginOffset(); + break; + case FOUND: + msgCount += pullResult.getMsgFoundList().size(); + boolean first = true; + for (MessageExt msg : pullResult.getMsgFoundList()) { + assertThat(new String(msg.getBody(), StandardCharsets.UTF_8)).as("offset=%d", offset).isEqualTo(String.format(Locale.ENGLISH, "%s-%d-%d", methodName, logicalQueueIdx, wantMsgIdx.poll())); + if (first) { + assertThat(msg.getQueueOffset()).isGreaterThanOrEqualTo(offset); + first = false; + } else { + assertThat(msg.getQueueOffset()).isGreaterThan(offset); + } + offset = msg.getQueueOffset(); + } + offset = pullResult.getNextBeginOffset(); + break; + default: + Assert.fail(String.format(Locale.ENGLISH, "unexpected pull offset=%d status: %s", offset, pullResult)); + } + } + } + + brokerController = brokerControllerMap.get(fromBrokerName); + rotateBrokerCommitLog(brokerController); + deleteCommitLogFiles(brokerController, 1); + + { + forLoop: + for (long offset = mqAdminExt.minOffset(migratedMessageQueue); ; ) { + PullResult pullResult = consumer.pull(migratedMessageQueue, "*", offset, 3 * MSG_SENT_TIMES); + // commit log rotate and cleaned, so there is no message. + switch (pullResult.getPullStatus()) { + case NO_MATCHED_MSG: + case NO_NEW_MSG: + assertThat(pullResult.getNextBeginOffset()).isEqualTo(3 * MSG_SENT_TIMES); + break forLoop; + case OFFSET_ILLEGAL: + offset = pullResult.getNextBeginOffset(); + break; + default: + Assert.fail(String.format(Locale.ENGLISH, "unexpected pull offset=%d status: %s", offset, pullResult)); + } + } + } + + { + LogicalQueuesInfo logicalQueuesInfo = mqAdminExt.queryTopicLogicalQueueMapping(brokerController.getBrokerAddr(), topic); + List logicalQueueRouteDataList = logicalQueuesInfo.get(logicalQueueIdx); + assertThat(logicalQueueRouteDataList).isEqualTo(Collections.singletonList(new LogicalQueueRouteData(logicalQueueIdx, 2 * MSG_SENT_TIMES, new MessageQueue(topic, fromBrokerName, logicalQueueIdx), MessageQueueRouteState.Normal, MSG_SENT_TIMES, -1, -1, -1, fromBrokerAddr))); + } + + // try migrate to this broker which has a expired queue, expect it will reuse the expired one, pull it to verify if delta works well + new MigrateTopicLogicalQueueCommand().execute(mqAdminExt, topic, logicalQueueIdx, toBrokerName, null); + + { + List logicalQueueRouteDataList; + topicRouteInfo = mqAdminExt.examineTopicRouteInfo(topic); + logicalQueueRouteDataList = topicRouteInfo.getLogicalQueuesInfo().get(logicalQueueIdx); + assertThat(logicalQueueRouteDataList).isEqualTo(Arrays.asList( + new LogicalQueueRouteData(logicalQueueIdx, 2 * MSG_SENT_TIMES, new MessageQueue(topic, fromBrokerName, logicalQueueIdx), MessageQueueRouteState.Expired, MSG_SENT_TIMES, 2 * MSG_SENT_TIMES, 0, 0, fromBrokerAddr) + , new LogicalQueueRouteData(logicalQueueIdx, 3 * MSG_SENT_TIMES, new MessageQueue(topic, toBrokerName, QUEUE_NUMBERS), MessageQueueRouteState.Normal, MSG_SENT_TIMES, -1, -1, -1, toBrokerAddr) + )); + + LogicalQueuesInfo info; + info = mqAdminExt.queryTopicLogicalQueueMapping(fromBrokerAddr, topic); + logicalQueueRouteDataList = info.get(logicalQueueIdx); + assertThat(logicalQueueRouteDataList).isEqualTo(Arrays.asList( + new LogicalQueueRouteData(logicalQueueIdx, 2 * MSG_SENT_TIMES, new MessageQueue(topic, fromBrokerName, logicalQueueIdx), MessageQueueRouteState.Expired, MSG_SENT_TIMES, 2 * MSG_SENT_TIMES, 0, 0, fromBrokerAddr) + , new LogicalQueueRouteData(logicalQueueIdx, 3 * MSG_SENT_TIMES, new MessageQueue(topic, toBrokerName, QUEUE_NUMBERS), MessageQueueRouteState.Normal, MSG_SENT_TIMES, -1, -1, -1, toBrokerAddr) + )); + info = mqAdminExt.queryTopicLogicalQueueMapping(toBrokerAddr, topic); + logicalQueueRouteDataList = info.get(logicalQueueIdx); + assertThat(logicalQueueRouteDataList).isEqualTo(Collections.singletonList(new LogicalQueueRouteData(logicalQueueIdx, 3 * MSG_SENT_TIMES, new MessageQueue(topic, toBrokerName, QUEUE_NUMBERS), MessageQueueRouteState.Normal, MSG_SENT_TIMES, -1, -1, -1, toBrokerAddr))); + } + + msgIdx = 3 * MSG_SENT_TIMES; + for (int i = 0; i < MSG_SENT_TIMES; i++) { + SendResult sendResult = producer.send(new Message(topic, String.format(Locale.ENGLISH, "%s-%d-%d", methodName, logicalQueueIdx, msgIdx++).getBytes(StandardCharsets.UTF_8)), migratedMessageQueue); + SendResultForLogicalQueue sendResult2 = (SendResultForLogicalQueue) sendResult; + assertThat(sendResult2.getOrigBrokerName()).isEqualTo(toBrokerName); + assertThat(sendResult2.getOrigQueueId()).isEqualTo(QUEUE_NUMBERS); + } + + { + int msgCount = 0; + Queue wantMsgIdx = new LinkedList<>(); + wantMsgIdx.addAll(IntStream.range(3 * MSG_SENT_TIMES, 4 * MSG_SENT_TIMES).boxed().collect(Collectors.toList())); + LOOP: + for (long offset = 0L; ; ) { + PullResult pullResult = consumer.pull(migratedMessageQueue, "*", offset, 3 * MSG_SENT_TIMES); + switch (pullResult.getPullStatus()) { + case NO_NEW_MSG: + assertThat(msgCount).as("msgCount with offset=%d", offset).isEqualTo(MSG_SENT_TIMES); + break LOOP; + case OFFSET_ILLEGAL: + assertThat(pullResult.getNextBeginOffset()).isNotEqualTo(Long.MIN_VALUE); + offset = pullResult.getNextBeginOffset(); + break; + case FOUND: + msgCount += pullResult.getMsgFoundList().size(); + boolean first = true; + for (MessageExt msg : pullResult.getMsgFoundList()) { + assertThat(new String(msg.getBody(), StandardCharsets.UTF_8)).as("offset=%d", offset).isEqualTo(String.format(Locale.ENGLISH, "%s-%d-%d", methodName, logicalQueueIdx, wantMsgIdx.poll())); + if (first) { + assertThat(msg.getQueueOffset()).isGreaterThanOrEqualTo(offset); + first = false; + } else { + assertThat(msg.getQueueOffset()).isGreaterThan(offset); + } + offset = msg.getQueueOffset(); + } + offset = pullResult.getNextBeginOffset(); + break; + default: + Assert.fail(String.format(Locale.ENGLISH, "unexpected pull offset=%d status: %s", offset, pullResult)); + } + } + } + } + + @Test + public void test006_LogicalQueueNumChanged() throws Exception { + String methodName = getCurrentMethodName(); + int logicalQueueNum = brokerNum * QUEUE_NUMBERS; + + List publishMessageQueues; + publishMessageQueues = producer.fetchPublishMessageQueues(topic); + assertThat(publishMessageQueues).hasSize(logicalQueueNum); + List subscribeMessageQueues; + subscribeMessageQueues = consumer.fetchSubscribeMessageQueues(topic).stream().sorted().collect(Collectors.toList()); + assertThat(subscribeMessageQueues).hasSize(logicalQueueNum); + + logicalQueueNum++; + new UpdateTopicLogicalQueueNumCommand().execute(mqAdminExt, clusterName, topic, logicalQueueNum); + + int newAddLogicalQueueIdx = logicalQueueNum - 1; + MessageQueue newAddLogicalQueue = new MessageQueue(topic, MixAll.LOGICAL_QUEUE_MOCK_BROKER_NAME, newAddLogicalQueueIdx); + String newAddLogicalQueueBrokerName; + { + TopicRouteData topicRouteInfo = mqAdminExt.examineTopicRouteInfo(topic); + LogicalQueuesInfo info = topicRouteInfo.getLogicalQueuesInfo(); + assertThat(info).isNotNull(); + List queueRouteDataList = info.get(newAddLogicalQueueIdx); + assertThat(queueRouteDataList).isNotNull(); + assertThat(queueRouteDataList).hasSize(1); + LogicalQueueRouteData queueRouteData = queueRouteDataList.get(0); + newAddLogicalQueueBrokerName = queueRouteData.getBrokerName(); + assertThat(queueRouteData.getState()).isEqualTo(MessageQueueRouteState.Normal); + assertThat(queueRouteData.getLogicalQueueDelta()).isEqualTo(0); + assertThat(queueRouteData.getLogicalQueueIndex()).isEqualTo(newAddLogicalQueueIdx); + } + + publishMessageQueues = producer.fetchPublishMessageQueues(topic); + assertThat(publishMessageQueues).hasSize(logicalQueueNum); + Set logicalQueueIds = IntStream.range(0, logicalQueueNum).boxed().collect(Collectors.toSet()); + Map> queueIds = Maps.newHashMap(); + for (String brokerName : Arrays.asList(broker1Name, broker2Name)) { + queueIds.put(brokerName, IntStream.range(0, QUEUE_NUMBERS).boxed().collect(Collectors.toSet())); + } + queueIds.get(newAddLogicalQueueBrokerName).add(QUEUE_NUMBERS); + for (MessageQueue messageQueue : publishMessageQueues) { + assertThat(messageQueue.getBrokerName()).isEqualTo(MixAll.LOGICAL_QUEUE_MOCK_BROKER_NAME); + assertThat(logicalQueueIds.remove(messageQueue.getQueueId())).isTrue(); + for (int i = 0; i < MSG_SENT_TIMES; i++) { + SendResult sendResult = producer.send(new Message(topic, String.format(Locale.ENGLISH, "%s-%d-%d", methodName, messageQueue.getQueueId(), i).getBytes(StandardCharsets.UTF_8)), messageQueue); + assertThat(sendResult.getMessageQueue().getBrokerName()).isEqualTo(messageQueue.getBrokerName()); + assertThat(sendResult.getMessageQueue().getQueueId()).isEqualTo(messageQueue.getQueueId()); + if (i == 0) { + SendResultForLogicalQueue sendResult2 = (SendResultForLogicalQueue) sendResult; + assertThat(queueIds.get(sendResult2.getOrigBrokerName()).remove(sendResult2.getOrigQueueId())).as("brokerName %s queueId %d", sendResult2.getOrigBrokerName(), sendResult2.getOrigQueueId()).isTrue(); + } + } + } + assertThat(logicalQueueIds).isEmpty(); + + subscribeMessageQueues = consumer.fetchSubscribeMessageQueues(topic).stream().sorted().collect(Collectors.toList()); + assertThat(subscribeMessageQueues).hasSize(logicalQueueNum); + subscribeMessageQueues.sort(Comparator.comparingInt(MessageQueue::getQueueId)); + logicalQueueIds.addAll(IntStream.range(0, logicalQueueNum).boxed().collect(Collectors.toSet())); + for (MessageQueue messageQueue : subscribeMessageQueues) { + assertThat(messageQueue.getBrokerName()).isEqualTo(MixAll.LOGICAL_QUEUE_MOCK_BROKER_NAME); + assertThat(logicalQueueIds.remove(messageQueue.getQueueId())).isTrue(); + long offset = mqAdminExt.minOffset(messageQueue); + assertThat(offset).isEqualTo(0); + PullResult pullResult = consumer.pull(messageQueue, "*", offset, 10); + assertThat(pullResult.getPullStatus()).isEqualTo(PullStatus.FOUND); + assertThat(pullResult.getMsgFoundList()).hasSize(MSG_SENT_TIMES); + for (int i = 0; i < MSG_SENT_TIMES; i++) { + MessageExt msg = pullResult.getMsgFoundList().get(i); + assertThat(msg.getBrokerName()).isEqualTo(MixAll.LOGICAL_QUEUE_MOCK_BROKER_NAME); + assertThat(msg.getQueueId()).isEqualTo(messageQueue.getQueueId()); + assertThat(new String(msg.getBody(), StandardCharsets.UTF_8)).isEqualTo(String.format(Locale.ENGLISH, "%s-%d-%d", methodName, messageQueue.getQueueId(), i)); + assertThat(msg.getQueueOffset()).isEqualTo(offset + i); + } + assertThat(maxOffsetUncommitted(messageQueue)).isEqualTo(offset + MSG_SENT_TIMES); + } + assertThat(logicalQueueIds).isEmpty(); + + // increase TopicConfig write queue first then increase logical queue, expect to reuse + String broker2Addr = brokerController2.getBrokerAddr(); + TopicConfig topicConfig = mqAdminExt.examineTopicConfig(broker2Addr, topic); + topicConfig.setWriteQueueNums(topicConfig.getWriteQueueNums() + 1); + topicConfig.setReadQueueNums(topicConfig.getReadQueueNums() + 1); + mqAdminExt.createAndUpdateTopicConfig(broker2Addr, topicConfig); + logicalQueueNum++; + new UpdateTopicLogicalQueueNumCommand().execute(mqAdminExt, clusterName, topic, logicalQueueNum); + { + newAddLogicalQueueIdx = logicalQueueNum -1; + TopicRouteData topicRouteInfo = mqAdminExt.examineTopicRouteInfo(topic); + LogicalQueuesInfo info = topicRouteInfo.getLogicalQueuesInfo(); + assertThat(info).isNotNull(); + List queueRouteDataList = info.get(newAddLogicalQueueIdx); + assertThat(queueRouteDataList).isNotNull(); + assertThat(queueRouteDataList).hasSize(1); + LogicalQueueRouteData queueRouteData = queueRouteDataList.get(0); + assertThat(queueRouteData.getState()).isEqualTo(MessageQueueRouteState.Normal); + assertThat(queueRouteData.getLogicalQueueDelta()).isEqualTo(0); + assertThat(queueRouteData.getLogicalQueueIndex()).isEqualTo(newAddLogicalQueueIdx); + assertThat(queueRouteData.getBrokerName()).isEqualTo(broker2Name); + assertThat(queueRouteData.getQueueId()).isEqualTo(topicConfig.getWriteQueueNums() -1); + } + + logicalQueueNum-=2; + new UpdateTopicLogicalQueueNumCommand().execute(mqAdminExt, clusterName, topic, logicalQueueNum); + + try { + producer.send(new Message(topic, "aaa".getBytes(StandardCharsets.UTF_8)), newAddLogicalQueue); + Assert.fail("write to decreased logical queue success, want it failed"); + } catch (MQBrokerException e) { + assertThat(e.getResponseCode()).isEqualTo(ResponseCode.NO_PERMISSION); + } + { + int offset = 0; + PullResult pullResult = consumer.pull(newAddLogicalQueue, "*", offset, 10); + assertThat(pullResult.getPullStatus()).isEqualTo(PullStatus.FOUND); + assertThat(pullResult.getMsgFoundList()).hasSize(MSG_SENT_TIMES); + for (int i = 0; i < MSG_SENT_TIMES; i++) { + MessageExt msg = pullResult.getMsgFoundList().get(i); + assertThat(msg.getBrokerName()).isEqualTo(MixAll.LOGICAL_QUEUE_MOCK_BROKER_NAME); + assertThat(msg.getQueueId()).isEqualTo(newAddLogicalQueue.getQueueId()); + assertThat(new String(msg.getBody(), StandardCharsets.UTF_8)).isEqualTo(String.format(Locale.ENGLISH, "%s-%d-%d", methodName, newAddLogicalQueue.getQueueId(), i)); + assertThat(msg.getQueueOffset()).isEqualTo(offset + i); + } + } + + // rotate to remove new add queue's data, and try pull again + { + BrokerController brokerController = brokerControllerMap.get(newAddLogicalQueueBrokerName); + rotateBrokerCommitLog(brokerController); + deleteCommitLogFiles(brokerController, 1); + } + { + int offset = 0; + PullResult pullResult = consumer.pull(newAddLogicalQueue, "*", offset, 10); + assertThat(pullResult.getPullStatus()).isIn(PullStatus.NO_NEW_MSG, PullStatus.NO_MATCHED_MSG); + } + } + + @Test + public void test007_LogicalQueueWritableEvenBrokerDown() throws Exception { + final String methodName = getCurrentMethodName(); + + final int logicalQueueIdx = 1; + + BrokerController brokerController3 = IntegrationTestBase.createAndStartBroker(nsAddr); + String broker3Name = brokerController3.getBrokerConfig().getBrokerName(); + brokerControllerMap.put(broker3Name, brokerController3); + await().atMost(30, TimeUnit.SECONDS).until(() -> mqAdminExt.examineBrokerClusterInfo().getBrokerAddrTable().containsKey(broker3Name)); + mqAdminExt.createAndUpdateTopicConfig(brokerController3.getBrokerAddr(), new TopicConfig(topic, 0, 0, PermName.PERM_READ | PermName.PERM_WRITE)); + + new MigrateTopicLogicalQueueCommand().execute(mqAdminExt, topic, logicalQueueIdx, brokerController3.getBrokerConfig().getBrokerName(), null); + + MessageQueue migrateMessageQueue = new MessageQueue(topic, MixAll.LOGICAL_QUEUE_MOCK_BROKER_NAME, logicalQueueIdx); + { + for (int i = 0; i < MSG_SENT_TIMES; i++) { + SendResult sendResult = producer.send(new Message(topic, String.format(Locale.ENGLISH, "%s-%d-%d", methodName, migrateMessageQueue.getQueueId(), i).getBytes(StandardCharsets.UTF_8)), migrateMessageQueue); + assertThat(sendResult.getMessageQueue().getBrokerName()).isEqualTo(migrateMessageQueue.getBrokerName()); + assertThat(sendResult.getMessageQueue().getQueueId()).isEqualTo(migrateMessageQueue.getQueueId()); + SendResultForLogicalQueue sendResult2 = (SendResultForLogicalQueue) sendResult; + assertThat(sendResult2.getOrigBrokerName()).isEqualTo(broker3Name); + assertThat(sendResult2.getOrigQueueId()).isEqualTo(0); + } + } + brokerController3.shutdown(); + brokerControllerMap.remove(broker3Name); + + assertThatThrownBy(() -> { + SendResult sendResult = producer.send(new Message(topic, "aaa".getBytes(StandardCharsets.UTF_8)), migrateMessageQueue); + logger.error("send should fail but got {}", sendResult); + }).isInstanceOf(RemotingException.class).hasMessageMatching("connect to [0-9.:]+ failed"); + + assertThatThrownBy(() -> { + new MigrateTopicLogicalQueueCommand().execute(mqAdminExt, topic, logicalQueueIdx, broker1Name, null); + }).hasRootCauseInstanceOf(RemotingConnectException.class).hasMessageContaining("migrateTopicLogicalQueuePrepare"); + + { + SendResult sendResult = producer.send(new Message(topic, "aaa".getBytes(StandardCharsets.UTF_8)), migrateMessageQueue); + assertThat(sendResult.getMessageQueue().getBrokerName()).isEqualTo(migrateMessageQueue.getBrokerName()); + assertThat(sendResult.getMessageQueue().getQueueId()).isEqualTo(migrateMessageQueue.getQueueId()); + assertThat(sendResult.getQueueOffset()).isEqualTo(-1); + SendResultForLogicalQueue sendResult2 = (SendResultForLogicalQueue) sendResult; + assertThat(sendResult2.getOrigBrokerName()).isEqualTo(broker1Name); + assertThat(sendResult2.getOrigQueueId()).isIn( + /* CommitLog not rotated, will not reuse */QUEUE_NUMBERS, + /* CommitLog rotated in other test cases, will reuse */logicalQueueIdx + ); + } + } + + private static String getBrokerCommitLogFileName(BrokerController brokerController) throws IllegalAccessException { + DefaultMessageStore defaultMessageStore = (DefaultMessageStore) brokerController.getMessageStore(); + MappedFileQueue mfq = (MappedFileQueue) FieldUtils.readDeclaredField(defaultMessageStore.getCommitLog(), "mappedFileQueue", true); + return mfq.getLastMappedFile().getFileName(); + } + + private static void deleteCommitLogFiles(BrokerController brokerController, + int keepNum) throws IllegalAccessException { + CommitLog commitLog = ((DefaultMessageStore) brokerController.getMessageStore()).getCommitLog(); + commitLog.flush(); + MappedFileQueue mfq = (MappedFileQueue) FieldUtils.readDeclaredField(commitLog, "mappedFileQueue", true); + AtomicInteger count = new AtomicInteger(); + waitAtMost(5, TimeUnit.SECONDS).until(() -> { + count.getAndAdd(commitLog.deleteExpiredFile(0, 0, 5000, true, 1)); + return mfq.getMappedFiles().size() <= keepNum; + }); + brokerController.getTopicConfigManager().getLogicalQueueCleanHook().execute((DefaultMessageStore) brokerController.getMessageStore(), count.get()); + logger.info("deleteCommitLogFiles {} count {}", brokerController.getBrokerConfig().getBrokerName(), count.get()); + } + + private static void rotateBrokerCommitLog(BrokerController brokerController) throws IllegalAccessException { + CommitLog commitLog = ((DefaultMessageStore) brokerController.getMessageStore()).getCommitLog(); + commitLog.flush(); + String brokerName = brokerController.getBrokerConfig().getBrokerName(); + String fileName1 = getBrokerCommitLogFileName(brokerController); + logger.info("rotateBrokerCommitLog {} first {}", brokerName, fileName1); + int msgSize = 4 * 1024; + byte[] data = RandomStringUtils.randomAscii(msgSize).getBytes(StandardCharsets.UTF_8); + Message msg = new Message(placeholderTopic, data); + MessageQueue mq = new MessageQueue(placeholderTopic, brokerName, 0); + waitAtMost(5, TimeUnit.SECONDS).until(() -> { + for (int i = 0; i < 128; i++) { + producer.send(msg, mq); + } + commitLog.flush(); + String fileName2 = getBrokerCommitLogFileName(brokerController); + if (!fileName1.equals(fileName2)) { + logger.info("rotateBrokerCommitLog {} 4K msg last {}", brokerName, fileName2); + return true; + } + return false; + }); + } + + private long maxOffsetUncommitted(MessageQueue mq) throws IllegalAccessException, MQClientException { + DefaultMQAdminExtImpl defaultMQAdminExtImpl = (DefaultMQAdminExtImpl) FieldUtils.readDeclaredField(mqAdminExt, "defaultMQAdminExtImpl", true); + return defaultMQAdminExtImpl.maxOffset(mq, false); + } +}