消息重放（构建consumequeue）

入口： org.apache.rocketmq.store.DefaultMessageStore.ReputMessageService#doReput

消息重放服务，主要是用来将commitlog中的消息发送至consumequeue和indexFile

消息重放服务ReputMessageService，在broker启动时，在消息储存服务启动方法中会启动消息重放服务：this.reputMessageService.start();其相当于一个定时任务，每隔一毫秒执行一次重放方法

重放方法中，根据上一次重放结束的偏移量(启动初始为所有queue的最大物理偏移量) , 在commitLog中查找消息其之后的所有消息, 然后循环获取到的消息执行重放至comsumequeue、indexFile、布隆过滤器

若broker开启了长轮询并且为主节点，还会主动唤醒消息拉取的长轮询的线程

class ReputMessageService extends ServiceThread {private volatile long reputFromOffset = 0;private boolean isCommitLogAvailable() {return this.reputFromOffset < DefaultMessageStore.this.commitLog.getMaxOffset();}private void doReput() {// reputFromOffset 启动时的值是所有consumequeue中的最大物理偏移量// 循环中，reputFromOffset 为上次重放的结束物理偏移量if (this.reputFromOffset < DefaultMessageStore.this.commitLog.getMinOffset()) {log.warn("The reputFromOffset={} is smaller than minPyOffset={}, this usually indicate that the dispatch behind too much and the commitlog has expired.",this.reputFromOffset, DefaultMessageStore.this.commitLog.getMinOffset());this.reputFromOffset = DefaultMessageStore.this.commitLog.getMinOffset();}for (boolean doNext = true; this.isCommitLogAvailable() && doNext; ) {if (DefaultMessageStore.this.getMessageStoreConfig().isDuplicationEnable()&& this.reputFromOffset >= DefaultMessageStore.this.getConfirmOffset()) {break;}// 返回 reputFromOffset 偏移量开始的全部有效数据（commitlog文件）。然后循环读取每一条消息SelectMappedBufferResult result = DefaultMessageStore.this.commitLog.getData(reputFromOffset);if (result != null) {try {this.reputFromOffset = result.getStartOffset();for (int readSize = 0; readSize < result.getSize() && doNext; ) {DispatchRequest dispatchRequest =DefaultMessageStore.this.commitLog.checkMessageAndReturnSize(result.getByteBuffer(), false, false);// 获取最前面一条消息的大小int size = dispatchRequest.getBufferSize() == -1 ? dispatchRequest.getMsgSize() : dispatchRequest.getBufferSize();if (dispatchRequest.isSuccess()) {if (size > 0) {// 构建consumequeue、index文件DefaultMessageStore.this.doDispatch(dispatchRequest);// Broker 端开启了长轮询模式, 并且角色为主节点, 则调用arriving方法唤醒挂起的长轮询线程if (BrokerRole.SLAVE != DefaultMessageStore.this.getMessageStoreConfig().getBrokerRole()&& DefaultMessageStore.this.brokerConfig.isLongPollingEnable()&& DefaultMessageStore.this.messageArrivingListener != null) {// 消息送达的监听器，生产者消息到达时通过NotifyMessageArrivingListener触发pullRequestHoldService通知pullRequestHoldServiceDefaultMessageStore.this.messageArrivingListener.arriving(dispatchRequest.getTopic(),dispatchRequest.getQueueId(), dispatchRequest.getConsumeQueueOffset() + 1,dispatchRequest.getTagsCode(), dispatchRequest.getStoreTimestamp(),dispatchRequest.getBitMap(), dispatchRequest.getPropertiesMap());}//设置重放偏起始移量加上当前消息大小this.reputFromOffset += size;//设置读取的大小加上当前消息大小readSize += size;//如果是SLAVE角色，那么存储数据的统计信息更新if (DefaultMessageStore.this.getMessageStoreConfig().getBrokerRole() == BrokerRole.SLAVE) {DefaultMessageStore.this.storeStatsService.getSinglePutMessageTopicTimesTotal(dispatchRequest.getTopic()).incrementAndGet();DefaultMessageStore.this.storeStatsService.getSinglePutMessageTopicSizeTotal(dispatchRequest.getTopic()).addAndGet(dispatchRequest.getMsgSize());}// 等于0表示读取到文件尾, 需要获取下一个文件的起始索引} else if (size == 0) {this.reputFromOffset = DefaultMessageStore.this.commitLog.rollNextFile(this.reputFromOffset);readSize = result.getSize();}} else if (!dispatchRequest.isSuccess()) {if (size > 0) {log.error("[BUG]read total count not equals msg total size. reputFromOffset={}", reputFromOffset);this.reputFromOffset += size;} else {doNext = false;// If user open the dledger pattern or the broker is master node,// it will not ignore the exception and fix the reputFromOffset variableif (DefaultMessageStore.this.getMessageStoreConfig().isEnableDLegerCommitLog() ||DefaultMessageStore.this.brokerConfig.getBrokerId() == MixAll.MASTER_ID) {log.error("[BUG]dispatch message to consume queue error, COMMITLOG OFFSET: {}",this.reputFromOffset);this.reputFromOffset += result.getSize() - readSize;}}}}} finally {result.release();}} else {doNext = false;}}}@Overridepublic void run() {DefaultMessageStore.log.info(this.getServiceName() + " service started");while (!this.isStopped()) {try {// 每1毫秒执行一次Thread.sleep(1);this.doReput();} catch (Exception e) {DefaultMessageStore.log.warn(this.getServiceName() + " service has exception. ", e);}}DefaultMessageStore.log.info(this.getServiceName() + " service end");}@Overridepublic String getServiceName() {return ReputMessageService.class.getSimpleName();}}

org.apache.rocketmq.store.DefaultMessageStore#doDispatch

遍历 dispatcherList 调用其dispatch方法, 参数为重放服务传入的消息, list内部的对象的类即为 CommitLogDispatcherBuildConsumeQueue和CommitLogDispatcherBuildIndex

都是用于构建consumequeue 和 indexFile的类

public void doDispatch(DispatchRequest req) {for (CommitLogDispatcher dispatcher : this.dispatcherList) {dispatcher.dispatch(req);}
}

org.apache.rocketmq.store.DefaultMessageStore.CommitLogDispatcherBuildConsumeQueue#dispatch

class CommitLogDispatcherBuildConsumeQueue implements CommitLogDispatcher {@Overridepublic void dispatch(DispatchRequest request) {final int tranType = MessageSysFlag.getTransactionValue(request.getSysFlag());switch (tranType) {case MessageSysFlag.TRANSACTION_NOT_TYPE:case MessageSysFlag.TRANSACTION_COMMIT_TYPE:// 将该条消息放入consumequeueDefaultMessageStore.this.putMessagePositionInfo(request);break;case MessageSysFlag.TRANSACTION_PREPARED_TYPE:case MessageSysFlag.TRANSACTION_ROLLBACK_TYPE:break;}}
}

public void putMessagePositionInfo(DispatchRequest dispatchRequest) {// 根据topic queueId找到对应的queueConsumeQueue cq = this.findConsumeQueue(dispatchRequest.getTopic(), dispatchRequest.getQueueId());cq.putMessagePositionInfoWrapper(dispatchRequest);
}

/*** 将消息信息追加到ConsumeQueue索引文件中*/
public void putMessagePositionInfoWrapper(DispatchRequest request) {// 最大重试次数final int maxRetries = 30;// 检查ConsumeQueue文件是否可写boolean canWrite = this.defaultMessageStore.getRunningFlags().isCQWriteable();for (int i = 0; i < maxRetries && canWrite; i++) {long tagsCode = request.getTagsCode();// 如果支持consumequeue扩展信息存储，默认为false，其存储在store/consumequeue_ext目录下if (isExtWriteEnable()) {ConsumeQueueExt.CqExtUnit cqExtUnit = new ConsumeQueueExt.CqExtUnit();cqExtUnit.setFilterBitMap(request.getBitMap());cqExtUnit.setMsgStoreTime(request.getStoreTimestamp());cqExtUnit.setTagsCode(request.getTagsCode());long extAddr = this.consumeQueueExt.put(cqExtUnit);if (isExtAddr(extAddr)) {tagsCode = extAddr;} else {log.warn("Save consume queue extend fail, So just save tagsCode! {}, topic:{}, queueId:{}, offset:{}", cqExtUnit,topic, queueId, request.getCommitLogOffset());}}// 消息写入到consumequeueboolean result = this.putMessagePositionInfo(request.getCommitLogOffset(),request.getMsgSize(), tagsCode, request.getConsumeQueueOffset());if (result) {if (this.defaultMessageStore.getMessageStoreConfig().getBrokerRole() == BrokerRole.SLAVE ||this.defaultMessageStore.getMessageStoreConfig().isEnableDLegerCommitLog()) {this.defaultMessageStore.getStoreCheckpoint().setPhysicMsgTimestamp(request.getStoreTimestamp());}// 修改最新consumeQueue文件的刷盘时间戳，单位毫秒this.defaultMessageStore.getStoreCheckpoint().setLogicsMsgTimestamp(request.getStoreTimestamp());return;} else {// XXX: warn and notify melog.warn("[BUG]put commit log position info to " + topic + ":" + queueId + " " + request.getCommitLogOffset()+ " failed, retry " + i + " times");try {Thread.sleep(1000);} catch (InterruptedException e) {log.warn("", e);}}}// XXX: warn and notify melog.error("[BUG]consume queue can not write, {} {}", this.topic, this.queueId);this.defaultMessageStore.getRunningFlags().makeLogicsQueueError();
}