From 31f3bed34748a2a0172eb28b15fabfbf339b3787 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Sat, 11 Jun 2022 20:39:16 +0800 Subject: [PATCH 01/78] support index cache --- include/common/tmsg.h | 1 + source/common/src/tmsg.c | 2 + source/libs/catalog/inc/catalogInt.h | 60 +- source/libs/catalog/src/catalog.c | 38 +- source/libs/catalog/src/ctgAsync.c | 44 +- source/libs/catalog/src/ctgCache.c | 810 +++++++++++++--------- source/libs/catalog/src/ctgDbg.c | 8 +- source/libs/catalog/src/ctgRemote.c | 5 +- source/libs/catalog/src/ctgUtil.c | 88 ++- source/libs/catalog/test/catalogTests.cpp | 4 +- source/libs/qcom/src/querymsg.c | 10 +- source/libs/qworker/src/qworker.c | 1 - source/libs/scalar/src/filter.c | 5 + source/util/src/tarray.c | 3 + 14 files changed, 657 insertions(+), 422 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 48a677a686..7c37484ef6 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -2513,6 +2513,7 @@ typedef struct { } STableIndexInfo; typedef struct { + int32_t version; SArray* pIndex; } STableIndexRsp; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index a7d4903696..3805a8b5b4 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -2437,6 +2437,7 @@ int32_t tSerializeSTableIndexRsp(void *buf, int32_t bufLen, const STableIndexRsp tEncoderInit(&encoder, buf, bufLen); if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeI32(&encoder, pRsp->version) < 0) return -1; int32_t num = taosArrayGetSize(pRsp->pIndex); if (tEncodeI32(&encoder, num) < 0) return -1; if (num > 0) { @@ -2471,6 +2472,7 @@ int32_t tDeserializeSTableIndexRsp(void *buf, int32_t bufLen, STableIndexRsp *pR tDecoderInit(&decoder, buf, bufLen); if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeI32(&decoder, &pRsp->version) < 0) return -1; int32_t num = 0; if (tDecodeI32(&decoder, &num) < 0) return -1; if (num > 0) { diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 1d56b6d5d4..ebd4d56aaf 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -57,6 +57,8 @@ enum { CTG_OP_DROP_TB_META, CTG_OP_UPDATE_USER, CTG_OP_UPDATE_VG_EPSET, + CTG_OP_UPDATE_TB_INDEX, + CTG_OP_DROP_TB_INDEX, CTG_OP_MAX }; @@ -128,19 +130,27 @@ typedef struct SCtgUserCtx { SUserAuthInfo user; } SCtgUserCtx; -typedef struct SCtgTbMetaCache { - SRWLatch stbLock; - SRWLatch metaLock; // RC between cache destroy and all other operations - SHashObj *metaCache; //key:tbname, value:STableMeta - SHashObj *stbCache; //key:suid, value:STableMeta* -} SCtgTbMetaCache; +typedef STableIndexRsp STableIndex; + +typedef struct SCtgTbCache { + SRWLatch metaLock; + STableMeta *pMeta; + SRWLatch indexLock; + STableIndex *pIndex; +} SCtgTbCache; + +typedef struct SCtgVgCache { + SRWLatch vgLock; + SDBVgInfo *vgInfo; +} SCtgVgCache; typedef struct SCtgDBCache { - SRWLatch vgLock; + SRWLatch dbLock; // RC between destroy tbCache/stbCache and all reads uint64_t dbId; int8_t deleted; - SDBVgInfo *vgInfo; - SCtgTbMetaCache tbCache; + SCtgVgCache vgCache; + SHashObj *tbCache; // key:tbname, value:SCtgTbCache + SHashObj *stbCache; // key:suid, value:STableMeta* } SCtgDBCache; typedef struct SCtgRentSlot { @@ -245,8 +255,10 @@ typedef struct SCtgCacheStat { uint64_t userNum; uint64_t vgHitNum; uint64_t vgMissNum; - uint64_t tblHitNum; - uint64_t tblMissNum; + uint64_t tbMetaHitNum; + uint64_t tbMetaMissNum; + uint64_t tbIndexHitNum; + uint64_t tbIndexMissNum; uint64_t userHitNum; uint64_t userMissNum; } SCtgCacheStat; @@ -268,10 +280,10 @@ typedef struct SCtgUpdateVgMsg { SDBVgInfo* dbInfo; } SCtgUpdateVgMsg; -typedef struct SCtgUpdateTblMsg { - SCatalog* pCtg; - STableMetaOutput* output; -} SCtgUpdateTblMsg; +typedef struct SCtgUpdateTbMetaMsg { + SCatalog* pCtg; + STableMetaOutput* pMeta; +} SCtgUpdateTbMetaMsg; typedef struct SCtgDropDBMsg { SCatalog* pCtg; @@ -305,6 +317,19 @@ typedef struct SCtgUpdateUserMsg { SGetUserAuthRsp userAuth; } SCtgUpdateUserMsg; +typedef struct SCtgUpdateTbIndexMsg { + SCatalog* pCtg; + char dbFName[TSDB_DB_FNAME_LEN]; + char tbName[TSDB_TABLE_NAME_LEN]; + STableIndex* pIndex; +} SCtgUpdateTbIndexMsg; + +typedef struct SCtgDropTbIndexMsg { + SCatalog* pCtg; + char dbFName[TSDB_DB_FNAME_LEN]; + char tbName[TSDB_TABLE_NAME_LEN]; +} SCtgDropTbIndexMsg; + typedef struct SCtgUpdateEpsetMsg { SCatalog* pCtg; char dbFName[TSDB_DB_FNAME_LEN]; @@ -465,8 +490,7 @@ int32_t ctgOpUpdateUser(SCtgCacheOperation *action); int32_t ctgOpUpdateEpset(SCtgCacheOperation *operation); int32_t ctgAcquireVgInfoFromCache(SCatalog* pCtg, const char *dbFName, SCtgDBCache **pCache); void ctgReleaseDBCache(SCatalog *pCtg, SCtgDBCache *dbCache); -void ctgReleaseVgInfo(SCtgDBCache *dbCache); -int32_t ctgAcquireVgInfoFromCache(SCatalog* pCtg, const char *dbFName, SCtgDBCache **pCache); +void ctgRUnlockVgInfo(SCtgDBCache *dbCache); int32_t ctgTbMetaExistInCache(SCatalog* pCtg, char *dbFName, char* tbName, int32_t *exist); int32_t ctgReadTbMetaFromCache(SCatalog* pCtg, SCtgTbMetaCtx* ctx, STableMeta** pTableMeta); int32_t ctgReadTbVerFromCache(SCatalog *pCtg, const SName *pTableName, int32_t *sver, int32_t *tver, int32_t *tbType, uint64_t *suid, char *stbName); @@ -479,6 +503,7 @@ int32_t ctgUpdateVgroupEnqueue(SCatalog* pCtg, const char *dbFName, int64_t dbId int32_t ctgUpdateTbMetaEnqueue(SCatalog* pCtg, STableMetaOutput *output, bool syncReq); int32_t ctgUpdateUserEnqueue(SCatalog* pCtg, SGetUserAuthRsp *pAuth, bool syncReq); int32_t ctgUpdateVgEpsetEnqueue(SCatalog* pCtg, char *dbFName, int32_t vgId, SEpSet* pEpSet); +int32_t ctgUpdateTbIndexEnqueue(SCatalog* pCtg, SName* pName, STableIndex *pIndex, bool syncOp); int32_t ctgMetaRentInit(SCtgRentMgmt *mgmt, uint32_t rentSec, int8_t type); int32_t ctgMetaRentAdd(SCtgRentMgmt *mgmt, void *meta, int64_t id, int32_t size); int32_t ctgMetaRentGet(SCtgRentMgmt *mgmt, void **res, uint32_t *num, int32_t size); @@ -521,6 +546,7 @@ void ctgFreeSTableMetaOutput(STableMetaOutput* pOutput); int32_t ctgUpdateMsgCtx(SCtgMsgCtx* pCtx, int32_t reqType, void* out, char* target); char *ctgTaskTypeStr(CTG_TASK_TYPE type); int32_t ctgUpdateSendTargetInfo(SMsgSendInfo *pMsgSendInfo, int32_t msgType, SCtgTask* pTask); +int32_t ctgCloneTableIndex(SArray* pIndex, SArray** pRes); extern SCatalogMgmt gCtgMgmt; diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index f1a81d1112..bac13bf476 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -97,8 +97,7 @@ int32_t ctgRefreshDBVgInfo(SCatalog* pCtg, SRequestConnInfo *pConn, const char* if (NULL != dbCache) { input.dbId = dbCache->dbId; - ctgReleaseVgInfo(dbCache); - ctgReleaseDBCache(pCtg, dbCache); + ctgReleaseVgInfoToCache(pCtg, dbCache); } input.vgVersion = CTG_DEFAULT_INVALID_VERSION; @@ -383,6 +382,18 @@ _return: return TSDB_CODE_SUCCESS; } +int32_t ctgGetTbIndex(SCatalog* pCtg, SRequestConnInfo *pConn, const SName* pTableName, SArray** pRes) { + CTG_ERR_RET(ctgReadTbIndexFromCache(pCtg, pTableName, pRes)); + if (*pRes) { + return TSDB_CODE_SUCCESS; + } + + CTG_ERR_RET(ctgGetTbIndexFromMnode(pCtg, pConn, (SName*)pTableName, pRes, NULL)); + + return TSDB_CODE_SUCCESS; +} + + int32_t ctgGetTbDistVgInfo(SCatalog* pCtg, SRequestConnInfo *pConn, SName* pTableName, SArray** pVgList) { STableMeta *tbMeta = NULL; int32_t code = 0; @@ -443,7 +454,7 @@ int32_t ctgGetTbDistVgInfo(SCatalog* pCtg, SRequestConnInfo *pConn, SName* pTabl _return: if (dbCache) { - ctgReleaseVgInfo(dbCache); + ctgRUnlockVgInfo(dbCache); ctgReleaseDBCache(pCtg, dbCache); } @@ -633,8 +644,7 @@ int32_t catalogGetDBVgVersion(SCatalog* pCtg, const char* dbFName, int32_t* vers *dbId = dbCache->dbId; *tableNum = dbCache->vgInfo->numOfTable; - ctgReleaseVgInfo(dbCache); - ctgReleaseDBCache(pCtg, dbCache); + ctgReleaseVgInfoToCache(pCtg, dbCache); ctgDebug("Got db vgVersion from cache, dbFName:%s, vgVersion:%d", dbFName, *version); @@ -659,7 +669,7 @@ int32_t catalogGetDBVgInfo(SCatalog* pCtg, SRequestConnInfo *pConn, const char* SDBVgInfo *vgInfo = NULL; CTG_ERR_JRET(ctgGetDBVgInfo(pCtg, pConn, dbFName, &dbCache, &vgInfo)); if (dbCache) { - vgHash = dbCache->vgInfo->vgHash; + vgHash = dbCache->vgCache.vgInfo->vgHash; } else { vgHash = vgInfo->vgHash; } @@ -672,7 +682,7 @@ int32_t catalogGetDBVgInfo(SCatalog* pCtg, SRequestConnInfo *pConn, const char* _return: if (dbCache) { - ctgReleaseVgInfo(dbCache); + ctgRUnlockVgInfo(dbCache); ctgReleaseDBCache(pCtg, dbCache); } @@ -940,7 +950,7 @@ int32_t catalogGetTableHashVgroup(SCatalog *pCtg, SRequestConnInfo *pConn, const _return: if (dbCache) { - ctgReleaseVgInfo(dbCache); + ctgRUnlockVgInfo(dbCache); ctgReleaseDBCache(pCtg, dbCache); } @@ -1143,7 +1153,17 @@ int32_t catalogGetTableIndex(SCatalog* pCtg, SRequestConnInfo *pConn, const SNam CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } - CTG_API_LEAVE(ctgGetTbIndexFromMnode(pCtg, pConn, (SName*)pTableName, pRes, NULL)); + int32_t code = 0; + CTG_ERR_JRET(ctgGetTbIndexFromMnode(pCtg, pConn, (SName*)pTableName, pRes, NULL)); + + SArray* pInfo = NULL; + CTG_ERR_JRET(ctgCloneTableIndex(*pRes, &pInfo)); + + CTG_ERR_JRET(ctgUpdateTbIndexEnqueue(pCtg, pTableName, pInfo, false)); + +_return: + + CTG_API_LEAVE(code); } diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index a96e4cec30..69eddd7ba6 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -344,6 +344,11 @@ int32_t ctgHandleForceUpdate(SCatalog* pCtg, SCtgJob *pJob, const SCatalogReq* p } } + for (int32_t i = 0; i < pJob->tbIndexNum; ++i) { + SName* name = taosArrayGet(pReq->pTableIndex, i); + ctgDropTbIndexEnqueue(pCtg, name, true); + } + return TSDB_CODE_SUCCESS; } @@ -524,7 +529,7 @@ int32_t ctgDumpTbIndexRes(SCtgTask* pTask) { } SMetaRes res = {.code = pTask->code, .pRes = pTask->res}; - taosArrayPush(pJob->jobRes.pTableHash, &res); + taosArrayPush(pJob->jobRes.pTableIndex, &res); return TSDB_CODE_SUCCESS; } @@ -687,8 +692,7 @@ int32_t ctgHandleGetTbMetaRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf * ctx->vgId = vgInfo.vgId; CTG_ERR_JRET(ctgGetTbMetaFromVnode(pCtg, pConn, ctx->pName, &vgInfo, NULL, pTask)); - ctgReleaseVgInfo(dbCache); - ctgReleaseDBCache(pCtg, dbCache); + ctgReleaseVgInfoToCache(pCtg, dbCache); } else { SBuildUseDBInput input = {0}; @@ -786,7 +790,7 @@ int32_t ctgHandleGetTbMetaRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf * _return: if (dbCache) { - ctgReleaseVgInfo(dbCache); + ctgRUnlockVgInfo(dbCache); ctgReleaseDBCache(pCtg, dbCache); } @@ -866,9 +870,16 @@ _return: int32_t ctgHandleGetTbIndexRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { int32_t code = 0; - CTG_ERR_JRET(ctgProcessRspMsg(pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); + CTG_ERR_JRET(ctgProcessRspMsg(&pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); - TSWAP(pTask->res, pTask->msgCtx.out); + STableIndex* pOut = (STableIndex*)pTask->msgCtx.out; + SArray* pInfo = NULL; + CTG_ERR_JRET(ctgCloneTableIndex(pOut->pIndex, &pInfo)); + pTask->res = pInfo; + pTask->msgCtx.out = NULL; + + SCtgTbIndexCtx* ctx = pTask->taskCtx; + CTG_ERR_JRET(ctgUpdateTbIndexEnqueue(pTask->pJob->pCtg, ctx->pName, pOut, false)); _return: @@ -1024,8 +1035,7 @@ int32_t ctgAsyncRefreshTbMeta(SCtgTask *pTask) { _return: if (dbCache) { - ctgReleaseVgInfo(dbCache); - ctgReleaseDBCache(pCtg, dbCache); + ctgReleaseVgInfoToCache(pCtg, dbCache); } CTG_RET(code); @@ -1070,8 +1080,7 @@ int32_t ctgLaunchGetDbVgTask(SCtgTask *pTask) { _return: if (dbCache) { - ctgReleaseVgInfo(dbCache); - ctgReleaseDBCache(pCtg, dbCache); + ctgReleaseVgInfoToCache(pCtg, dbCache); } CTG_RET(code); @@ -1105,8 +1114,7 @@ int32_t ctgLaunchGetTbHashTask(SCtgTask *pTask) { _return: if (dbCache) { - ctgReleaseVgInfo(dbCache); - ctgReleaseDBCache(pCtg, dbCache); + ctgReleaseVgInfoToCache(pCtg, dbCache); } CTG_RET(code); @@ -1117,6 +1125,15 @@ int32_t ctgLaunchGetTbIndexTask(SCtgTask *pTask) { SCatalog* pCtg = pTask->pJob->pCtg; SRequestConnInfo* pConn = &pTask->pJob->conn; SCtgTbIndexCtx* pCtx = (SCtgTbIndexCtx*)pTask->taskCtx; + SArray* pRes = NULL; + + CTG_ERR_RET(ctgReadTbIndexFromCache(pCtg, pCtx->pName, &pRes)); + if (pRes) { + pTask->res = pRes; + + CTG_ERR_RET(ctgHandleTaskEnd(pTask, 0)); + return TSDB_CODE_SUCCESS; + } CTG_ERR_RET(ctgGetTbIndexFromMnode(pCtg, pConn, pCtx->pName, NULL, pTask)); return TSDB_CODE_SUCCESS; @@ -1167,8 +1184,7 @@ int32_t ctgLaunchGetDbInfoTask(SCtgTask *pTask) { _return: if (dbCache) { - ctgReleaseVgInfo(dbCache); - ctgReleaseDBCache(pCtg, dbCache); + ctgReleaseVgInfoToCache(pCtg, dbCache); } CTG_RET(code); diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index add44d6a16..2b56eb6b49 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -59,6 +59,11 @@ SCtgOperation gCtgCacheOperation[CTG_OP_MAX] = { CTG_OP_UPDATE_VG_EPSET, "update epset", ctgOpUpdateEpset + }, + { + CTG_OP_UPDATE_TB_INDEX, + "update tbIndex", + ctgOpUpdateTbIndex } }; @@ -66,11 +71,11 @@ SCtgOperation gCtgCacheOperation[CTG_OP_MAX] = { -int32_t ctgAcquireVgInfo(SCatalog *pCtg, SCtgDBCache *dbCache, bool *inCache) { - CTG_LOCK(CTG_READ, &dbCache->vgLock); +int32_t ctgRLockVgInfo(SCatalog *pCtg, SCtgDBCache *dbCache, bool *inCache) { + CTG_LOCK(CTG_READ, &dbCache->vgCache.vgLock); if (dbCache->deleted) { - CTG_UNLOCK(CTG_READ, &dbCache->vgLock); + CTG_UNLOCK(CTG_READ, &dbCache->vgCache.vgLock); ctgDebug("db is dropping, dbId:%"PRIx64, dbCache->dbId); @@ -79,8 +84,8 @@ int32_t ctgAcquireVgInfo(SCatalog *pCtg, SCtgDBCache *dbCache, bool *inCache) { } - if (NULL == dbCache->vgInfo) { - CTG_UNLOCK(CTG_READ, &dbCache->vgLock); + if (NULL == dbCache->vgCache.vgInfo) { + CTG_UNLOCK(CTG_READ, &dbCache->vgCache.vgLock); *inCache = false; ctgDebug("db vgInfo is empty, dbId:%"PRIx64, dbCache->dbId); @@ -92,50 +97,47 @@ int32_t ctgAcquireVgInfo(SCatalog *pCtg, SCtgDBCache *dbCache, bool *inCache) { return TSDB_CODE_SUCCESS; } -int32_t ctgWAcquireVgInfo(SCatalog *pCtg, SCtgDBCache *dbCache) { - CTG_LOCK(CTG_WRITE, &dbCache->vgLock); +int32_t ctgWLockVgInfo(SCatalog *pCtg, SCtgDBCache *dbCache) { + CTG_LOCK(CTG_WRITE, &dbCache->vgCache.vgLock); if (dbCache->deleted) { ctgDebug("db is dropping, dbId:%"PRIx64, dbCache->dbId); - CTG_UNLOCK(CTG_WRITE, &dbCache->vgLock); + CTG_UNLOCK(CTG_WRITE, &dbCache->vgCache.vgLock); CTG_ERR_RET(TSDB_CODE_CTG_DB_DROPPED); } return TSDB_CODE_SUCCESS; } +void ctgRUnlockVgInfo(SCtgDBCache *dbCache) { + CTG_UNLOCK(CTG_READ, &dbCache->vgCache.vgLock); +} + +void ctgWUnlockVgInfo(SCtgDBCache *dbCache) { + CTG_UNLOCK(CTG_WRITE, &dbCache->vgCache.vgLock); +} + void ctgReleaseDBCache(SCatalog *pCtg, SCtgDBCache *dbCache) { - taosHashRelease(pCtg->dbCache, dbCache); + CTG_UNLOCK(CTG_READ, &dbCache->dbLock); } -void ctgReleaseVgInfo(SCtgDBCache *dbCache) { - CTG_UNLOCK(CTG_READ, &dbCache->vgLock); -} - -void ctgWReleaseVgInfo(SCtgDBCache *dbCache) { - CTG_UNLOCK(CTG_WRITE, &dbCache->vgLock); -} - - int32_t ctgAcquireDBCacheImpl(SCatalog* pCtg, const char *dbFName, SCtgDBCache **pCache, bool acquire) { char *p = strchr(dbFName, '.'); if (p && CTG_IS_SYS_DBNAME(p + 1)) { dbFName = p + 1; } - SCtgDBCache *dbCache = NULL; - if (acquire) { - dbCache = (SCtgDBCache *)taosHashAcquire(pCtg->dbCache, dbFName, strlen(dbFName)); - } else { - dbCache = (SCtgDBCache *)taosHashGet(pCtg->dbCache, dbFName, strlen(dbFName)); - } - + SCtgDBCache *dbCache = (SCtgDBCache *)taosHashGet(pCtg->dbCache, dbFName, strlen(dbFName)); if (NULL == dbCache) { *pCache = NULL; ctgDebug("db not in cache, dbFName:%s", dbFName); return TSDB_CODE_SUCCESS; } + if (acquire) { + CTG_LOCK(CTG_READ, &dbCache->dbLock); + } + if (dbCache->deleted) { if (acquire) { ctgReleaseDBCache(pCtg, dbCache); @@ -159,15 +161,35 @@ int32_t ctgGetDBCache(SCatalog* pCtg, const char *dbFName, SCtgDBCache **pCache) CTG_RET(ctgAcquireDBCacheImpl(pCtg, dbFName, pCache, false)); } +void ctgReleaseVgInfoToCache(SCatalog* pCtg, SCtgDBCache *dbCache) { + ctgRUnlockVgInfo(dbCache); + ctgReleaseDBCache(pCtg, dbCache); +} + +void ctgReleaseTbMetaToCache(SCatalog* pCtg, SCtgDBCache *dbCache, SCtgTbCache* pCache) { + if (pCache) { + CTG_UNLOCK(CTG_READ, &pCache->metaLock); + taosHashRelease(dbCache->tbCache, pCache); + } + + if (dbCache) { + ctgReleaseDBCache(pCtg, dbCache); + } +} + +void ctgReleaseTbIndexToCache(SCatalog* pCtg, SCtgDBCache *dbCache, SCtgTbCache* pCache) { + if (pCache) { + CTG_UNLOCK(CTG_READ, &pCache->indexLock); + taosHashRelease(dbCache->tbCache, pCache); + } + + if (dbCache) { + ctgReleaseDBCache(pCtg, dbCache); + } +} int32_t ctgAcquireVgInfoFromCache(SCatalog* pCtg, const char *dbFName, SCtgDBCache **pCache) { SCtgDBCache *dbCache = NULL; - - if (NULL == pCtg->dbCache) { - ctgDebug("empty db cache, dbFName:%s", dbFName); - goto _return; - } - ctgAcquireDBCache(pCtg, dbFName, &dbCache); if (NULL == dbCache) { ctgDebug("db %s not in cache", dbFName); @@ -175,7 +197,7 @@ int32_t ctgAcquireVgInfoFromCache(SCatalog* pCtg, const char *dbFName, SCtgDBCac } bool inCache = false; - ctgAcquireVgInfo(pCtg, dbCache, &inCache); + ctgRLockVgInfo(pCtg, dbCache, &inCache); if (!inCache) { ctgDebug("vgInfo of db %s not in cache", dbFName); goto _return; @@ -202,54 +224,107 @@ _return: return TSDB_CODE_SUCCESS; } -int32_t ctgTbMetaExistInCache(SCatalog* pCtg, char *dbFName, char* tbName, int32_t *exist) { - if (NULL == pCtg->dbCache) { - *exist = 0; - ctgWarn("empty db cache, dbFName:%s, tbName:%s", dbFName, tbName); - return TSDB_CODE_SUCCESS; - } - +int32_t ctgAcquireTbMetaFromCache(SCatalog* pCtg, char *dbFName, char* tbName, SCtgDBCache **pDb, SCtgTbCache** pTb) { SCtgDBCache *dbCache = NULL; ctgAcquireDBCache(pCtg, dbFName, &dbCache); if (NULL == dbCache) { - *exist = 0; - return TSDB_CODE_SUCCESS; + ctgDebug("db %s not in cache", dbFName); + goto _return; + } + + int32_t sz = 0; + SCtgTbCache* pCache = taosHashAcquire(dbCache->tbCache, tbName, strlen(tbName)); + if (NULL == pCache) { + ctgDebug("tb %s not in cache, dbFName:%s", tbName, dbFName); + goto _return; } - size_t sz = 0; - CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock); - STableMeta *tbMeta = taosHashGet(dbCache->tbCache.metaCache, tbName, strlen(tbName)); - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); - - if (NULL == tbMeta) { - ctgReleaseDBCache(pCtg, dbCache); - - *exist = 0; - ctgDebug("tbmeta not in cache, dbFName:%s, tbName:%s", dbFName, tbName); - return TSDB_CODE_SUCCESS; + CTG_LOCK(CTG_READ, &pCache->metaLock); + if (NULL == pCache->pMeta) { + ctgDebug("tb %s meta not in cache, dbFName:%s", tbName, dbFName); + goto _return; } - *exist = 1; + *pDb = dbCache; + *pTb = pCache; - ctgReleaseDBCache(pCtg, dbCache); + ctgDebug("tb %s meta got in cache, dbFName:%s", tbName, dbFName); - ctgDebug("tbmeta is in cache, dbFName:%s, tbName:%s", dbFName, tbName); + CTG_CACHE_STAT_INC(tbMetaHitNum, 1); + + return TSDB_CODE_SUCCESS; + +_return: + + ctgReleaseTbMetaToCache(pCtg, dbCache, pCache); + + CTG_CACHE_STAT_INC(tbMetaMissNum, 1); + + return TSDB_CODE_SUCCESS; +} + +int32_t ctgAcquireTbIndexFromCache(SCatalog* pCtg, char *dbFName, char* tbName, SCtgDBCache **pDb, SCtgTbCache** pTb) { + SCtgDBCache *dbCache = NULL; + ctgAcquireDBCache(pCtg, dbFName, &dbCache); + if (NULL == dbCache) { + ctgDebug("db %s not in cache", dbFName); + goto _return; + } + + int32_t sz = 0; + SCtgTbCache* pCache = taosHashAcquire(dbCache->tbCache, tbName, strlen(tbName)); + if (NULL == pCache) { + ctgDebug("tb %s not in cache, dbFName:%s", tbName, dbFName); + goto _return; + } + + CTG_LOCK(CTG_READ, &pCache->indexLock); + if (NULL == pCache->pIndex) { + ctgDebug("tb %s index not in cache, dbFName:%s", tbName, dbFName); + goto _return; + } + + *pDb = dbCache; + *pTb = pCache; + + ctgDebug("tb %s index got in cache, dbFName:%s", tbName, dbFName); + + CTG_CACHE_STAT_INC(tbIndexHitNum, 1); + + return TSDB_CODE_SUCCESS; + +_return: + + ctgReleaseTbIndexToCache(pCtg, dbCache, pCache); + + CTG_CACHE_STAT_INC(tbIndexMissNum, 1); return TSDB_CODE_SUCCESS; } +int32_t ctgTbMetaExistInCache(SCatalog* pCtg, char *dbFName, char* tbName, int32_t *exist) { + SCtgDBCache *dbCache = NULL; + ctgAcquireTbMetaFromCache(pCtg, dbFName, tbName, &dbCache, &tbCache); + if (NULL == tbCache) { + ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); + + *exist = 0; + return TSDB_CODE_SUCCESS; + } + + *exist = 1; + ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); + + return TSDB_CODE_SUCCESS; +} + int32_t ctgReadTbMetaFromCache(SCatalog* pCtg, SCtgTbMetaCtx* ctx, STableMeta** pTableMeta) { int32_t code = 0; SCtgDBCache *dbCache = NULL; - + SCtgTbCache *tbCache = NULL; *pTableMeta = NULL; - if (NULL == pCtg->dbCache) { - ctgDebug("empty tbmeta cache, tbName:%s", ctx->pName->tname); - return TSDB_CODE_SUCCESS; - } - char dbFName[TSDB_DB_FNAME_LEN] = {0}; if (CTG_FLAG_IS_SYS_DB(ctx->flag)) { strcpy(dbFName, ctx->pName->dbname); @@ -257,78 +332,53 @@ int32_t ctgReadTbMetaFromCache(SCatalog* pCtg, SCtgTbMetaCtx* ctx, STableMeta** tNameGetFullDbName(ctx->pName, dbFName); } - ctgAcquireDBCache(pCtg, dbFName, &dbCache); - if (NULL == dbCache) { - ctgDebug("db %d.%s not in cache", ctx->pName->acctId, ctx->pName->dbname); - return TSDB_CODE_SUCCESS; - } - - int32_t sz = 0; - CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock); - taosHashGetDup_m(dbCache->tbCache.metaCache, ctx->pName->tname, strlen(ctx->pName->tname), (void **)pTableMeta, &sz); - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); - - if (NULL == *pTableMeta) { - ctgReleaseDBCache(pCtg, dbCache); - ctgDebug("tbl not in cache, dbFName:%s, tbName:%s", dbFName, ctx->pName->tname); + ctgAcquireTbMetaFromCache(pCtg, dbFName, ctx->pName->tname, &dbCache, &tbCache); + if (NULL == tbCache) { + ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); return TSDB_CODE_SUCCESS; } - STableMeta* tbMeta = *pTableMeta; + STableMeta* tbMeta = tbCache->pMeta; ctx->tbInfo.inCache = true; ctx->tbInfo.dbId = dbCache->dbId; ctx->tbInfo.suid = tbMeta->suid; ctx->tbInfo.tbType = tbMeta->tableType; if (tbMeta->tableType != TSDB_CHILD_TABLE) { - ctgReleaseDBCache(pCtg, dbCache); - ctgDebug("Got meta from cache, type:%d, dbFName:%s, tbName:%s", tbMeta->tableType, dbFName, ctx->pName->tname); - - CTG_CACHE_STAT_INC(tblHitNum, 1); + ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); + ctgDebug("Got tb %s meta from cache, type:%d, dbFName:%s", ctx->pName->tname, tbMeta->tableType, dbFName); return TSDB_CODE_SUCCESS; } - CTG_LOCK(CTG_READ, &dbCache->tbCache.stbLock); - - STableMeta **stbMeta = taosHashGet(dbCache->tbCache.stbCache, &tbMeta->suid, sizeof(tbMeta->suid)); + STableMeta **stbMeta = taosHashGet(dbCache->stbCache, &tbMeta->suid, sizeof(tbMeta->suid)); if (NULL == stbMeta || NULL == *stbMeta) { - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); ctgError("stb not in stbCache, suid:%"PRIx64, tbMeta->suid); goto _return; } if ((*stbMeta)->suid != tbMeta->suid) { - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); - ctgError("stable suid in stbCache mis-match, expected suid:%"PRIx64 ",actual suid:%"PRIx64, tbMeta->suid, (*stbMeta)->suid); + ctgError("stb suid %" PRIx64 " in stbCache mis-match, expected suid:%"PRIx64 , (*stbMeta)->suid, tbMeta->suid); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } int32_t metaSize = CTG_META_SIZE(*stbMeta); *pTableMeta = taosMemoryRealloc(*pTableMeta, metaSize); if (NULL == *pTableMeta) { - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); - ctgError("realloc size[%d] failed", metaSize); CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); } memcpy(&(*pTableMeta)->sversion, &(*stbMeta)->sversion, metaSize - sizeof(SCTableMeta)); - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); + ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); - ctgReleaseDBCache(pCtg, dbCache); - - CTG_CACHE_STAT_INC(tblHitNum, 1); - - ctgDebug("Got tbmeta from cache, dbFName:%s, tbName:%s", dbFName, ctx->pName->tname); + ctgDebug("Got tb %s meta from cache, dbFName:%s", ctx->pName->tname, dbFName); return TSDB_CODE_SUCCESS; _return: - ctgReleaseDBCache(pCtg, dbCache); + ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); taosMemoryFreeClear(*pTableMeta); - - CTG_CACHE_STAT_INC(tblMissNum, 1); CTG_RET(code); } @@ -338,62 +388,42 @@ int32_t ctgReadTbVerFromCache(SCatalog *pCtg, const SName *pTableName, int32_t * *sver = -1; *tver = -1; - if (NULL == pCtg->dbCache) { - ctgDebug("empty tbmeta cache, tbName:%s", pTableName->tname); - return TSDB_CODE_SUCCESS; - } - SCtgDBCache *dbCache = NULL; + SCtgTbCache *tbCache = NULL; char dbFName[TSDB_DB_FNAME_LEN] = {0}; tNameGetFullDbName(pTableName, dbFName); - ctgAcquireDBCache(pCtg, dbFName, &dbCache); - if (NULL == dbCache) { - ctgDebug("db %s not in cache", pTableName->tname); + ctgAcquireTbMetaFromCache(pCtg, dbFName, pTableName->tname, &dbCache, &tbCache); + if (NULL == tbCache) { + ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); return TSDB_CODE_SUCCESS; } - CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock); - STableMeta *tbMeta = taosHashGet(dbCache->tbCache.metaCache, pTableName->tname, strlen(pTableName->tname)); - if (tbMeta) { - *tbType = tbMeta->tableType; - *suid = tbMeta->suid; - if (*tbType != TSDB_CHILD_TABLE) { - *sver = tbMeta->sversion; - *tver = tbMeta->tversion; - } - } - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); - - if (NULL == tbMeta) { - ctgReleaseDBCache(pCtg, dbCache); - return TSDB_CODE_SUCCESS; - } + STableMeta* tbMeta = tbCache->pMeta; + *tbType = tbMeta->tableType; + *suid = tbMeta->suid; if (*tbType != TSDB_CHILD_TABLE) { - ctgReleaseDBCache(pCtg, dbCache); - ctgDebug("Got sver %d tver %d from cache, type:%d, dbFName:%s, tbName:%s", *sver, *tver, *tbType, dbFName, pTableName->tname); + *sver = tbMeta->sversion; + *tver = tbMeta->tversion; + ctgDebug("Got tb %s ver from cache, dbFName:%s, tbType:%d, sver:%d, tver:%d, suid:%" PRIx64, + pTableName->tname, dbFName, *tbType, *sver, *tver, *suid); + + ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); return TSDB_CODE_SUCCESS; } - ctgDebug("Got subtable meta from cache, dbFName:%s, tbName:%s, suid:%" PRIx64, dbFName, pTableName->tname, *suid); - - CTG_LOCK(CTG_READ, &dbCache->tbCache.stbLock); - - STableMeta **stbMeta = taosHashGet(dbCache->tbCache.stbCache, suid, sizeof(*suid)); + STableMeta **stbMeta = taosHashGet(dbCache->stbCache, suid, sizeof(*suid)); if (NULL == stbMeta || NULL == *stbMeta) { - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); - ctgReleaseDBCache(pCtg, dbCache); + ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); ctgDebug("stb not in stbCache, suid:%" PRIx64, *suid); return TSDB_CODE_SUCCESS; } if ((*stbMeta)->suid != *suid) { - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); - ctgReleaseDBCache(pCtg, dbCache); - ctgError("stable suid in stbCache mis-match, expected suid:%" PRIx64 ",actual suid:%" PRIx64, *suid, - (*stbMeta)->suid); + ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); + ctgError("stb suid %" PRIx64 " in stbCache mis-match, expected suid:%" PRIx64 , (*stbMeta)->suid, *suid); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } @@ -406,52 +436,55 @@ int32_t ctgReadTbVerFromCache(SCatalog *pCtg, const SName *pTableName, int32_t * *sver = (*stbMeta)->sversion; *tver = (*stbMeta)->tversion; - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); + ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); - ctgReleaseDBCache(pCtg, dbCache); - - ctgDebug("Got sver %d tver %d from cache, type:%d, dbFName:%s, tbName:%s", *sver, *tver, *tbType, dbFName, pTableName->tname); + ctgDebug("Got tb %s sver %d tver %d from cache, type:%d, dbFName:%s", pTableName->tname, *sver, *tver, *tbType, dbFName); return TSDB_CODE_SUCCESS; } int32_t ctgReadTbTypeFromCache(SCatalog* pCtg, const char* dbFName, const char *tableName, int32_t *tbType) { - if (NULL == pCtg->dbCache) { - ctgWarn("empty db cache, dbFName:%s, tbName:%s", dbFName, tableName); - return TSDB_CODE_SUCCESS; - } - SCtgDBCache *dbCache = NULL; - ctgAcquireDBCache(pCtg, dbFName, &dbCache); - if (NULL == dbCache) { + SCtgTbCache *tbCache = NULL; + ctgAcquireTbMetaFromCache(pCtg, dbFName, tableName, &dbCache, &tbCache); + if (NULL == tbCache) { + ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); return TSDB_CODE_SUCCESS; } - CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock); - STableMeta *pTableMeta = (STableMeta *)taosHashAcquire(dbCache->tbCache.metaCache, tableName, strlen(tableName)); + *tbType = tbCache->pMeta->tableType; + ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); - if (NULL == pTableMeta) { - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); - ctgWarn("tbl not in cache, dbFName:%s, tbName:%s", dbFName, tableName); - ctgReleaseDBCache(pCtg, dbCache); - - return TSDB_CODE_SUCCESS; - } - - *tbType = atomic_load_8(&pTableMeta->tableType); - - taosHashRelease(dbCache->tbCache.metaCache, pTableMeta); - - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); - - ctgReleaseDBCache(pCtg, dbCache); - - ctgDebug("Got tbtype from cache, dbFName:%s, tbName:%s, type:%d", dbFName, tableName, *tbType); + ctgDebug("Got tb %s tbType %d from cache, dbFName:%s", tableName, *tbType, dbFName); return TSDB_CODE_SUCCESS; } +int32_t ctgReadTbIndexFromCache(SCatalog* pCtg, const SName* pTableName, SArray** pRes) { + int32_t code = 0; + SCtgDBCache *dbCache = NULL; + SCtgTbCache *tbCache = NULL; + char dbFName[TSDB_DB_FNAME_LEN] = {0}; + tNameGetFullDbName(pTableName, dbFName); + + *pRes = NULL; + + ctgAcquireTbIndexFromCache(pCtg, dbFName, pTableName->tname, &dbCache, &tbCache); + if (NULL == tbCache) { + ctgReleaseTbIndexToCache(pCtg, dbCache, tbCache); + return TSDB_CODE_SUCCESS; + } + + CTG_ERR_JRET(ctgCloneTableIndex(tbCache->pIndex->pIndex, pRes)); + +_return: + + ctgReleaseTbIndexToCache(pCtg, dbCache, tbCache); + + CTG_RET(code); +} + int32_t ctgChkAuthFromCache(SCatalog* pCtg, const char* user, const char* dbFName, AUTH_TYPE type, bool *inCache, bool *pass) { if (NULL == pCtg->userCache) { ctgDebug("empty user auth cache, user:%s", user); @@ -718,9 +751,9 @@ int32_t ctgUpdateTbMetaEnqueue(SCatalog* pCtg, STableMetaOutput *output, bool sy op->opId = CTG_OP_UPDATE_TB_META; op->syncOp = syncOp; - SCtgUpdateTblMsg *msg = taosMemoryMalloc(sizeof(SCtgUpdateTblMsg)); + SCtgUpdateTbMetaMsg *msg = taosMemoryMalloc(sizeof(SCtgUpdateTbMetaMsg)); if (NULL == msg) { - ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateTblMsg)); + ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateTbMetaMsg)); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } @@ -730,7 +763,7 @@ int32_t ctgUpdateTbMetaEnqueue(SCatalog* pCtg, STableMetaOutput *output, bool sy } msg->pCtg = pCtg; - msg->output = output; + msg->pMeta = output; op->data = msg; @@ -805,6 +838,68 @@ _return: CTG_RET(code); } +int32_t ctgUpdateTbIndexEnqueue(SCatalog* pCtg, SName* pName, STableIndex *pIndex, bool syncOp) { + int32_t code = 0; + SCtgCacheOperation *op = taosMemoryCalloc(1, sizeof(SCtgCacheOperation)); + op->opId = CTG_OP_UPDATE_TB_INDEX; + op->syncOp = syncOp; + + SCtgUpdateTbIndexMsg *msg = taosMemoryMalloc(sizeof(SCtgUpdateTbIndexMsg)); + if (NULL == msg) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateTbIndexMsg)); + CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); + } + + msg->pCtg = pCtg; + msg->pIndex = pIndex; + tNameGetFullDbName(pName, msg->dbFName); + strcpy(msg->tbName, pName->tname); + + op->data = msg; + + CTG_ERR_JRET(ctgEnqueue(pCtg, op)); + + return TSDB_CODE_SUCCESS; + +_return: + + taosArrayDestroyEx(pIndex, tFreeSTableIndexInfo); + taosMemoryFreeClear(msg); + + CTG_RET(code); +} + +int32_t ctgDropTbIndexEnqueue(SCatalog* pCtg, SName* pName, bool syncOp) { + int32_t code = 0; + SCtgCacheOperation *op = taosMemoryCalloc(1, sizeof(SCtgCacheOperation)); + op->opId = CTG_OP_DROP_TB_INDEX; + op->syncOp = syncOp; + + SCtgDropTbIndexMsg *msg = taosMemoryMalloc(sizeof(SCtgDropTbIndexMsg)); + if (NULL == msg) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgDropTbIndexMsg)); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } + + msg->pCtg = pCtg; + tNameGetFullDbName(pName, msg->dbFName); + strcpy(msg->tbName, pName->tname); + + op->data = msg; + + CTG_ERR_JRET(ctgEnqueue(pCtg, op)); + + return TSDB_CODE_SUCCESS; + +_return: + + taosArrayDestroyEx(pIndex, tFreeSTableIndexInfo); + taosMemoryFreeClear(msg); + + CTG_RET(code); +} + + int32_t ctgMetaRentInit(SCtgRentMgmt *mgmt, uint32_t rentSec, int8_t type) { mgmt->slotRIdx = 0; mgmt->slotNum = rentSec / CTG_RENT_SLOT_SECOND; @@ -1006,14 +1101,14 @@ int32_t ctgAddNewDBCache(SCatalog *pCtg, const char *dbFName, uint64_t dbId) { SCtgDBCache newDBCache = {0}; newDBCache.dbId = dbId; - newDBCache.tbCache.metaCache = taosHashInit(gCtgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); - if (NULL == newDBCache.tbCache.metaCache) { + newDBCache.tbCache = taosHashInit(gCtgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + if (NULL == newDBCache.tbCache) { ctgError("taosHashInit %d metaCache failed", gCtgMgmt.cfg.maxTblCacheNum); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } - newDBCache.tbCache.stbCache = taosHashInit(gCtgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), true, HASH_ENTRY_LOCK); - if (NULL == newDBCache.tbCache.stbCache) { + newDBCache.stbCache = taosHashInit(gCtgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), true, HASH_ENTRY_LOCK); + if (NULL == newDBCache.stbCache) { ctgError("taosHashInit %d stbCache failed", gCtgMgmt.cfg.maxTblCacheNum); CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); } @@ -1050,22 +1145,22 @@ _return: } -void ctgRemoveStbRent(SCatalog* pCtg, SCtgTbMetaCache *cache) { - CTG_LOCK(CTG_WRITE, &cache->stbLock); - if (cache->stbCache) { - void *pIter = taosHashIterate(cache->stbCache, NULL); - while (pIter) { - uint64_t *suid = NULL; - suid = taosHashGetKey(pIter, NULL); - - if (TSDB_CODE_SUCCESS == ctgMetaRentRemove(&pCtg->stbRent, *suid, ctgStbVersionSortCompare, ctgStbVersionSearchCompare)) { - ctgDebug("stb removed from rent, suid:%"PRIx64, *suid); - } - - pIter = taosHashIterate(cache->stbCache, pIter); - } +void ctgRemoveStbRent(SCatalog* pCtg, SCtgDBCache *dbCache) { + if (NULL == dbCache->stbCache) { + return; + } + + void *pIter = taosHashIterate(dbCache->stbCache, NULL); + while (pIter) { + uint64_t *suid = NULL; + suid = taosHashGetKey(pIter, NULL); + + if (TSDB_CODE_SUCCESS == ctgMetaRentRemove(&pCtg->stbRent, *suid, ctgStbVersionSortCompare, ctgStbVersionSearchCompare)) { + ctgDebug("stb removed from rent, suid:%"PRIx64, *suid); + } + + pIter = taosHashIterate(dbCache->stbCache, pIter); } - CTG_UNLOCK(CTG_WRITE, &cache->stbLock); } @@ -1074,15 +1169,16 @@ int32_t ctgRemoveDBFromCache(SCatalog* pCtg, SCtgDBCache *dbCache, const char* d ctgInfo("start to remove db from cache, dbFName:%s, dbId:%"PRIx64, dbFName, dbCache->dbId); + CTG_LOCK(CTG_WRITE, &dbCache->dbLock); + atomic_store_8(&dbCache->deleted, 1); - - ctgRemoveStbRent(pCtg, &dbCache->tbCache); - + ctgRemoveStbRent(pCtg, &dbCache); ctgFreeDbCache(dbCache); - CTG_ERR_RET(ctgMetaRentRemove(&pCtg->dbRent, dbCache->dbId, ctgDbVgVersionSortCompare, ctgDbVgVersionSearchCompare)); - - ctgDebug("db removed from rent, dbFName:%s, dbId:%"PRIx64, dbFName, dbCache->dbId); + CTG_UNLOCK(CTG_WRITE, &dbCache->dbLock); + + CTG_ERR_RET(ctgMetaRentRemove(&pCtg->dbRent, dbId, ctgDbVgVersionSortCompare, ctgDbVgVersionSearchCompare)); + ctgDebug("db removed from rent, dbFName:%s, dbId:%"PRIx64, dbFName, dbId); if (taosHashRemove(pCtg->dbCache, dbFName, strlen(dbFName))) { ctgInfo("taosHashRemove from dbCache failed, may be removed, dbFName:%s", dbFName); @@ -1090,7 +1186,6 @@ int32_t ctgRemoveDBFromCache(SCatalog* pCtg, SCtgDBCache *dbCache, const char* d } CTG_CACHE_STAT_DEC(dbNum, 1); - ctgInfo("db removed from cache, dbFName:%s, dbId:%"PRIx64, dbFName, dbId); return TSDB_CODE_SUCCESS; @@ -1138,120 +1233,54 @@ int32_t ctgGetAddDBCache(SCatalog* pCtg, const char *dbFName, uint64_t dbId, SCt return TSDB_CODE_SUCCESS; } - -int32_t ctgWriteDBVgInfoToCache(SCatalog* pCtg, const char* dbFName, uint64_t dbId, SDBVgInfo** pDbInfo) { - int32_t code = 0; - SDBVgInfo* dbInfo = *pDbInfo; - - if (NULL == dbInfo->vgHash) { - return TSDB_CODE_SUCCESS; - } - - if (dbInfo->vgVersion < 0 || taosHashGetSize(dbInfo->vgHash) <= 0) { - ctgError("invalid db vgInfo, dbFName:%s, vgHash:%p, vgVersion:%d, vgHashSize:%d", - dbFName, dbInfo->vgHash, dbInfo->vgVersion, taosHashGetSize(dbInfo->vgHash)); - CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); - } - - bool newAdded = false; - SDbVgVersion vgVersion = {.dbId = dbId, .vgVersion = dbInfo->vgVersion, .numOfTable = dbInfo->numOfTable}; - - SCtgDBCache *dbCache = NULL; - CTG_ERR_RET(ctgGetAddDBCache(pCtg, dbFName, dbId, &dbCache)); - if (NULL == dbCache) { - ctgInfo("conflict db update, ignore this update, dbFName:%s, dbId:%"PRIx64, dbFName, dbId); - CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); - } - - SDBVgInfo *vgInfo = NULL; - CTG_ERR_RET(ctgWAcquireVgInfo(pCtg, dbCache)); - - if (dbCache->vgInfo) { - if (dbInfo->vgVersion < dbCache->vgInfo->vgVersion) { - ctgDebug("db vgVersion is old, dbFName:%s, vgVersion:%d, currentVersion:%d", dbFName, dbInfo->vgVersion, dbCache->vgInfo->vgVersion); - ctgWReleaseVgInfo(dbCache); - - return TSDB_CODE_SUCCESS; - } - - if (dbInfo->vgVersion == dbCache->vgInfo->vgVersion && dbInfo->numOfTable == dbCache->vgInfo->numOfTable) { - ctgDebug("no new db vgVersion or numOfTable, dbFName:%s, vgVersion:%d, numOfTable:%d", dbFName, dbInfo->vgVersion, dbInfo->numOfTable); - ctgWReleaseVgInfo(dbCache); - - return TSDB_CODE_SUCCESS; - } - - ctgFreeVgInfo(dbCache->vgInfo); - } - - dbCache->vgInfo = dbInfo; - - *pDbInfo = NULL; - - ctgDebug("db vgInfo updated, dbFName:%s, vgVersion:%d, dbId:%"PRIx64, dbFName, vgVersion.vgVersion, vgVersion.dbId); - - ctgWReleaseVgInfo(dbCache); - - dbCache = NULL; - - strncpy(vgVersion.dbFName, dbFName, sizeof(vgVersion.dbFName)); - CTG_ERR_RET(ctgMetaRentUpdate(&pCtg->dbRent, &vgVersion, vgVersion.dbId, sizeof(SDbVgVersion), ctgDbVgVersionSortCompare, ctgDbVgVersionSearchCompare)); - - CTG_RET(code); -} - - int32_t ctgWriteTbMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFName, uint64_t dbId, char *tbName, STableMeta *meta, int32_t metaSize) { - SCtgTbMetaCache *tbCache = &dbCache->tbCache; - - CTG_LOCK(CTG_READ, &tbCache->metaLock); - if (dbCache->deleted || NULL == tbCache->metaCache || NULL == tbCache->stbCache) { - CTG_UNLOCK(CTG_READ, &tbCache->metaLock); + if (NULL == dbCache->tbCache || NULL == dbCache->stbCache) { + taosMemoryFree(meta); ctgError("db is dropping, dbId:%"PRIx64, dbCache->dbId); CTG_ERR_RET(TSDB_CODE_CTG_DB_DROPPED); } + bool isStb = meta->tableType == TSDB_SUPER_TABLE; + SCtgTbCache* pCache = taosHashGet(dbCache->tbCache, tbName, strlen(tbName)); + STableMeta *orig = (pCache ? pCache->pMeta : NULL); int8_t origType = 0; uint64_t origSuid = 0; - bool isStb = meta->tableType == TSDB_SUPER_TABLE; - STableMeta *orig = taosHashGet(tbCache->metaCache, tbName, strlen(tbName)); + if (orig) { origType = orig->tableType; if (origType == meta->tableType && orig->uid == meta->uid && orig->sversion >= meta->sversion && orig->tversion >= meta->tversion) { - CTG_UNLOCK(CTG_READ, &tbCache->metaLock); + taosMemoryFree(meta); + ctgDebug("ignore table %s meta update", tbName); return TSDB_CODE_SUCCESS; } if (origType == TSDB_SUPER_TABLE) { - CTG_LOCK(CTG_WRITE, &tbCache->stbLock); - if (taosHashRemove(tbCache->stbCache, &orig->suid, sizeof(orig->suid))) { + if (taosHashRemove(dbCache->stbCache, &orig->suid, sizeof(orig->suid))) { ctgError("stb not exist in stbCache, dbFName:%s, stb:%s, suid:%"PRIx64, dbFName, tbName, orig->suid); } else { CTG_CACHE_STAT_DEC(stblNum, 1); + ctgDebug("stb removed from stbCache, dbFName:%s, stb:%s, suid:%"PRIx64, dbFName, tbName, orig->suid); } - CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock); - - ctgDebug("stb removed from stbCache, dbFName:%s, stb:%s, suid:%"PRIx64, dbFName, tbName, orig->suid); ctgMetaRentRemove(&pCtg->stbRent, orig->suid, ctgStbVersionSortCompare, ctgStbVersionSearchCompare); - origSuid = orig->suid; } } - if (isStb) { - CTG_LOCK(CTG_WRITE, &tbCache->stbLock); - } - - if (taosHashPut(tbCache->metaCache, tbName, strlen(tbName), meta, metaSize) != 0) { - if (isStb) { - CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock); + if (NULL == pCache) { + SCtgTbCache cache = {0}; + cache.pMeta = meta; + if (taosHashPut(dbCache->tbCache, tbName, strlen(tbName), &cache, sizeof(SCtgTbCache)) != 0) { + taosMemoryFree(meta); + ctgError("taosHashPut new tbCache failed, dbFName:%s, tbName:%s, tbType:%d", dbFName, tbName, meta->tableType); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } - CTG_UNLOCK(CTG_READ, &tbCache->metaLock); - ctgError("taosHashPut tbmeta to cache failed, dbFName:%s, tbName:%s, tbType:%d", dbFName, tbName, meta->tableType); - CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + pCache = taosHashGet(dbCache->tbCache, tbName, strlen(tbName)); + } else { + taosMemoryFree(pCache->pMeta); + pCache->pMeta = meta; } if (NULL == orig) { @@ -1262,23 +1291,15 @@ int32_t ctgWriteTbMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFNam ctgdShowTableMeta(pCtg, tbName, meta); if (!isStb) { - CTG_UNLOCK(CTG_READ, &tbCache->metaLock); return TSDB_CODE_SUCCESS; } - STableMeta *tbMeta = taosHashGet(tbCache->metaCache, tbName, strlen(tbName)); - if (taosHashPut(tbCache->stbCache, &meta->suid, sizeof(meta->suid), &tbMeta, POINTER_BYTES) != 0) { - CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock); - CTG_UNLOCK(CTG_READ, &tbCache->metaLock); - ctgError("taosHashPut stable to stable cache failed, suid:%"PRIx64, meta->suid); + if (origSuid != meta->suid && taosHashPut(dbCache->stbCache, &meta->suid, sizeof(meta->suid), &pCache, POINTER_BYTES) != 0) { + ctgError("taosHashPut to stable cache failed, suid:%"PRIx64, meta->suid); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } CTG_CACHE_STAT_INC(stblNum, 1); - - CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock); - - CTG_UNLOCK(CTG_READ, &tbCache->metaLock); ctgDebug("stb updated to stbCache, dbFName:%s, tbName:%s, tbType:%d", dbFName, tbName, meta->tableType); @@ -1290,6 +1311,43 @@ int32_t ctgWriteTbMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFNam return TSDB_CODE_SUCCESS; } +int32_t ctgWriteTbIndexToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *tbName, STableIndex **index) { + if (NULL == dbCache->tbCache) { + taosMemoryFreeClear(*index); + ctgError("db is dropping, dbId:%"PRIx64, dbCache->dbId); + CTG_ERR_RET(TSDB_CODE_CTG_DB_DROPPED); + } + + STableIndex* pIndex = *index; + SCtgTbCache* pCache = taosHashGet(dbCache->tbCache, tbName, strlen(tbName)); + if (NULL == pCache) { + SCtgTbCache cache = {0}; + cache.pIndex = pIndex; + + if (taosHashPut(table->tbCache, tbName, strlen(tbName), &cache, sizeof(cache)) != 0) { + taosMemoryFreeClear(*index); + ctgError("taosHashPut new tbCache failed, tbName:%s", tbName); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } + + *index = NULL; + ctgDebug("table %s index updated to cache, ver:%d, num:%d", tbName, pIndex->version, taosArrayGetSize(pIndex->pIndex)); + return TSDB_CODE_SUCCESS; + } + + if (pCache->pIndex) { + taosArrayDestroyEx(pCache->pIndex->pIndex, tFreeSTableIndexInfo); + taosMemoryFreeClear(pCache->pIndex); + } + + pCache->pIndex = pIndex; + *index = NULL; + + ctgDebug("table %s index updated to cache, ver:%d, num:%d", tbName, pIndex->version, taosArrayGetSize(pIndex->pIndex)); + + return TSDB_CODE_SUCCESS; +} + int32_t ctgUpdateTbMetaToCache(SCatalog* pCtg, STableMetaOutput* pOut, bool syncReq) { STableMetaOutput* pOutput = NULL; int32_t code = 0; @@ -1305,12 +1363,66 @@ _return: CTG_RET(code); } - int32_t ctgOpUpdateVgroup(SCtgCacheOperation *operation) { int32_t code = 0; SCtgUpdateVgMsg *msg = operation->data; + SDBVgInfo* dbInfo = msg->dbInfo; + char* dbFName = msg->dbFName; - CTG_ERR_JRET(ctgWriteDBVgInfoToCache(msg->pCtg, msg->dbFName, msg->dbId, &msg->dbInfo)); + if (NULL == dbInfo->vgHash) { + return TSDB_CODE_SUCCESS; + } + + if (dbInfo->vgVersion < 0 || taosHashGetSize(dbInfo->vgHash) <= 0) { + ctgError("invalid db vgInfo, dbFName:%s, vgHash:%p, vgVersion:%d, vgHashSize:%d", + dbFName, dbInfo->vgHash, dbInfo->vgVersion, taosHashGetSize(dbInfo->vgHash)); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } + + bool newAdded = false; + SDbVgVersion vgVersion = {.dbId = msg->dbId, .vgVersion = dbInfo->vgVersion, .numOfTable = dbInfo->numOfTable}; + + SCtgDBCache *dbCache = NULL; + CTG_ERR_RET(ctgGetAddDBCache(msg->pCtg, dbFName, msg->dbId, &dbCache)); + if (NULL == dbCache) { + ctgInfo("conflict db update, ignore this update, dbFName:%s, dbId:%"PRIx64, dbFName, msg->dbId); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + SCtgVgCache *vgCache = &dbCache->vgCache; + CTG_ERR_RET(ctgWLockVgInfo(msg->pCtg, dbCache)); + + if (vgCache->vgInfo) { + SDBVgInfo *vgInfo = vgCache->vgInfo; + + if (dbInfo->vgVersion < vgInfo->vgVersion) { + ctgDebug("db vgVer is old, dbFName:%s, vgVer:%d, curVer:%d", dbFName, dbInfo->vgVersion, vgInfo->vgVersion); + ctgWUnlockVgInfo(dbCache); + + return TSDB_CODE_SUCCESS; + } + + if (dbInfo->vgVersion == vgInfo->vgVersion && dbInfo->numOfTable == vgInfo->numOfTable) { + ctgDebug("no new db vgVer or numOfTable, dbFName:%s, vgVer:%d, numOfTable:%d", dbFName, dbInfo->vgVersion, dbInfo->numOfTable); + ctgWUnlockVgInfo(dbCache); + + return TSDB_CODE_SUCCESS; + } + + ctgFreeVgInfo(vgInfo); + } + + vgCache->vgInfo = dbInfo; + msg->dbInfo = NULL; + + ctgDebug("db vgInfo updated, dbFName:%s, vgVer:%d, dbId:%"PRIx64, dbFName, vgVersion.vgVersion, vgVersion.dbId); + + ctgWUnlockVgInfo(dbCache); + + dbCache = NULL; + + strncpy(vgVersion.dbFName, dbFName, sizeof(vgVersion.dbFName)); + CTG_ERR_RET(ctgMetaRentUpdate(&msg->pCtg->dbRent, &vgVersion, vgVersion.dbId, sizeof(SDbVgVersion), ctgDbVgVersionSortCompare, ctgDbVgVersionSearchCompare)); _return: @@ -1356,14 +1468,14 @@ int32_t ctgOpDropDbVgroup(SCtgCacheOperation *operation) { goto _return; } - CTG_ERR_RET(ctgWAcquireVgInfo(pCtg, dbCache)); + CTG_ERR_RET(ctgWLockVgInfo(pCtg, dbCache)); - ctgFreeVgInfo(dbCache->vgInfo); - dbCache->vgInfo = NULL; + ctgFreeVgInfo(dbCache->vgCache.vgInfo); + dbCache->vgCache.vgInfo = NULL; ctgDebug("db vgInfo removed, dbFName:%s", msg->dbFName); - ctgWReleaseVgInfo(dbCache); + ctgWUnlockVgInfo(dbCache); _return: @@ -1375,42 +1487,47 @@ _return: int32_t ctgOpUpdateTbMeta(SCtgCacheOperation *operation) { int32_t code = 0; - SCtgUpdateTblMsg *msg = operation->data; + SCtgUpdateTbMetaMsg *msg = operation->data; SCatalog* pCtg = msg->pCtg; - STableMetaOutput* output = msg->output; + STableMetaOutput* pMeta = msg->pMeta; SCtgDBCache *dbCache = NULL; - if ((!CTG_IS_META_CTABLE(output->metaType)) && NULL == output->tbMeta) { - ctgError("no valid tbmeta got from meta rsp, dbFName:%s, tbName:%s", output->dbFName, output->tbName); + if ((!CTG_IS_META_CTABLE(pMeta->metaType)) && NULL == pMeta->tbMeta) { + ctgError("no valid tbmeta got from meta rsp, dbFName:%s, tbName:%s", pMeta->dbFName, pMeta->tbName); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } - if (CTG_IS_META_BOTH(output->metaType) && TSDB_SUPER_TABLE != output->tbMeta->tableType) { - ctgError("table type error, expected:%d, actual:%d", TSDB_SUPER_TABLE, output->tbMeta->tableType); + if (CTG_IS_META_BOTH(pMeta->metaType) && TSDB_SUPER_TABLE != pMeta->tbMeta->tableType) { + ctgError("table type error, expected:%d, actual:%d", TSDB_SUPER_TABLE, pMeta->tbMeta->tableType); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } - CTG_ERR_JRET(ctgGetAddDBCache(pCtg, output->dbFName, output->dbId, &dbCache)); + CTG_ERR_JRET(ctgGetAddDBCache(pCtg, pMeta->dbFName, pMeta->dbId, &dbCache)); if (NULL == dbCache) { - ctgInfo("conflict db update, ignore this update, dbFName:%s, dbId:%"PRIx64, output->dbFName, output->dbId); + ctgInfo("conflict db update, ignore this update, dbFName:%s, dbId:%" PRIx64, pMeta->dbFName, pMeta->dbId); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } - if (CTG_IS_META_TABLE(output->metaType) || CTG_IS_META_BOTH(output->metaType)) { - int32_t metaSize = CTG_META_SIZE(output->tbMeta); - - CTG_ERR_JRET(ctgWriteTbMetaToCache(pCtg, dbCache, output->dbFName, output->dbId, output->tbName, output->tbMeta, metaSize)); + if (CTG_IS_META_TABLE(pMeta->metaType) || CTG_IS_META_BOTH(pMeta->metaType)) { + int32_t metaSize = CTG_META_SIZE(pMeta->tbMeta); + CTG_ERR_JRET(ctgWriteTbMetaToCache(pCtg, dbCache, pMeta->dbFName, pMeta->dbId, pMeta->tbName, pMeta->tbMeta, metaSize)); + pMeta->tbMeta = NULL; } - if (CTG_IS_META_CTABLE(output->metaType) || CTG_IS_META_BOTH(output->metaType)) { - CTG_ERR_JRET(ctgWriteTbMetaToCache(pCtg, dbCache, output->dbFName, output->dbId, output->ctbName, (STableMeta *)&output->ctbMeta, sizeof(output->ctbMeta))); + if (CTG_IS_META_CTABLE(pMeta->metaType) || CTG_IS_META_BOTH(pMeta->metaType)) { + SCTableMeta* ctbMeta = taosMemoryMalloc(sizeof(SCTableMeta)); + if (NULL == ctbMeta) { + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } + memcpy(ctbMeta, &pMeta->ctbMeta, sizeof(SCTableMeta)); + CTG_ERR_JRET(ctgWriteTbMetaToCache(pCtg, dbCache, pMeta->dbFName, pMeta->dbId, pMeta->ctbName, (STableMeta *)ctbMeta, sizeof(SCTableMeta))); } _return: - if (output) { - taosMemoryFreeClear(output->tbMeta); - taosMemoryFreeClear(output); + if (pMeta) { + taosMemoryFreeClear(pMeta->tbMeta); + taosMemoryFreeClear(pMeta); } taosMemoryFreeClear(msg); @@ -1435,22 +1552,17 @@ int32_t ctgOpDropStbMeta(SCtgCacheOperation *operation) { return TSDB_CODE_SUCCESS; } - CTG_LOCK(CTG_WRITE, &dbCache->tbCache.stbLock); - if (taosHashRemove(dbCache->tbCache.stbCache, &msg->suid, sizeof(msg->suid))) { + if (taosHashRemove(dbCache->stbCache, &msg->suid, sizeof(msg->suid))) { ctgDebug("stb not exist in stbCache, may be removed, dbFName:%s, stb:%s, suid:%"PRIx64, msg->dbFName, msg->stbName, msg->suid); } else { CTG_CACHE_STAT_DEC(stblNum, 1); } - CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock); - if (taosHashRemove(dbCache->tbCache.metaCache, msg->stbName, strlen(msg->stbName))) { + if (taosHashRemove(dbCache->tbCache, msg->stbName, strlen(msg->stbName))) { ctgError("stb not exist in cache, dbFName:%s, stb:%s, suid:%"PRIx64, msg->dbFName, msg->stbName, msg->suid); } else { CTG_CACHE_STAT_DEC(tblNum, 1); } - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); - - CTG_UNLOCK(CTG_WRITE, &dbCache->tbCache.stbLock); ctgInfo("stb removed from cache, dbFName:%s, stbName:%s, suid:%"PRIx64, msg->dbFName, msg->stbName, msg->suid); @@ -1477,21 +1589,18 @@ int32_t ctgOpDropTbMeta(SCtgCacheOperation *operation) { } if (dbCache->dbId != msg->dbId) { - ctgDebug("dbId already modified, dbFName:%s, current:%"PRIx64", dbId:%"PRIx64", tbName:%s", msg->dbFName, dbCache->dbId, msg->dbId, msg->tbName); + ctgDebug("dbId %" PRIx64 " not match with curId %"PRIx64", dbFName:%s, tbName:%s"msg->dbId, dbCache->dbId, msg->dbFName, msg->tbName); return TSDB_CODE_SUCCESS; } - CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock); - if (taosHashRemove(dbCache->tbCache.metaCache, msg->tbName, strlen(msg->tbName))) { - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); - ctgError("stb not exist in cache, dbFName:%s, tbName:%s", msg->dbFName, msg->tbName); - CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + if (taosHashRemove(dbCache->tbCache, msg->tbName, strlen(msg->tbName))) { + ctgError("tb %s not exist in cache, dbFName:%s", msg->tbName, msg->dbFName); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } else { CTG_CACHE_STAT_DEC(tblNum, 1); } - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); - ctgInfo("table removed from cache, dbFName:%s, tbName:%s", msg->dbFName, msg->tbName); + ctgDebug("table %s removed from cache, dbFName:%s", msg->tbName, msg->dbFName); _return: @@ -1553,7 +1662,6 @@ int32_t ctgOpUpdateUser(SCtgCacheOperation *operation) { _return: - taosHashCleanup(msg->userAuth.createdDbs); taosHashCleanup(msg->userAuth.readDbs); taosHashCleanup(msg->userAuth.writeDbs); @@ -1569,24 +1677,22 @@ int32_t ctgOpUpdateEpset(SCtgCacheOperation *operation) { SCatalog* pCtg = msg->pCtg; SCtgDBCache *dbCache = NULL; - CTG_ERR_RET(ctgAcquireDBCache(pCtg, msg->dbFName, &dbCache)); + CTG_ERR_JRET(ctgGetDBCache(pCtg, msg->dbFName, &dbCache)); if (NULL == dbCache) { ctgDebug("db %s not exist, ignore epset update", msg->dbFName); goto _return; } - SDBVgInfo *vgInfo = NULL; - CTG_ERR_RET(ctgWAcquireVgInfo(pCtg, dbCache)); - - if (NULL == dbCache->vgInfo) { - ctgWReleaseVgInfo(dbCache); + CTG_ERR_JRET(ctgWLockVgInfo(pCtg, dbCache)); + + SDBVgInfo *vgInfo = dbCache->vgCache.vgInfo; + if (NULL == vgInfo) { ctgDebug("vgroup in db %s not cached, ignore epset update", msg->dbFName); goto _return; } - SVgroupInfo* pInfo = taosHashGet(dbCache->vgInfo->vgHash, &msg->vgId, sizeof(msg->vgId)); + SVgroupInfo* pInfo = taosHashGet(vgInfo->vgHash, &msg->vgId, sizeof(msg->vgId)); if (NULL == pInfo) { - ctgWReleaseVgInfo(dbCache); ctgDebug("no vgroup %d in db %s, ignore epset update", msg->vgId, msg->dbFName); goto _return; } @@ -1599,12 +1705,10 @@ int32_t ctgOpUpdateEpset(SCtgCacheOperation *operation) { pInfo->epSet = msg->epSet; - ctgWReleaseVgInfo(dbCache); - _return: if (dbCache) { - ctgReleaseDBCache(msg->pCtg, dbCache); + ctgWUnlockVgInfo(dbCache); } taosMemoryFreeClear(msg); @@ -1612,6 +1716,32 @@ _return: CTG_RET(code); } +int32_t ctgOpUpdateTbIndex(SCtgCacheOperation *operation) { + int32_t code = 0; + SCtgUpdateTbIndexMsg *msg = operation->data; + SCatalog* pCtg = msg->pCtg; + STableIndex* pIndex = msg->pIndex; + SCtgDBCache *dbCache = NULL; + + CTG_ERR_JRET(ctgGetAddDBCache(pCtg, msg->dbFName, 0, &dbCache)); + if (NULL == dbCache) { + CTG_ERR_JRET(code); + } + + CTG_ERR_JRET(ctgWriteTbIndexToCache(pCtg, dbCache, msg->tbName, &pIndex)); + +_return: + + if (pIndex) { + taosArrayDestroyEx(pIndex->pIndex, tFreeSTableIndexInfo); + taosMemoryFreeClear(pIndex); + } + + taosMemoryFreeClear(msg); + + CTG_RET(code); +} + void ctgUpdateThreadUnexpectedStopped(void) { if (CTG_IS_LOCKED(&gCtgMgmt.lock) > 0) CTG_UNLOCK(CTG_READ, &gCtgMgmt.lock); diff --git a/source/libs/catalog/src/ctgDbg.c b/source/libs/catalog/src/ctgDbg.c index d70a176631..52f29e3acb 100644 --- a/source/libs/catalog/src/ctgDbg.c +++ b/source/libs/catalog/src/ctgDbg.c @@ -266,11 +266,11 @@ int32_t ctgdGetStatNum(char *option, void *res) { } int32_t ctgdGetTbMetaNum(SCtgDBCache *dbCache) { - return dbCache->tbCache.metaCache ? (int32_t)taosHashGetSize(dbCache->tbCache.metaCache) : 0; + return dbCache->table.tbCache ? (int32_t)taosHashGetSize(dbCache->table.tbCache) : 0; } int32_t ctgdGetStbNum(SCtgDBCache *dbCache) { - return dbCache->tbCache.stbCache ? (int32_t)taosHashGetSize(dbCache->tbCache.stbCache) : 0; + return dbCache->table.stbCache ? (int32_t)taosHashGetSize(dbCache->table.stbCache) : 0; } int32_t ctgdGetRentNum(SCtgRentMgmt *rent) { @@ -363,8 +363,8 @@ void ctgdShowDBCache(SCatalog* pCtg, SHashObj *dbHash) { dbFName = taosHashGetKey(pIter, &len); - int32_t metaNum = dbCache->tbCache.metaCache ? taosHashGetSize(dbCache->tbCache.metaCache) : 0; - int32_t stbNum = dbCache->tbCache.stbCache ? taosHashGetSize(dbCache->tbCache.stbCache) : 0; + int32_t metaNum = dbCache->table.tbCache ? taosHashGetSize(dbCache->table.tbCache) : 0; + int32_t stbNum = dbCache->table.stbCache ? taosHashGetSize(dbCache->table.stbCache) : 0; int32_t vgVersion = CTG_DEFAULT_INVALID_VERSION; int32_t hashMethod = -1; int32_t vgNum = 0; diff --git a/source/libs/catalog/src/ctgRemote.c b/source/libs/catalog/src/ctgRemote.c index bfd5057c69..c2ebd9e210 100644 --- a/source/libs/catalog/src/ctgRemote.c +++ b/source/libs/catalog/src/ctgRemote.c @@ -448,10 +448,7 @@ int32_t ctgGetTbIndexFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, SName *n } if (pTask) { - void* pOut = taosMemoryCalloc(1, POINTER_BYTES); - if (NULL == pOut) { - CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); - } + void* pOut = NULL; CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, pOut, (char*)tbFName)); CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask, reqType, msg, msgLen)); diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index a792c555ee..7ab9a6ab9c 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -110,25 +110,39 @@ void ctgFreeMetaRent(SCtgRentMgmt *mgmt) { taosMemoryFreeClear(mgmt->slots); } - -void ctgFreeTbMetaCache(SCtgTbMetaCache *cache) { - CTG_LOCK(CTG_WRITE, &cache->stbLock); - if (cache->stbCache) { - int32_t stblNum = taosHashGetSize(cache->stbCache); - taosHashCleanup(cache->stbCache); - cache->stbCache = NULL; - CTG_CACHE_STAT_DEC(stblNum, stblNum); +void ctgFreeStbMetaCache(SCtgDBCache *dbCache) { + if (NULL == dbCache->stbCache) { + return; } - CTG_UNLOCK(CTG_WRITE, &cache->stbLock); - CTG_LOCK(CTG_WRITE, &cache->metaLock); - if (cache->metaCache) { - int32_t tblNum = taosHashGetSize(cache->metaCache); - taosHashCleanup(cache->metaCache); - cache->metaCache = NULL; - CTG_CACHE_STAT_DEC(tblNum, tblNum); + int32_t stblNum = taosHashGetSize(dbCache->stbCache); + taosHashCleanup(dbCache->stbCache); + dbCache->stbCache = NULL; + CTG_CACHE_STAT_DEC(stblNum, stblNum); +} + +void ctgFreeTbCacheImpl(SCtgTbCache *pCache) { + taosMemoryFreeClear(pCache->pMeta); + if (pCache->pIndex) { + taosArrayDestroyEx(pCache->pIndex->pIndex, tFreeSTableIndexInfo); + taosMemoryFreeClear(pCache->pIndex); } - CTG_UNLOCK(CTG_WRITE, &cache->metaLock); +} + +void ctgFreeTbCache(SCtgDBCache *dbCache) { + if (NULL == dbCache->tbCache) { + return; + } + + int32_t tblNum = taosHashGetSize(dbCache->tbCache); + SCtgTbCache *pCache = taosHashIterate(dbCache->tbCache, NULL); + while (NULL != pCache) { + ctgFreeTbCacheImpl(pCache); + pCache = taosHashIterate(dbCache->tbCache, pCache); + } + taosHashCleanup(dbCache->tbCache); + dbCache->tbCache = NULL; + CTG_CACHE_STAT_DEC(tblNum, tblNum); } void ctgFreeVgInfo(SDBVgInfo *vgInfo) { @@ -144,16 +158,18 @@ void ctgFreeVgInfo(SDBVgInfo *vgInfo) { taosMemoryFreeClear(vgInfo); } +void ctgFreeVgInfoCache(SCtgDBCache *dbCache) { + ctgFreeVgInfo(dbCache->vgCache.vgInfo); +} + void ctgFreeDbCache(SCtgDBCache *dbCache) { if (NULL == dbCache) { return; } - CTG_LOCK(CTG_WRITE, &dbCache->vgLock); - ctgFreeVgInfo (dbCache->vgInfo); - CTG_UNLOCK(CTG_WRITE, &dbCache->vgLock); - - ctgFreeTbMetaCache(&dbCache->tbCache); + ctgFreeVgInfoCache(dbCache); + ctgFreeStbMetaCache(dbCache); + ctgFreeTbCache(dbCache); } @@ -167,16 +183,13 @@ void ctgFreeHandle(SCatalog* pCtg) { void *pIter = taosHashIterate(pCtg->dbCache, NULL); while (pIter) { SCtgDBCache *dbCache = pIter; - atomic_store_8(&dbCache->deleted, 1); - ctgFreeDbCache(dbCache); pIter = taosHashIterate(pCtg->dbCache, pIter); } taosHashCleanup(pCtg->dbCache); - CTG_CACHE_STAT_DEC(dbNum, dbNum); } @@ -186,14 +199,12 @@ void ctgFreeHandle(SCatalog* pCtg) { void *pIter = taosHashIterate(pCtg->userCache, NULL); while (pIter) { SCtgUserAuth *userCache = pIter; - ctgFreeSCtgUserAuth(userCache); pIter = taosHashIterate(pCtg->userCache, pIter); } taosHashCleanup(pCtg->userCache); - CTG_CACHE_STAT_DEC(userNum, userNum); } @@ -252,9 +263,9 @@ void ctgFreeMsgCtx(SCtgMsgCtx* pCtx) { break; } case TDMT_MND_GET_TABLE_INDEX: { - SArray** pOut = (SArray**)pCtx->out; + STableIndex* pOut = (STableIndex*)pCtx->out; if (pOut) { - taosArrayDestroyEx(*pOut, tFreeSTableIndexInfo); + taosArrayDestroyEx(pOut->pIndex, tFreeSTableIndexInfo); taosMemoryFreeClear(pCtx->out); } break; @@ -640,6 +651,27 @@ int32_t ctgCloneMetaOutput(STableMetaOutput *output, STableMetaOutput **pOutput) return TSDB_CODE_SUCCESS; } +int32_t ctgCloneTableIndex(SArray* pIndex, SArray** pRes) { + if (NULL == pIndex) { + *pRes = NULL; + return TSDB_CODE_SUCCESS; + } + + int32_t num = taosArrayGetSize(pIndex); + *pRes = taosArrayInit(num, sizeof(STableIndexInfo)); + if (NULL == *pRes) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + for (int32_t i = 0; i < num; ++i) { + STableIndexInfo *pInfo = taosArrayGet(pIndex, i); + taosArrayPush(*pRes, pInfo); + } + + return TSDB_CODE_SUCCESS; +} + + int32_t ctgUpdateSendTargetInfo(SMsgSendInfo *pMsgSendInfo, int32_t msgType, SCtgTask* pTask) { if (msgType == TDMT_VND_TABLE_META) { SCtgTbMetaCtx* ctx = (SCtgTbMetaCtx*)pTask->taskCtx; diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index 04dbc26774..254525f78f 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -895,9 +895,9 @@ void *ctgTestSetCtableMetaThread(void *param) { output = (STableMetaOutput *)taosMemoryMalloc(sizeof(STableMetaOutput)); ctgTestBuildCTableMetaOutput(output); - SCtgUpdateTblMsg *msg = (SCtgUpdateTblMsg *)taosMemoryMalloc(sizeof(SCtgUpdateTblMsg)); + SCtgUpdateTbMetaMsg *msg = (SCtgUpdateTbMetaMsg *)taosMemoryMalloc(sizeof(SCtgUpdateTbMetaMsg)); msg->pCtg = pCtg; - msg->output = output; + msg->pMeta = output; operation.data = msg; code = ctgOpUpdateTbMeta(&operation); diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c index 10d0f4a44a..edc468a6e2 100644 --- a/source/libs/qcom/src/querymsg.c +++ b/source/libs/qcom/src/querymsg.c @@ -484,13 +484,17 @@ int32_t queryProcessGetTbIndexRsp(void *output, char *msg, int32_t msgSize) { return TSDB_CODE_TSC_INVALID_INPUT; } - STableIndexRsp out = {0}; - if (tDeserializeSTableIndexRsp(msg, msgSize, &out) != 0) { + STableIndexRsp *out = taosMemoryCalloc(1, sizeof(STableIndexRsp)); + if (NULL == out) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + if (tDeserializeSTableIndexRsp(msg, msgSize, out) != 0) { qError("tDeserializeSTableIndexRsp failed, msgSize:%d", msgSize); return TSDB_CODE_INVALID_MSG; } - *(void **)output = out.pIndex; + *(void **)output = out; return TSDB_CODE_SUCCESS; } diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index ae0eda5449..451607e7d0 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -837,7 +837,6 @@ void qwProcessHbTimerEvent(void *param, void *tmrId) { SQWorker *mgmt = qwAcquire(refId); if (NULL == mgmt) { QW_DLOG("qwAcquire %" PRIx64 "failed", refId); - taosMemoryFree(param); return; } diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 6704bfdde3..a446aa7bdb 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -1476,6 +1476,11 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options) for (uint32_t i = 0; i < info->fields[FLD_TYPE_VALUE].num; ++i) { SFilterField *field = &info->fields[FLD_TYPE_VALUE].fields[i]; if (field->desc) { + if (QUERY_NODE_VALUE != nodeType(field->desc)) { + qDebug("VAL%d => [type:not value node][val:NIL]", i); //TODO + continue; + } + SValueNode *var = (SValueNode *)field->desc; SDataType *dType = &var->node.resType; if (dType->type == TSDB_DATA_TYPE_VALUE_ARRAY) { diff --git a/source/util/src/tarray.c b/source/util/src/tarray.c index a34b332b9d..2357f760d1 100644 --- a/source/util/src/tarray.c +++ b/source/util/src/tarray.c @@ -458,6 +458,9 @@ static void taosArrayInsertSort(SArray* pArray, __ext_compar_fn_t fn, const void } SArray* taosArrayDeepCopy(const SArray* pSrc, FCopy deepCopy) { + if (NULL == pSrc) { + return NULL; + } ASSERT(pSrc->elemSize == sizeof(void*)); SArray* pArray = taosArrayInit(pSrc->size, sizeof(void*)); for (int32_t i = 0; i < pSrc->size; i++) { From aad288ab1751e73e52079427139165d941209477 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Sun, 12 Jun 2022 15:21:56 +0800 Subject: [PATCH 02/78] refactor(sync): add debug log --- include/libs/sync/sync.h | 13 ++- source/dnode/mnode/impl/src/mndMain.c | 12 +-- source/dnode/mnode/impl/src/mndSync.c | 7 +- source/dnode/vnode/src/vnd/vnodeSvr.c | 6 +- source/dnode/vnode/src/vnd/vnodeSync.c | 3 + source/libs/sync/inc/syncInt.h | 4 + source/libs/sync/src/syncAppendEntries.c | 2 +- source/libs/sync/src/syncCommit.c | 2 +- source/libs/sync/src/syncMain.c | 130 +++++++++++++++++------ 9 files changed, 124 insertions(+), 55 deletions(-) diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index 3a77cc1e19..2673bc382b 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -47,8 +47,9 @@ typedef enum { typedef enum { TAOS_SYNC_PROPOSE_SUCCESS = 0, TAOS_SYNC_PROPOSE_NOT_LEADER = 1, - TAOS_SYNC_PROPOSE_OTHER_ERROR = 2, - TAOS_SYNC_ONLY_ONE_REPLICA = 3, + TAOS_SYNC_ONLY_ONE_REPLICA = 2, + TAOS_SYNC_NOT_IN_NEW_CONFIG = 3, + TAOS_SYNC_OTHER_ERROR = 100, } ESyncProposeCode; typedef enum { @@ -199,15 +200,13 @@ bool syncIsRestoreFinish(int64_t rid); int32_t syncGetSnapshotMeta(int64_t rid, struct SSnapshotMeta* sMeta); int32_t syncReconfig(int64_t rid, const SSyncCfg* pNewCfg); -int32_t syncReconfigRaw(int64_t rid, const SSyncCfg* pNewCfg, SRpcMsg* pRpcMsg); + +// build SRpcMsg, need to call syncPropose with SRpcMsg +int32_t syncReconfigBuild(int64_t rid, const SSyncCfg* pNewCfg, SRpcMsg* pRpcMsg); int32_t syncLeaderTransfer(int64_t rid); int32_t syncLeaderTransferTo(int64_t rid, SNodeInfo newLeader); -// to be moved to static -void syncStartNormal(int64_t rid); -void syncStartStandBy(int64_t rid); - #ifdef __cplusplus } #endif diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index d8a61461dd..8b86845627 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -382,22 +382,22 @@ void mndStop(SMnode *pMnode) { int32_t mndProcessSyncMsg(SRpcMsg *pMsg) { SMnode *pMnode = pMsg->info.node; SSyncMgmt *pMgmt = &pMnode->syncMgmt; - int32_t code = TAOS_SYNC_PROPOSE_OTHER_ERROR; + int32_t code = TAOS_SYNC_OTHER_ERROR; if (!syncEnvIsStart()) { mError("failed to process sync msg:%p type:%s since syncEnv stop", pMsg, TMSG_INFO(pMsg->msgType)); - return TAOS_SYNC_PROPOSE_OTHER_ERROR; + return TAOS_SYNC_OTHER_ERROR; } SSyncNode *pSyncNode = syncNodeAcquire(pMgmt->sync); if (pSyncNode == NULL) { mError("failed to process sync msg:%p type:%s since syncNode is null", pMsg, TMSG_INFO(pMsg->msgType)); - return TAOS_SYNC_PROPOSE_OTHER_ERROR; + return TAOS_SYNC_OTHER_ERROR; } if (mndAcquireSyncRef(pMnode) != 0) { mError("failed to process sync msg:%p type:%s since %s", pMsg, TMSG_INFO(pMsg->msgType), terrstr()); - return TAOS_SYNC_PROPOSE_OTHER_ERROR; + return TAOS_SYNC_OTHER_ERROR; } char logBuf[512] = {0}; @@ -457,7 +457,7 @@ int32_t mndProcessSyncMsg(SRpcMsg *pMsg) { } else { mError("failed to process msg:%p since invalid type:%s", pMsg, TMSG_INFO(pMsg->msgType)); - code = TAOS_SYNC_PROPOSE_OTHER_ERROR; + code = TAOS_SYNC_OTHER_ERROR; } } else { @@ -495,7 +495,7 @@ int32_t mndProcessSyncMsg(SRpcMsg *pMsg) { syncAppendEntriesReplyDestroy(pSyncMsg); } else { mError("failed to process msg:%p since invalid type:%s", pMsg, TMSG_INFO(pMsg->msgType)); - code = TAOS_SYNC_PROPOSE_OTHER_ERROR; + code = TAOS_SYNC_OTHER_ERROR; } } diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index e0b4cc6a57..24b4f1926d 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -236,7 +236,7 @@ int32_t mndSyncPropose(SMnode *pMnode, SSdbRaw *pRaw, int32_t transId) { tsem_wait(&pMgmt->syncSem); } else if (code == TAOS_SYNC_PROPOSE_NOT_LEADER) { terrno = TSDB_CODE_APP_NOT_READY; - } else if (code == TAOS_SYNC_PROPOSE_OTHER_ERROR) { + } else if (code == TAOS_SYNC_OTHER_ERROR) { terrno = TSDB_CODE_SYN_INTERNAL_ERROR; } else { terrno = TSDB_CODE_APP_ERROR; @@ -254,13 +254,16 @@ int32_t mndSyncPropose(SMnode *pMnode, SSdbRaw *pRaw, int32_t transId) { void mndSyncStart(SMnode *pMnode) { SSyncMgmt *pMgmt = &pMnode->syncMgmt; syncSetMsgCb(pMgmt->sync, &pMnode->msgCb); + syncStart(pMgmt->sync); + mDebug("mnode sync started, id:%" PRId64 " standby:%d", pMgmt->sync, pMgmt->standby); +/* if (pMgmt->standby) { syncStartStandBy(pMgmt->sync); } else { syncStart(pMgmt->sync); } - mDebug("mnode sync started, id:%" PRId64 " standby:%d", pMgmt->sync, pMgmt->standby); +*/ } void mndSyncStop(SMnode *pMnode) {} diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 51ed739693..4d1a5060dd 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -291,7 +291,7 @@ void vnodeUpdateMetaRsp(SVnode *pVnode, STableMetaRsp *pMetaRsp) { } int32_t vnodeProcessSyncReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { - int32_t ret = TAOS_SYNC_PROPOSE_OTHER_ERROR; + int32_t ret = TAOS_SYNC_OTHER_ERROR; if (syncEnvIsStart()) { SSyncNode *pSyncNode = syncNodeAcquire(pVnode->sync); @@ -372,13 +372,13 @@ int32_t vnodeProcessSyncReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { } else { vError("==vnodeProcessSyncReq== error msg type:%d", pRpcMsg->msgType); - ret = TAOS_SYNC_PROPOSE_OTHER_ERROR; + ret = TAOS_SYNC_OTHER_ERROR; } syncNodeRelease(pSyncNode); } else { vError("==vnodeProcessSyncReq== error syncEnv stop"); - ret = TAOS_SYNC_PROPOSE_OTHER_ERROR; + ret = TAOS_SYNC_OTHER_ERROR; } return ret; diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index b4021ec73d..ec95451b31 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -290,11 +290,14 @@ int32_t vnodeSyncOpen(SVnode *pVnode, char *path) { void vnodeSyncStart(SVnode *pVnode) { syncSetMsgCb(pVnode->sync, &pVnode->msgCb); + syncStart(pVnode->sync); + /* if (pVnode->config.standby) { syncStartStandBy(pVnode->sync); } else { syncStart(pVnode->sync); } + */ } void vnodeSyncClose(SVnode *pVnode) { syncStop(pVnode->sync); } diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index e7777af749..8a44dcd9bc 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -168,6 +168,7 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo); void syncNodeStart(SSyncNode* pSyncNode); void syncNodeStartStandBy(SSyncNode* pSyncNode); void syncNodeClose(SSyncNode* pSyncNode); +int32_t syncNodePropose(SSyncNode* pSyncNode, const SRpcMsg* pMsg, bool isWeak); // option bool syncNodeSnapshotEnable(SSyncNode* pSyncNode); @@ -232,6 +233,9 @@ int32_t syncNodeCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endIndex, bool syncNodeInRaftGroup(SSyncNode* ths, SRaftId* pRaftId); SSyncSnapshotSender* syncNodeGetSnapshotSender(SSyncNode* ths, SRaftId* pDestId); +void syncStartNormal(int64_t rid); +void syncStartStandBy(int64_t rid); + // for debug -------------- void syncNodePrint(SSyncNode* pObj); void syncNodePrint2(char* s, SSyncNode* pObj); diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index 29ee263756..08a4081ad3 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -995,7 +995,7 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs ths->commitIndex = snapshot.lastApplyIndex; sDebug("vgId:%d sync event commit by snapshot from index:%ld to index:%ld, %s", ths->vgId, commitBegin, - commitEnd, syncUtilState2String(ths->state)); + commitEnd, syncUtilState2String(ths->state)); } SyncIndex beginIndex = ths->commitIndex + 1; diff --git a/source/libs/sync/src/syncCommit.c b/source/libs/sync/src/syncCommit.c index eac9c1fbd8..3424fac5e7 100644 --- a/source/libs/sync/src/syncCommit.c +++ b/source/libs/sync/src/syncCommit.c @@ -57,7 +57,7 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) { pSyncNode->commitIndex = snapshot.lastApplyIndex; sDebug("vgId:%d sync event commit by snapshot from index:%ld to index:%ld, %s", pSyncNode->vgId, - pSyncNode->commitIndex, snapshot.lastApplyIndex, syncUtilState2String(pSyncNode->state)); + pSyncNode->commitIndex, snapshot.lastApplyIndex, syncUtilState2String(pSyncNode->state)); } // update commit index diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 4d410644bb..fbac8c3d9d 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -149,12 +149,12 @@ void syncStop(int64_t rid) { int32_t syncSetStandby(int64_t rid) { SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); if (pSyncNode == NULL) { - return -1; + return TAOS_SYNC_OTHER_ERROR; } if (pSyncNode->state != TAOS_SYNC_STATE_LEADER) { taosReleaseRef(tsNodeRefId, pSyncNode->rid); - return -1; + return TAOS_SYNC_OTHER_ERROR; } // state change @@ -173,14 +173,68 @@ int32_t syncSetStandby(int64_t rid) { return 0; } -int32_t syncReconfig(int64_t rid, const SSyncCfg* pSyncCfg) { - int32_t ret = 0; - char* newconfig = syncCfg2Str((SSyncCfg*)pSyncCfg); +int32_t syncReconfigBuild(int64_t rid, const SSyncCfg* pNewCfg, SRpcMsg* pRpcMsg) { + SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); + if (pSyncNode == NULL) { + return TAOS_SYNC_OTHER_ERROR; + } + ASSERT(rid == pSyncNode->rid); + int32_t ret = 0; + bool IamInNew = false; + for (int i = 0; i < pNewCfg->replicaNum; ++i) { + SRaftId newId; + newId.addr = syncUtilAddr2U64((pNewCfg->nodeInfo)[i].nodeFqdn, (pNewCfg->nodeInfo)[i].nodePort); + newId.vgId = pSyncNode->vgId; + if (syncUtilSameId(&(pSyncNode->myRaftId), &newId)) { + IamInNew = true; + } + } + if (!IamInNew) { + taosReleaseRef(tsNodeRefId, pSyncNode->rid); + return TAOS_SYNC_NOT_IN_NEW_CONFIG; + } + + char* newconfig = syncCfg2Str((SSyncCfg*)pNewCfg); + pRpcMsg->msgType = TDMT_SYNC_CONFIG_CHANGE; + pRpcMsg->info.noResp = 1; + pRpcMsg->contLen = strlen(newconfig) + 1; + pRpcMsg->pCont = rpcMallocCont(pRpcMsg->contLen); + snprintf(pRpcMsg->pCont, pRpcMsg->contLen, "%s", newconfig); + taosMemoryFree(newconfig); + + taosReleaseRef(tsNodeRefId, pSyncNode->rid); + return ret; +} + +int32_t syncReconfig(int64_t rid, const SSyncCfg* pNewCfg) { + SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); + if (pSyncNode == NULL) { + return TAOS_SYNC_OTHER_ERROR; + } + ASSERT(rid == pSyncNode->rid); + + bool IamInNew = false; + for (int i = 0; i < pNewCfg->replicaNum; ++i) { + SRaftId newId; + newId.addr = syncUtilAddr2U64((pNewCfg->nodeInfo)[i].nodeFqdn, (pNewCfg->nodeInfo)[i].nodePort); + newId.vgId = pSyncNode->vgId; + if (syncUtilSameId(&(pSyncNode->myRaftId), &newId)) { + IamInNew = true; + } + } + if (!IamInNew) { + taosReleaseRef(tsNodeRefId, pSyncNode->rid); + return TAOS_SYNC_NOT_IN_NEW_CONFIG; + } + + char* newconfig = syncCfg2Str((SSyncCfg*)pNewCfg); if (gRaftDetailLog) { sInfo("==syncReconfig== newconfig:%s", newconfig); } + int32_t ret = 0; + SRpcMsg rpcMsg = {0}; rpcMsg.msgType = TDMT_SYNC_CONFIG_CHANGE; rpcMsg.info.noResp = 1; @@ -188,27 +242,42 @@ int32_t syncReconfig(int64_t rid, const SSyncCfg* pSyncCfg) { rpcMsg.pCont = rpcMallocCont(rpcMsg.contLen); snprintf(rpcMsg.pCont, rpcMsg.contLen, "%s", newconfig); taosMemoryFree(newconfig); - ret = syncPropose(rid, &rpcMsg, false); + ret = syncNodePropose(pSyncNode, &rpcMsg, false); + + taosReleaseRef(tsNodeRefId, pSyncNode->rid); return ret; } int32_t syncLeaderTransfer(int64_t rid) { - int32_t ret = 0; + SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); + if (pSyncNode == NULL) { + return TAOS_SYNC_OTHER_ERROR; + } + ASSERT(rid == pSyncNode->rid); + if (pSyncNode->peersNum > 0) { + taosReleaseRef(tsNodeRefId, pSyncNode->rid); + return TAOS_SYNC_OTHER_ERROR; + } + + SNodeInfo newLeader = (pSyncNode->peersNodeInfo)[0]; + taosReleaseRef(tsNodeRefId, pSyncNode->rid); + + int32_t ret = syncLeaderTransferTo(rid, newLeader); return ret; } int32_t syncLeaderTransferTo(int64_t rid, SNodeInfo newLeader) { SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); if (pSyncNode == NULL) { - return false; + return TAOS_SYNC_OTHER_ERROR; } - assert(rid == pSyncNode->rid); + ASSERT(rid == pSyncNode->rid); int32_t ret = 0; if (pSyncNode->replicaNum == 1) { - taosReleaseRef(tsNodeRefId, pSyncNode->rid); sError("only one replica, cannot drop leader"); + taosReleaseRef(tsNodeRefId, pSyncNode->rid); return TAOS_SYNC_ONLY_ONE_REPLICA; } @@ -220,26 +289,11 @@ int32_t syncLeaderTransferTo(int64_t rid, SNodeInfo newLeader) { syncLeaderTransfer2RpcMsg(pMsg, &rpcMsg); syncLeaderTransferDestroy(pMsg); - ret = syncPropose(rid, &rpcMsg, false); - + ret = syncNodePropose(pSyncNode, &rpcMsg, false); taosReleaseRef(tsNodeRefId, pSyncNode->rid); return ret; } -int32_t syncReconfigRaw(int64_t rid, const SSyncCfg* pNewCfg, SRpcMsg* pRpcMsg) { - int32_t ret = 0; - char* newconfig = syncCfg2Str((SSyncCfg*)pNewCfg); - - pRpcMsg->msgType = TDMT_SYNC_CONFIG_CHANGE; - pRpcMsg->info.noResp = 1; - pRpcMsg->contLen = strlen(newconfig) + 1; - pRpcMsg->pCont = rpcMallocCont(pRpcMsg->contLen); - snprintf(pRpcMsg->pCont, pRpcMsg->contLen, "%s", newconfig); - taosMemoryFree(newconfig); - - return ret; -} - bool syncCanLeaderTransfer(int64_t rid) { SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); if (pSyncNode == NULL) { @@ -272,8 +326,6 @@ bool syncCanLeaderTransfer(int64_t rid) { return matchOK; } -int32_t syncGiveUpLeader(int64_t rid) { return 0; } - int32_t syncForwardToPeer(int64_t rid, const SRpcMsg* pMsg, bool isWeak) { int32_t ret = syncPropose(rid, pMsg, isWeak); return ret; @@ -467,10 +519,19 @@ int32_t syncPropose(int64_t rid, const SRpcMsg* pMsg, bool isWeak) { SSyncNode* pSyncNode = taosAcquireRef(tsNodeRefId, rid); if (pSyncNode == NULL) { - return TAOS_SYNC_PROPOSE_OTHER_ERROR; + return TAOS_SYNC_OTHER_ERROR; } assert(rid == pSyncNode->rid); sDebug("vgId:%d sync event propose msgType:%s", pSyncNode->vgId, TMSG_INFO(pMsg->msgType)); + ret = syncNodePropose(pSyncNode, pMsg, isWeak); + + taosReleaseRef(tsNodeRefId, pSyncNode->rid); + return ret; +} + +int32_t syncNodePropose(SSyncNode* pSyncNode, const SRpcMsg* pMsg, bool isWeak) { + int32_t ret = TAOS_SYNC_PROPOSE_SUCCESS; + sDebug("vgId:%d sync event propose msgType:%s", pSyncNode->vgId, TMSG_INFO(pMsg->msgType)); if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) { SRespStub stub; @@ -485,15 +546,14 @@ int32_t syncPropose(int64_t rid, const SRpcMsg* pMsg, bool isWeak) { if (pSyncNode->FpEqMsg != NULL && (*pSyncNode->FpEqMsg)(pSyncNode->msgcb, &rpcMsg) == 0) { ret = TAOS_SYNC_PROPOSE_SUCCESS; } else { - sTrace("syncPropose pSyncNode->FpEqMsg is NULL"); + sError("syncPropose pSyncNode->FpEqMsg is NULL"); } syncClientRequestDestroy(pSyncMsg); } else { - sTrace("syncPropose not leader, %s", syncUtilState2String(pSyncNode->state)); + sError("syncPropose not leader, %s", syncUtilState2String(pSyncNode->state)); ret = TAOS_SYNC_PROPOSE_NOT_LEADER; } - taosReleaseRef(tsNodeRefId, pSyncNode->rid); return ret; } @@ -1241,7 +1301,7 @@ void syncNodeUpdateTerm(SSyncNode* pSyncNode, SyncTerm term) { void syncNodeBecomeFollower(SSyncNode* pSyncNode, const char* debugStr) { sDebug("vgId:%d sync event become follower, isStandBy:%d, %s", pSyncNode->vgId, pSyncNode->pRaftCfg->isStandBy, - debugStr); + debugStr); // maybe clear leader cache if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) { @@ -1276,7 +1336,7 @@ void syncNodeBecomeFollower(SSyncNode* pSyncNode, const char* debugStr) { // void syncNodeBecomeLeader(SSyncNode* pSyncNode, const char* debugStr) { sDebug("vgId:%d sync event become leader, isStandBy:%d, %s", pSyncNode->vgId, pSyncNode->pRaftCfg->isStandBy, - debugStr); + debugStr); // state change pSyncNode->state = TAOS_SYNC_STATE_LEADER; @@ -1885,7 +1945,7 @@ int32_t syncNodeCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endIndex, int32_t code = 0; ESyncState state = flag; sDebug("vgId:%d sync event commit by wal from index:%" PRId64 " to index:%" PRId64 ", %s", ths->vgId, beginIndex, - endIndex, syncUtilState2String(state)); + endIndex, syncUtilState2String(state)); // execute fsm if (ths->pFsm != NULL) { From 17f6275015745c9b348a74c7b390964be34df239 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 13 Jun 2022 10:38:45 +0800 Subject: [PATCH 03/78] add sma cache --- include/common/tmsg.h | 16 +++---- include/libs/catalog/catalog.h | 7 ++-- source/client/src/clientHb.c | 19 +++++---- source/common/src/tmsg.c | 8 ++-- source/dnode/mnode/impl/inc/mndStb.h | 2 +- source/dnode/mnode/impl/src/mndProfile.c | 2 +- source/dnode/mnode/impl/src/mndSma.c | 17 ++++++-- source/dnode/mnode/impl/src/mndStb.c | 51 +++++++++++++++++------ source/libs/catalog/inc/catalogInt.h | 2 +- source/libs/catalog/src/catalog.c | 4 +- source/libs/catalog/src/ctgCache.c | 40 ++++++++++++++---- source/libs/catalog/src/ctgUtil.c | 8 ++-- source/libs/catalog/test/catalogTests.cpp | 8 ++-- 13 files changed, 124 insertions(+), 60 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 7c37484ef6..ed35e46c98 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1136,12 +1136,13 @@ int32_t tDeserializeSTableMetaRsp(void* buf, int32_t bufLen, STableMetaRsp* pRsp void tFreeSTableMetaRsp(STableMetaRsp* pRsp); typedef struct { - SArray* pArray; // Array of STableMetaRsp -} STableMetaBatchRsp; + SArray* pArray; // Array of STableMetaRsp + STableIndexRsp* pIndexRsp; +} SSTbHbRsp; -int32_t tSerializeSTableMetaBatchRsp(void* buf, int32_t bufLen, STableMetaBatchRsp* pRsp); -int32_t tDeserializeSTableMetaBatchRsp(void* buf, int32_t bufLen, STableMetaBatchRsp* pRsp); -void tFreeSTableMetaBatchRsp(STableMetaBatchRsp* pRsp); +int32_t tSerializeSSTbHbRsp(void* buf, int32_t bufLen, SSTbHbRsp* pRsp); +int32_t tDeserializeSSTbHbRsp(void* buf, int32_t bufLen, SSTbHbRsp* pRsp); +void tFreeSSTbHbRsp(SSTbHbRsp* pRsp); typedef struct { int32_t numOfTables; @@ -2513,8 +2514,9 @@ typedef struct { } STableIndexInfo; typedef struct { - int32_t version; - SArray* pIndex; + uint64_t suid; + int32_t version; + SArray* pIndex; } STableIndexRsp; int32_t tSerializeSTableIndexRsp(void* buf, int32_t bufLen, const STableIndexRsp* pRsp); diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index 89fdeac7b9..80b677dbd2 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -98,14 +98,15 @@ typedef struct SCatalogCfg { uint32_t stbRentSec; } SCatalogCfg; -typedef struct SSTableMetaVersion { +typedef struct SSTableVersion { char dbFName[TSDB_DB_FNAME_LEN]; char stbName[TSDB_TABLE_NAME_LEN]; uint64_t dbId; uint64_t suid; int16_t sversion; int16_t tversion; -} SSTableMetaVersion; + int32_t smaVer; +} SSTableVersion; typedef struct SDbVgVersion { char dbFName[TSDB_DB_FNAME_LEN]; @@ -269,7 +270,7 @@ int32_t catalogAsyncGetAllMeta(SCatalog* pCtg, SRequestConnInfo* pConn, uint64_t int32_t catalogGetQnodeList(SCatalog* pCatalog, SRequestConnInfo* pConn, SArray* pQnodeList); -int32_t catalogGetExpiredSTables(SCatalog* pCatalog, SSTableMetaVersion **stables, uint32_t *num); +int32_t catalogGetExpiredSTables(SCatalog* pCatalog, SSTableVersion **stables, uint32_t *num); int32_t catalogGetExpiredDBs(SCatalog* pCatalog, SDbVgVersion **dbs, uint32_t *num); diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 9df0e0c1c8..1f6b00466c 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -99,15 +99,15 @@ static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog static int32_t hbProcessStbInfoRsp(void *value, int32_t valueLen, struct SCatalog *pCatalog) { int32_t code = 0; - STableMetaBatchRsp batchMetaRsp = {0}; - if (tDeserializeSTableMetaBatchRsp(value, valueLen, &batchMetaRsp) != 0) { + SSTbHbRsp hbRsp = {0}; + if (tDeserializeSSTbHbRsp(value, valueLen, &hbRsp) != 0) { terrno = TSDB_CODE_INVALID_MSG; return -1; } - int32_t numOfBatchs = taosArrayGetSize(batchMetaRsp.pArray); + int32_t numOfBatchs = taosArrayGetSize(hbRsp.pArray); for (int32_t i = 0; i < numOfBatchs; ++i) { - STableMetaRsp *rsp = taosArrayGet(batchMetaRsp.pArray, i); + STableMetaRsp *rsp = taosArrayGet(hbRsp.pArray, i); if (rsp->numOfColumns < 0) { tscDebug("hb remove stb, db:%s, stb:%s", rsp->dbFName, rsp->stbName); @@ -116,7 +116,7 @@ static int32_t hbProcessStbInfoRsp(void *value, int32_t valueLen, struct SCatalo tscDebug("hb update stb, db:%s, stb:%s", rsp->dbFName, rsp->stbName); if (rsp->pSchemas[0].colId != PRIMARYKEY_TIMESTAMP_COL_ID) { tscError("invalid colId[%" PRIi16 "] for the first column in table meta rsp msg", rsp->pSchemas[0].colId); - tFreeSTableMetaBatchRsp(&batchMetaRsp); + tFreeSSTbHbRsp(&hbRsp); return TSDB_CODE_TSC_INVALID_VALUE; } @@ -124,7 +124,7 @@ static int32_t hbProcessStbInfoRsp(void *value, int32_t valueLen, struct SCatalo } } - tFreeSTableMetaBatchRsp(&batchMetaRsp); + tFreeSSTbHbRsp(&hbRsp); return TSDB_CODE_SUCCESS; } @@ -455,7 +455,7 @@ int32_t hbGetExpiredDBInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SCl } int32_t hbGetExpiredStbInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SClientHbReq *req) { - SSTableMetaVersion *stbs = NULL; + SSTableVersion *stbs = NULL; uint32_t stbNum = 0; int32_t code = 0; @@ -469,15 +469,16 @@ int32_t hbGetExpiredStbInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SC } for (int32_t i = 0; i < stbNum; ++i) { - SSTableMetaVersion *stb = &stbs[i]; + SSTableVersion *stb = &stbs[i]; stb->suid = htobe64(stb->suid); stb->sversion = htons(stb->sversion); stb->tversion = htons(stb->tversion); + stb->smaVer = htonl(stb->smaVer); } SKv kv = { .key = HEARTBEAT_KEY_STBINFO, - .valueLen = sizeof(SSTableMetaVersion) * stbNum, + .valueLen = sizeof(SSTableVersion) * stbNum, .value = stbs, }; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 3805a8b5b4..138aaef3da 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -2437,6 +2437,7 @@ int32_t tSerializeSTableIndexRsp(void *buf, int32_t bufLen, const STableIndexRsp tEncoderInit(&encoder, buf, bufLen); if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeU64(&encoder, pRsp->suid) < 0) return -1; if (tEncodeI32(&encoder, pRsp->version) < 0) return -1; int32_t num = taosArrayGetSize(pRsp->pIndex); if (tEncodeI32(&encoder, num) < 0) return -1; @@ -2472,6 +2473,7 @@ int32_t tDeserializeSTableIndexRsp(void *buf, int32_t bufLen, STableIndexRsp *pR tDecoderInit(&decoder, buf, bufLen); if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeU64(&decoder, &pRsp->suid) < 0) return -1; if (tDecodeI32(&decoder, &pRsp->version) < 0) return -1; int32_t num = 0; if (tDecodeI32(&decoder, &num) < 0) return -1; @@ -2632,7 +2634,7 @@ int32_t tSerializeSTableMetaRsp(void *buf, int32_t bufLen, STableMetaRsp *pRsp) return tlen; } -int32_t tSerializeSTableMetaBatchRsp(void *buf, int32_t bufLen, STableMetaBatchRsp *pRsp) { +int32_t tSerializeSSTbHbRsp(void *buf, int32_t bufLen, SSTbHbRsp *pRsp) { SEncoder encoder = {0}; tEncoderInit(&encoder, buf, bufLen); @@ -2663,7 +2665,7 @@ int32_t tDeserializeSTableMetaRsp(void *buf, int32_t bufLen, STableMetaRsp *pRsp return 0; } -int32_t tDeserializeSTableMetaBatchRsp(void *buf, int32_t bufLen, STableMetaBatchRsp *pRsp) { +int32_t tDeserializeSSTbHbRsp(void *buf, int32_t bufLen, SSTbHbRsp *pRsp) { SDecoder decoder = {0}; tDecoderInit(&decoder, buf, bufLen); @@ -2691,7 +2693,7 @@ int32_t tDeserializeSTableMetaBatchRsp(void *buf, int32_t bufLen, STableMetaBatc void tFreeSTableMetaRsp(STableMetaRsp *pRsp) { taosMemoryFreeClear(pRsp->pSchemas); } -void tFreeSTableMetaBatchRsp(STableMetaBatchRsp *pRsp) { +void tFreeSSTbHbRsp(SSTbHbRsp *pRsp) { int32_t numOfBatch = taosArrayGetSize(pRsp->pArray); for (int32_t i = 0; i < numOfBatch; ++i) { STableMetaRsp *pMetaRsp = taosArrayGet(pRsp->pArray, i); diff --git a/source/dnode/mnode/impl/inc/mndStb.h b/source/dnode/mnode/impl/inc/mndStb.h index 28d3215b98..d3de4e3b3f 100644 --- a/source/dnode/mnode/impl/inc/mndStb.h +++ b/source/dnode/mnode/impl/inc/mndStb.h @@ -27,7 +27,7 @@ void mndCleanupStb(SMnode *pMnode); SStbObj *mndAcquireStb(SMnode *pMnode, char *stbName); void mndReleaseStb(SMnode *pMnode, SStbObj *pStb); SSdbRaw *mndStbActionEncode(SStbObj *pStb); -int32_t mndValidateStbInfo(SMnode *pMnode, SSTableMetaVersion *pStbs, int32_t numOfStbs, void **ppRsp, +int32_t mndValidateStbInfo(SMnode *pMnode, SSTableVersion *pStbs, int32_t numOfStbs, void **ppRsp, int32_t *pRspLen); int32_t mndGetNumOfStbs(SMnode *pMnode, char *dbName, int32_t *pNumOfStbs); diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index bacdf2f366..4cfcf389aa 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -433,7 +433,7 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb case HEARTBEAT_KEY_STBINFO: { void *rspMsg = NULL; int32_t rspLen = 0; - mndValidateStbInfo(pMnode, kv->value, kv->valueLen / sizeof(SSTableMetaVersion), &rspMsg, &rspLen); + mndValidateStbInfo(pMnode, kv->value, kv->valueLen / sizeof(SSTableVersion), &rspMsg, &rspLen); if (rspMsg && rspLen > 0) { SKv kv1 = {.key = HEARTBEAT_KEY_STBINFO, .valueLen = rspLen, .value = rspMsg}; taosArrayPush(hbRsp.info, &kv1); diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index cde36eac58..edb8a29ac0 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -872,18 +872,29 @@ static int32_t mndGetSma(SMnode *pMnode, SUserIndexReq *indexReq, SUserIndexRsp return code; } -static int32_t mndGetTableSma(SMnode *pMnode, STableIndexReq *indexReq, STableIndexRsp *rsp, bool *exist) { +static int32_t mndGetTableSma(SMnode *pMnode, char *tbFName, STableIndexRsp *rsp, bool *exist) { int32_t code = 0; SSmaObj *pSma = NULL; SSdb *pSdb = pMnode->pSdb; void *pIter = NULL; STableIndexInfo info; + SStbObj* pStb = mndAcquireStb(pMnode, tbFName); + if (NULL == pStb) { + *exist = false; + return TSDB_CODE_SUCCESS; + } + + rsp->suid = pStb->uid; + rsp->version = pStb->smaVer; + mndReleaseStb(pMnode, pStb); + + while (1) { pIter = sdbFetch(pSdb, SDB_SMA, pIter, (void **)&pSma); if (pIter == NULL) break; - if (pSma->stb[0] != indexReq->tbFName[0] || strcmp(pSma->stb, indexReq->tbFName)) { + if (pSma->stb[0] != tbFName[0] || strcmp(pSma->stb, tbFName)) { continue; } @@ -995,7 +1006,7 @@ static int32_t mndProcessGetTbSmaReq(SRpcMsg *pReq) { goto _OVER; } - code = mndGetTableSma(pMnode, &indexReq, &rsp, &exist); + code = mndGetTableSma(pMnode, indexReq.tbFName, &rsp, &exist); if (code) { goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 9981dc8530..4b5b9e8a7b 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -1271,7 +1271,7 @@ static int32_t mndBuildStbSchemaImp(SDbObj *pDb, SStbObj *pStb, const char *tbNa return 0; } -static int32_t mndBuildStbSchema(SMnode *pMnode, const char *dbFName, const char *tbName, STableMetaRsp *pRsp) { +static int32_t mndBuildStbSchema(SMnode *pMnode, const char *dbFName, const char *tbName, STableMetaRsp *pRsp, int32_t *smaVer) { char tbFName[TSDB_TABLE_FNAME_LEN] = {0}; snprintf(tbFName, sizeof(tbFName), "%s.%s", dbFName, tbName); @@ -1288,6 +1288,10 @@ static int32_t mndBuildStbSchema(SMnode *pMnode, const char *dbFName, const char return -1; } + if (smaVer) { + *smaVer = pStb->smaVer; + } + int32_t code = mndBuildStbSchemaImp(pDb, pStb, tbName, pRsp); mndReleaseDb(pMnode, pDb); mndReleaseStb(pMnode, pStb); @@ -1667,51 +1671,72 @@ _OVER: return code; } -int32_t mndValidateStbInfo(SMnode *pMnode, SSTableMetaVersion *pStbVersions, int32_t numOfStbs, void **ppRsp, +int32_t mndValidateStbInfo(SMnode *pMnode, SSTableVersion *pStbVersions, int32_t numOfStbs, void **ppRsp, int32_t *pRspLen) { - STableMetaBatchRsp batchMetaRsp = {0}; - batchMetaRsp.pArray = taosArrayInit(numOfStbs, sizeof(STableMetaRsp)); - if (batchMetaRsp.pArray == NULL) { + SSTbHbRsp hbRsp = {0}; + hbRsp.pArray = taosArrayInit(numOfStbs, sizeof(STableMetaRsp)); + if (hbRsp.pArray == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } + hbRsp.pIndexRsp = taosMemoryCalloc(1, sizeof(STableIndexRsp)); + if (NULL == hbRsp.pIndexRsp) { + taosArrayDestroy(hbRsp.pArray); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + for (int32_t i = 0; i < numOfStbs; ++i) { - SSTableMetaVersion *pStbVersion = &pStbVersions[i]; + SSTableVersion *pStbVersion = &pStbVersions[i]; pStbVersion->suid = be64toh(pStbVersion->suid); pStbVersion->sversion = ntohs(pStbVersion->sversion); pStbVersion->tversion = ntohs(pStbVersion->tversion); + pStbVersion->smaVer = ntohl(pStbVersion->smaVer); STableMetaRsp metaRsp = {0}; + int32_t smaVer = 0; mDebug("stb:%s.%s, start to retrieve meta", pStbVersion->dbFName, pStbVersion->stbName); - if (mndBuildStbSchema(pMnode, pStbVersion->dbFName, pStbVersion->stbName, &metaRsp) != 0) { + if (mndBuildStbSchema(pMnode, pStbVersion->dbFName, pStbVersion->stbName, &metaRsp, &smaVer) != 0) { metaRsp.numOfColumns = -1; metaRsp.suid = pStbVersion->suid; + taosArrayPush(hbRsp.pArray, &metaRsp); + continue; } if (pStbVersion->sversion != metaRsp.sversion || pStbVersion->tversion != metaRsp.tversion) { - taosArrayPush(batchMetaRsp.pArray, &metaRsp); + taosArrayPush(hbRsp.pArray, &metaRsp); } else { tFreeSTableMetaRsp(&metaRsp); } + + if (pStbVersion->smaVer != smaVer) { + bool exist = false; + char tbFName[TSDB_TABLE_FNAME_LEN]; + sprintf(tbFName, "%s.%s", pStbVersion->dbFName, pStbVersion->stbName); + int32_t code = mndGetTableSma(pMnode, tbFName, hbRsp.pIndexRsp, &exist); + if (code || !exist) { + taosMemoryFreeClear(hbRsp.pIndexRsp); + } + } } - int32_t rspLen = tSerializeSTableMetaBatchRsp(NULL, 0, &batchMetaRsp); + int32_t rspLen = tSerializeSSTbHbRsp(NULL, 0, &hbRsp); if (rspLen < 0) { - tFreeSTableMetaBatchRsp(&batchMetaRsp); + tFreeSSTbHbRsp(&hbRsp); terrno = TSDB_CODE_INVALID_MSG; return -1; } void *pRsp = taosMemoryMalloc(rspLen); if (pRsp == NULL) { - tFreeSTableMetaBatchRsp(&batchMetaRsp); + tFreeSSTbHbRsp(&hbRsp); terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - tSerializeSTableMetaBatchRsp(pRsp, rspLen, &batchMetaRsp); - tFreeSTableMetaBatchRsp(&batchMetaRsp); + tSerializeSSTbHbRsp(pRsp, rspLen, &hbRsp); + tFreeSSTbHbRsp(&hbRsp); *ppRsp = pRsp; *pRspLen = rspLen; return 0; diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index ebd4d56aaf..82bb151d86 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -156,7 +156,7 @@ typedef struct SCtgDBCache { typedef struct SCtgRentSlot { SRWLatch lock; bool needSort; - SArray *meta; // element is SDbVgVersion or SSTableMetaVersion + SArray *meta; // element is SDbVgVersion or SSTableVersion } SCtgRentSlot; typedef struct SCtgRentMgmt { diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index bac13bf476..d01fa00878 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -1074,14 +1074,14 @@ _return: CTG_API_LEAVE(TSDB_CODE_SUCCESS); } -int32_t catalogGetExpiredSTables(SCatalog* pCtg, SSTableMetaVersion **stables, uint32_t *num) { +int32_t catalogGetExpiredSTables(SCatalog* pCtg, SSTableVersion **stables, uint32_t *num) { CTG_API_ENTER(); if (NULL == pCtg || NULL == stables || NULL == num) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } - CTG_API_LEAVE(ctgMetaRentGet(&pCtg->stbRent, (void **)stables, num, sizeof(SSTableMetaVersion))); + CTG_API_LEAVE(ctgMetaRentGet(&pCtg->stbRent, (void **)stables, num, sizeof(SSTableVersion))); } int32_t catalogGetExpiredDBs(SCatalog* pCtg, SDbVgVersion **dbs, uint32_t *num) { diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 2b56eb6b49..49714ac2de 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -970,7 +970,7 @@ int32_t ctgMetaRentUpdate(SCtgRentMgmt *mgmt, void *meta, int64_t id, int32_t si void *orig = taosArraySearch(slot->meta, &id, searchCompare, TD_EQ); if (NULL == orig) { - qError("meta not found in slot, id:%"PRIx64", slot idx:%d, type:%d, size:%d", id, widx, mgmt->type, (int32_t)taosArrayGetSize(slot->meta)); + qDebug("meta not found in slot, id:%"PRIx64", slot idx:%d, type:%d, size:%d", id, widx, mgmt->type, (int32_t)taosArrayGetSize(slot->meta)); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } @@ -983,7 +983,7 @@ _return: CTG_UNLOCK(CTG_WRITE, &slot->lock); if (code) { - qWarn("meta in rent update failed, will try to add it, code:%x, id:%"PRIx64", slot idx:%d, type:%d", code, id, widx, mgmt->type); + qDebug("meta in rent update failed, will try to add it, code:%x, id:%"PRIx64", slot idx:%d, type:%d", code, id, widx, mgmt->type); CTG_RET(ctgMetaRentAdd(mgmt, meta, id, size)); } @@ -1233,6 +1233,27 @@ int32_t ctgGetAddDBCache(SCatalog* pCtg, const char *dbFName, uint64_t dbId, SCt return TSDB_CODE_SUCCESS; } +int32_t ctgUpdateRentStbVersion(SCatalog *pCtg, char* dbFName, char* tbName, uint64_t dbId, uint64_t suid, SCtgTbCache* pCache) { + SSTableVersion metaRent = {.dbId = dbId, .suid = suid}; + if (pCache->pMeta) { + metaRent.sversion = pCache->pMeta->sversion; + metaRent.tversion = pCache->pMeta->tversion; + } + + if (pCache->pIndex) { + metaRent.smaVer = pCache->pIndex->version; + } + + strcpy(metaRent.dbFName, dbFName); + strcpy(metaRent.stbName, tbName); + + CTG_ERR_RET(ctgMetaRentUpdate(&pCtg->stbRent, &metaRent, metaRent.suid, sizeof(SSTableVersion), ctgStbVersionSortCompare, ctgStbVersionSearchCompare)); + + ctgDebug("db %s,%" PRIx64 " stb %s,%" PRIx64 " sver %d tver %d smaVer %d updated to stbRent", + dbFName, dbId, tbName, suid, metaRent.sversion, metaRent.tversion, metaRent.smaVer); +} + + int32_t ctgWriteTbMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFName, uint64_t dbId, char *tbName, STableMeta *meta, int32_t metaSize) { if (NULL == dbCache->tbCache || NULL == dbCache->stbCache) { taosMemoryFree(meta); @@ -1263,7 +1284,6 @@ int32_t ctgWriteTbMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFNam ctgDebug("stb removed from stbCache, dbFName:%s, stb:%s, suid:%"PRIx64, dbFName, tbName, orig->suid); } - ctgMetaRentRemove(&pCtg->stbRent, orig->suid, ctgStbVersionSortCompare, ctgStbVersionSearchCompare); origSuid = orig->suid; } } @@ -1303,15 +1323,12 @@ int32_t ctgWriteTbMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFNam ctgDebug("stb updated to stbCache, dbFName:%s, tbName:%s, tbType:%d", dbFName, tbName, meta->tableType); - SSTableMetaVersion metaRent = {.dbId = dbId, .suid = meta->suid, .sversion = meta->sversion, .tversion = meta->tversion}; - strcpy(metaRent.dbFName, dbFName); - strcpy(metaRent.stbName, tbName); - CTG_ERR_RET(ctgMetaRentAdd(&pCtg->stbRent, &metaRent, metaRent.suid, sizeof(SSTableMetaVersion))); + CTG_ERR_RET(ctgUpdateRentStbVersion(pCtg, dbFName, tbName, dbId, meta->suid, pCache)); return TSDB_CODE_SUCCESS; } -int32_t ctgWriteTbIndexToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *tbName, STableIndex **index) { +int32_t ctgWriteTbIndexToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char* dbFName, char *tbName, STableIndex **index) { if (NULL == dbCache->tbCache) { taosMemoryFreeClear(*index); ctgError("db is dropping, dbId:%"PRIx64, dbCache->dbId); @@ -1332,6 +1349,9 @@ int32_t ctgWriteTbIndexToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *tbNam *index = NULL; ctgDebug("table %s index updated to cache, ver:%d, num:%d", tbName, pIndex->version, taosArrayGetSize(pIndex->pIndex)); + + CTG_ERR_RET(ctgUpdateRentStbVersion(pCtg, dbFName, tbName, dbCache->dbId, pIndex->suid, pCache)); + return TSDB_CODE_SUCCESS; } @@ -1344,6 +1364,8 @@ int32_t ctgWriteTbIndexToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *tbNam *index = NULL; ctgDebug("table %s index updated to cache, ver:%d, num:%d", tbName, pIndex->version, taosArrayGetSize(pIndex->pIndex)); + + CTG_ERR_RET(ctgUpdateRentStbVersion(pCtg, dbFName, tbName, dbCache->dbId, pIndex->suid, pCache)); return TSDB_CODE_SUCCESS; } @@ -1728,7 +1750,7 @@ int32_t ctgOpUpdateTbIndex(SCtgCacheOperation *operation) { CTG_ERR_JRET(code); } - CTG_ERR_JRET(ctgWriteTbIndexToCache(pCtg, dbCache, msg->tbName, &pIndex)); + CTG_ERR_JRET(ctgWriteTbIndexToCache(pCtg, dbCache, msg->dbFName, msg->tbName, &pIndex)); _return: diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index 7ab9a6ab9c..f0ed60a689 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -546,9 +546,9 @@ int32_t ctgGetVgInfoFromHashValue(SCatalog *pCtg, SDBVgInfo *dbInfo, const SName } int32_t ctgStbVersionSearchCompare(const void* key1, const void* key2) { - if (*(uint64_t *)key1 < ((SSTableMetaVersion*)key2)->suid) { + if (*(uint64_t *)key1 < ((SSTableVersion*)key2)->suid) { return -1; - } else if (*(uint64_t *)key1 > ((SSTableMetaVersion*)key2)->suid) { + } else if (*(uint64_t *)key1 > ((SSTableVersion*)key2)->suid) { return 1; } else { return 0; @@ -566,9 +566,9 @@ int32_t ctgDbVgVersionSearchCompare(const void* key1, const void* key2) { } int32_t ctgStbVersionSortCompare(const void* key1, const void* key2) { - if (((SSTableMetaVersion*)key1)->suid < ((SSTableMetaVersion*)key2)->suid) { + if (((SSTableVersion*)key1)->suid < ((SSTableVersion*)key2)->suid) { return -1; - } else if (((SSTableMetaVersion*)key1)->suid > ((SSTableMetaVersion*)key2)->suid) { + } else if (((SSTableVersion*)key1)->suid > ((SSTableVersion*)key2)->suid) { return 1; } else { return 0; diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index 254525f78f..aab341544f 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -989,7 +989,7 @@ TEST(tableMeta, normalTable) { ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); SDbVgVersion *dbs = NULL; - SSTableMetaVersion *stb = NULL; + SSTableVersion *stb = NULL; uint32_t dbNum = 0, stbNum = 0, allDbNum = 0, allStbNum = 0; int32_t i = 0; while (i < 5) { @@ -1098,7 +1098,7 @@ TEST(tableMeta, childTableCase) { ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); SDbVgVersion *dbs = NULL; - SSTableMetaVersion *stb = NULL; + SSTableVersion *stb = NULL; uint32_t dbNum = 0, stbNum = 0, allDbNum = 0, allStbNum = 0; int32_t i = 0; while (i < 5) { @@ -1220,7 +1220,7 @@ TEST(tableMeta, superTableCase) { ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); SDbVgVersion *dbs = NULL; - SSTableMetaVersion *stb = NULL; + SSTableVersion *stb = NULL; uint32_t dbNum = 0, stbNum = 0, allDbNum = 0, allStbNum = 0; int32_t i = 0; while (i < 5) { @@ -2299,7 +2299,7 @@ TEST(rentTest, allRent) { SArray *vgList = NULL; ctgTestStop = false; SDbVgVersion *dbs = NULL; - SSTableMetaVersion *stable = NULL; + SSTableVersion *stable = NULL; uint32_t num = 0; ctgTestInitLogFile(); From 34c906a1e11ef96187dcaf609732ccbbf10908e8 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Mon, 13 Jun 2022 10:39:32 +0800 Subject: [PATCH 04/78] refactor(sync): add debug log --- source/libs/sync/src/syncMain.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index fbac8c3d9d..fa61bf1b08 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -219,15 +219,22 @@ int32_t syncReconfig(int64_t rid, const SSyncCfg* pNewCfg) { SRaftId newId; newId.addr = syncUtilAddr2U64((pNewCfg->nodeInfo)[i].nodeFqdn, (pNewCfg->nodeInfo)[i].nodePort); newId.vgId = pSyncNode->vgId; + + sTrace("new[%d]: %lu, %d, my: %lu, %d", i, newId.addr, newId.vgId, pSyncNode->myRaftId.addr, pSyncNode->myRaftId.vgId); + if (syncUtilSameId(&(pSyncNode->myRaftId), &newId)) { IamInNew = true; } } + + if (!IamInNew) { + sError("sync reconfig error, not in new config"); taosReleaseRef(tsNodeRefId, pSyncNode->rid); return TAOS_SYNC_NOT_IN_NEW_CONFIG; } + char* newconfig = syncCfg2Str((SSyncCfg*)pNewCfg); if (gRaftDetailLog) { sInfo("==syncReconfig== newconfig:%s", newconfig); From 803363571704761d603b96ce8098192e887eb502 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Mon, 13 Jun 2022 11:29:53 +0800 Subject: [PATCH 05/78] other: add debug logs --- source/common/src/trow.c | 17 +++ source/dnode/vnode/src/sma/smaTimeRange.c | 178 ++++++++++++++++++++++ source/dnode/vnode/src/vnd/vnodeSvr.c | 6 +- 3 files changed, 200 insertions(+), 1 deletion(-) create mode 100644 source/dnode/vnode/src/sma/smaTimeRange.c diff --git a/source/common/src/trow.c b/source/common/src/trow.c index c8a28d7f28..fb4413478d 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -523,8 +523,24 @@ static int32_t tdAppendTpRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols SCellVal sVal = {0}; if (pRowCol->colId == pDataCol->colId) { if (tdGetTpRowValOfCol(&sVal, pRow, pBitmap, pRowCol->type, pRowCol->offset - sizeof(TSKEY), rcol - 1) < 0) { + ASSERT(0); return terrno; } + + if (pRowCol->colId == 2) { + ASSERT(sVal.valType == 0); + int32_t val = *(int32_t *)sVal.val; + ASSERT(val == 7); + } else if (pRowCol->colId == 3) { + ASSERT(sVal.valType == 0); + int64_t val = *(int64_t *)sVal.val; + ASSERT(val == 77777); + } else if (pRowCol->colId == 4) { + ASSERT(sVal.valType == 0); + int16_t val = *(int16_t *)sVal.val; + ASSERT(val == 777); + } + tdAppendValToDataCol(pDataCol, sVal.valType, sVal.val, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode, isMerge); ++dcol; @@ -535,6 +551,7 @@ static int32_t tdAppendTpRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols tdAppendValToDataCol(pDataCol, TD_VTYPE_NULL, NULL, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode, isMerge); ++dcol; + ASSERT(0); } } #if 0 diff --git a/source/dnode/vnode/src/sma/smaTimeRange.c b/source/dnode/vnode/src/sma/smaTimeRange.c new file mode 100644 index 0000000000..bca5b1543e --- /dev/null +++ b/source/dnode/vnode/src/sma/smaTimeRange.c @@ -0,0 +1,178 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "sma.h" +#include "tsdb.h" + +#define SMA_STORAGE_MINUTES_MAX 86400 +#define SMA_STORAGE_MINUTES_DAY 1440 +#define SMA_STORAGE_SPLIT_FACTOR 14400 // least records in tsma file + +/** + * @brief Judge the tsma file split days + * + * @param pCfg + * @param pCont + * @param contLen + * @param days unit is minute + * @return int32_t + */ +int32_t tdProcessTSmaGetDaysImpl(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *days) { + SDecoder coder = {0}; + tDecoderInit(&coder, pCont, contLen); + + STSma tsma = {0}; + if (tDecodeSVCreateTSmaReq(&coder, &tsma) < 0) { + terrno = TSDB_CODE_MSG_DECODE_ERROR; + goto _err; + } + STsdbCfg *pTsdbCfg = &pCfg->tsdbCfg; + int64_t sInterval = convertTimeFromPrecisionToUnit(tsma.interval, pTsdbCfg->precision, TIME_UNIT_SECOND); + if (sInterval <= 0) { + *days = pTsdbCfg->days; + return 0; + } + int64_t records = pTsdbCfg->days * 60 / sInterval; + if (records >= SMA_STORAGE_SPLIT_FACTOR) { + *days = pTsdbCfg->days; + } else { + int64_t mInterval = convertTimeFromPrecisionToUnit(tsma.interval, pTsdbCfg->precision, TIME_UNIT_MINUTE); + int64_t daysPerFile = mInterval * SMA_STORAGE_MINUTES_DAY * 2; + + if (daysPerFile > SMA_STORAGE_MINUTES_MAX) { + *days = SMA_STORAGE_MINUTES_MAX; + } else { + *days = (int32_t)daysPerFile; + } + + if (*days < pTsdbCfg->days) { + *days = pTsdbCfg->days; + } + } + tDecoderClear(&coder); + return 0; +_err: + tDecoderClear(&coder); + return -1; +} + +/** + * @brief create tsma meta and result stable + * + * @param pSma + * @param version + * @param pMsg + * @return int32_t + */ +int32_t tdProcessTSmaCreateImpl(SSma *pSma, int64_t version, const char *pMsg) { + SSmaCfg *pCfg = (SSmaCfg *)pMsg; + + if (TD_VID(pSma->pVnode) == pCfg->dstVgId) { + // create tsma meta in dstVgId + if (metaCreateTSma(SMA_META(pSma), version, pCfg) < 0) { + return -1; + } + + // create stable to save tsma result in dstVgId + SVCreateStbReq pReq = {0}; + pReq.name = pCfg->dstTbName; + pReq.suid = pCfg->dstTbUid; + pReq.schemaRow = pCfg->schemaRow; + pReq.schemaTag = pCfg->schemaTag; + + if (metaCreateSTable(SMA_META(pSma), version, &pReq) < 0) { + return -1; + } + } + + return 0; +} + +/** + * @brief Insert/Update Time-range-wise SMA data. + * + * @param pSma + * @param msg + * @return int32_t + */ +int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg) { + const SArray *pDataBlocks = (const SArray *)msg; + // TODO: destroy SSDataBlocks(msg) + if (!pDataBlocks) { + terrno = TSDB_CODE_TSMA_INVALID_PTR; + smaWarn("vgId:%d, insert tsma data failed since pDataBlocks is NULL", SMA_VID(pSma)); + return terrno; + } + + if (taosArrayGetSize(pDataBlocks) <= 0) { + terrno = TSDB_CODE_TSMA_INVALID_PARA; + smaWarn("vgId:%d, insert tsma data failed since pDataBlocks is empty", SMA_VID(pSma)); + return TSDB_CODE_FAILED; + } + + if (tdCheckAndInitSmaEnv(pSma, TSDB_SMA_TYPE_TIME_RANGE) != 0) { + terrno = TSDB_CODE_TSMA_INIT_FAILED; + return TSDB_CODE_FAILED; + } + + SSmaEnv *pEnv = SMA_TSMA_ENV(pSma); + SSmaStat *pStat = NULL; + SSmaStatItem *pItem = NULL; + + if (!pEnv || !(pStat = SMA_ENV_STAT(pEnv))) { + terrno = TSDB_CODE_TSMA_INVALID_STAT; + return TSDB_CODE_FAILED; + } + + tdRefSmaStat(pSma, pStat); + pItem = &pStat->tsmaStatItem; + + ASSERT(pItem); + + if (!pItem->pTSma) { + // cache smaMeta + STSma *pTSma = metaGetSmaInfoByIndex(SMA_META(pSma), indexUid); + if (!pTSma) { + terrno = TSDB_CODE_TSMA_NO_INDEX_IN_META; + smaWarn("vgId:%d, tsma insert for smaIndex %" PRIi64 " since %s", SMA_VID(pSma), indexUid, tstrerror(terrno)); + return TSDB_CODE_FAILED; + } + pItem->pTSma = pTSma; + } + + STSma *pTSma = pItem->pTSma; + + ASSERT(pTSma->indexUid == indexUid); + + SMetaReader mr = {0}; + + const char *dbName = "testDb"; + if (metaGetTableEntryByName(&mr, dbName) != 0) { + smaDebug("vgId:%d, tsma no table testTb exists for smaIndex %" PRIi64 " since %s", SMA_VID(pSma), indexUid, tstrerror(terrno)); + SVCreateStbReq pReq = {0}; + pReq.name = dbName; + pReq.suid = pTSma->dstTbUid; + pReq.schemaRow = pCfg->schemaRow; + pReq.schemaTag = pCfg->schemaTag; + } + + SSubmitReq *pSubmitReq = NULL; + buildSubmitReqFromDataBlock(&pSubmitReq, (const SArray *)msg, NULL, pItem->pTSma->dstVgId, + pItem->pTSma->dstTbUid); + + tdUnRefSmaStat(pSma, pStat); + + return TSDB_CODE_SUCCESS; +} \ No newline at end of file diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index ab2efa4791..9bb0834547 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -15,6 +15,8 @@ #include "vnd.h" +int32_t gForceCommit = 1; + static int32_t vnodeProcessCreateStbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessAlterStbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessDropStbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); @@ -182,7 +184,8 @@ int32_t vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp } // commit if need - if (vnodeShouldCommit(pVnode)) { + if ((gForceCommit == 1) || vnodeShouldCommit(pVnode)) { + gForceCommit = 0; vInfo("vgId:%d, commit at version %" PRId64, TD_VID(pVnode), version); // commit current change vnodeCommit(pVnode); @@ -719,6 +722,7 @@ static int32_t vnodeDebugPrintSubmitMsg(SVnode *pVnode, SSubmitReq *pMsg, const } static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { + gForceCommit = 1; SSubmitReq *pSubmitReq = (SSubmitReq *)pReq; SSubmitRsp submitRsp = {0}; SSubmitMsgIter msgIter = {0}; From b1093b4a791709b28a1c64423f59f4c19f3ed023 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Mon, 13 Jun 2022 12:13:35 +0800 Subject: [PATCH 06/78] fix(sync): use fqdn instead of raftId --- source/libs/sync/src/syncMain.c | 25 +++++++++++------- source/libs/sync/test/CMakeLists.txt | 14 ++++++++++ source/libs/sync/test/syncRaftIdCheck.cpp | 32 +++++++++++++++++++++++ 3 files changed, 61 insertions(+), 10 deletions(-) create mode 100644 source/libs/sync/test/syncRaftIdCheck.cpp diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index fa61bf1b08..82fde80be2 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -35,7 +35,7 @@ #include "syncVoteMgr.h" #include "tref.h" -bool gRaftDetailLog = false; +bool gRaftDetailLog = true; static int32_t tsNodeRefId = -1; @@ -216,25 +216,30 @@ int32_t syncReconfig(int64_t rid, const SSyncCfg* pNewCfg) { bool IamInNew = false; for (int i = 0; i < pNewCfg->replicaNum; ++i) { - SRaftId newId; - newId.addr = syncUtilAddr2U64((pNewCfg->nodeInfo)[i].nodeFqdn, (pNewCfg->nodeInfo)[i].nodePort); - newId.vgId = pSyncNode->vgId; - - sTrace("new[%d]: %lu, %d, my: %lu, %d", i, newId.addr, newId.vgId, pSyncNode->myRaftId.addr, pSyncNode->myRaftId.vgId); - - if (syncUtilSameId(&(pSyncNode->myRaftId), &newId)) { + if (strcmp((pNewCfg->nodeInfo)[i].nodeFqdn, pSyncNode->myNodeInfo.nodeFqdn) == 0 && + (pNewCfg->nodeInfo)[i].nodePort == pSyncNode->myNodeInfo.nodePort) { IamInNew = true; } + + /* + // some problem in inet_addr + + SRaftId newId = EMPTY_RAFT_ID; + newId.addr = syncUtilAddr2U64((pNewCfg->nodeInfo)[i].nodeFqdn, (pNewCfg->nodeInfo)[i].nodePort); + newId.vgId = pSyncNode->vgId; + + if (syncUtilSameId(&(pSyncNode->myRaftId), &newId)) { + IamInNew = true; + } + */ } - if (!IamInNew) { sError("sync reconfig error, not in new config"); taosReleaseRef(tsNodeRefId, pSyncNode->rid); return TAOS_SYNC_NOT_IN_NEW_CONFIG; } - char* newconfig = syncCfg2Str((SSyncCfg*)pNewCfg); if (gRaftDetailLog) { sInfo("==syncReconfig== newconfig:%s", newconfig); diff --git a/source/libs/sync/test/CMakeLists.txt b/source/libs/sync/test/CMakeLists.txt index d39035ba53..725343e373 100644 --- a/source/libs/sync/test/CMakeLists.txt +++ b/source/libs/sync/test/CMakeLists.txt @@ -1,4 +1,5 @@ add_executable(syncTest "") +add_executable(syncRaftIdCheck "") add_executable(syncEnvTest "") add_executable(syncPingTimerTest "") add_executable(syncIOTickQTest "") @@ -54,6 +55,10 @@ target_sources(syncTest PRIVATE "syncTest.cpp" ) +target_sources(syncRaftIdCheck + PRIVATE + "syncRaftIdCheck.cpp" +) target_sources(syncEnvTest PRIVATE "syncEnvTest.cpp" @@ -257,6 +262,11 @@ target_include_directories(syncTest "${TD_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) +target_include_directories(syncRaftIdCheck + PUBLIC + "${TD_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) target_include_directories(syncEnvTest PUBLIC "${TD_SOURCE_DIR}/include/libs/sync" @@ -508,6 +518,10 @@ target_link_libraries(syncTest sync gtest_main ) +target_link_libraries(syncRaftIdCheck + sync + gtest_main +) target_link_libraries(syncEnvTest sync gtest_main diff --git a/source/libs/sync/test/syncRaftIdCheck.cpp b/source/libs/sync/test/syncRaftIdCheck.cpp new file mode 100644 index 0000000000..f23c0db82b --- /dev/null +++ b/source/libs/sync/test/syncRaftIdCheck.cpp @@ -0,0 +1,32 @@ +#include +#include +#include "syncIO.h" +#include "syncInt.h" +#include "syncUtil.h" + +void usage(char* exe) { + printf("Usage: %s host port \n", exe); + printf("Usage: %s u64 \n", exe); +} + +int main(int argc, char** argv) { + if (argc == 2) { + uint64_t u64 = atoll(argv[1]); + char host[128]; + uint16_t port; + syncUtilU642Addr(u64, host, sizeof(host), &port); + printf("%s:%d \n", host, port); + + } else if (argc == 3) { + uint64_t u64; + char* host = argv[1]; + uint16_t port = atoi(argv[2]); + syncUtilAddr2U64(host, port); + printf("%lu \n", u64); + } else { + usage(argv[0]); + exit(-1); + } + + return 0; +} From c0964e03bee2cf02445dd797a630544cbf9f2ed1 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Mon, 13 Jun 2022 12:51:21 +0800 Subject: [PATCH 07/78] other: add debug logs --- source/dnode/vnode/src/tsdb/tsdbCommit.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 90093f2510..40a23447b7 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -1348,6 +1348,24 @@ static int tsdbCommitMemData(SCommitH *pCommith, SCommitIter *pIter, TSKEY keyLi isLast = true; } + SDataCols *pCols = pCommith->pDataCols; + SDataCol pCol_1 = pCols->cols[1]; + SDataCol pCol_2 = pCols->cols[2]; + SDataCol pCol_3 = pCols->cols[3]; + SCellVal sVal = {0}; + tdGetColDataOfRow(&sVal, &pCol_1, 0, pCols->bitmapMode); + ASSERT(sVal.valType == 0); + int32_t val1 = *(int32_t *)sVal.val; + ASSERT(val1 == 7); + tdGetColDataOfRow(&sVal, &pCol_2, 0, pCols->bitmapMode); + ASSERT(sVal.valType == 0); + int64_t val2 = *(int64_t *)sVal.val; + ASSERT(val2 == 77777); + tdGetColDataOfRow(&sVal, &pCol_3, 0, pCols->bitmapMode); + ASSERT(sVal.valType == 0); + int16_t val3 = *(int16_t *)sVal.val; + ASSERT(val3 == 777); + if (tsdbWriteBlock(pCommith, pDFile, pCommith->pDataCols, &block, isLast, true) < 0) return -1; if (tsdbCommitAddBlock(pCommith, &block, NULL, 0) < 0) { From eff9f9be4db8c0b5081c0d6b14a87e0c30128a35 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 13 Jun 2022 14:06:11 +0800 Subject: [PATCH 08/78] add sma update --- include/common/tmsg.h | 6 +- source/client/src/clientHb.c | 16 ++++- source/common/src/tmsg.c | 102 +++++++++++++++++++++++---- source/dnode/mnode/impl/src/mndSma.c | 2 + source/dnode/mnode/impl/src/mndStb.c | 24 ++++--- source/libs/catalog/inc/catalogInt.h | 4 +- source/libs/catalog/src/catalog.c | 54 ++++++++++++-- source/libs/catalog/src/ctgAsync.c | 3 +- source/libs/catalog/src/ctgCache.c | 16 ++--- source/libs/catalog/src/ctgRemote.c | 8 ++- source/libs/qcom/src/querymsg.c | 8 +-- 11 files changed, 189 insertions(+), 54 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index ed35e46c98..8a2b2a31f2 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1136,8 +1136,8 @@ int32_t tDeserializeSTableMetaRsp(void* buf, int32_t bufLen, STableMetaRsp* pRsp void tFreeSTableMetaRsp(STableMetaRsp* pRsp); typedef struct { - SArray* pArray; // Array of STableMetaRsp - STableIndexRsp* pIndexRsp; + SArray* pMetaRsp; // Array of STableMetaRsp + SArray* pIndexRsp; // Array of STableIndexRsp; } SSTbHbRsp; int32_t tSerializeSSTbHbRsp(void* buf, int32_t bufLen, SSTbHbRsp* pRsp); @@ -2514,6 +2514,8 @@ typedef struct { } STableIndexInfo; typedef struct { + char tbName[TSDB_TABLE_NAME_LEN]; + char dbFName[TSDB_DB_FNAME_LEN]; uint64_t suid; int32_t version; SArray* pIndex; diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 1f6b00466c..e111883381 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -105,9 +105,9 @@ static int32_t hbProcessStbInfoRsp(void *value, int32_t valueLen, struct SCatalo return -1; } - int32_t numOfBatchs = taosArrayGetSize(hbRsp.pArray); - for (int32_t i = 0; i < numOfBatchs; ++i) { - STableMetaRsp *rsp = taosArrayGet(hbRsp.pArray, i); + int32_t numOfMeta = taosArrayGetSize(hbRsp.pMetaRsp); + for (int32_t i = 0; i < numOfMeta; ++i) { + STableMetaRsp *rsp = taosArrayGet(hbRsp.pMetaRsp, i); if (rsp->numOfColumns < 0) { tscDebug("hb remove stb, db:%s, stb:%s", rsp->dbFName, rsp->stbName); @@ -124,6 +124,16 @@ static int32_t hbProcessStbInfoRsp(void *value, int32_t valueLen, struct SCatalo } } + int32_t numOfIndex = taosArrayGetSize(hbRsp.pIndexRsp); + for (int32_t i = 0; i < numOfIndex; ++i) { + STableIndexRsp *rsp = taosArrayGet(hbRsp.pIndexRsp, i); + + catalogUpdateTableIndex(pCatalog, rsp); + } + + taosArrayDestroy(hbRsp.pIndexRsp); + hbRsp.pIndexRsp = NULL; + tFreeSSTbHbRsp(&hbRsp); return TSDB_CODE_SUCCESS; } diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 138aaef3da..5d0cc2b28d 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -2437,6 +2437,8 @@ int32_t tSerializeSTableIndexRsp(void *buf, int32_t bufLen, const STableIndexRsp tEncoderInit(&encoder, buf, bufLen); if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeCStr(&encoder, pRsp->tbName) < 0) return -1; + if (tEncodeCStr(&encoder, pRsp->dbFName) < 0) return -1; if (tEncodeU64(&encoder, pRsp->suid) < 0) return -1; if (tEncodeI32(&encoder, pRsp->version) < 0) return -1; int32_t num = taosArrayGetSize(pRsp->pIndex); @@ -2473,6 +2475,8 @@ int32_t tDeserializeSTableIndexRsp(void *buf, int32_t bufLen, STableIndexRsp *pR tDecoderInit(&decoder, buf, bufLen); if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeCStrTo(&decoder, pRsp->tbName) < 0) return -1; + if (tDecodeCStrTo(&decoder, pRsp->dbFName) < 0) return -1; if (tDecodeU64(&decoder, &pRsp->suid) < 0) return -1; if (tDecodeI32(&decoder, &pRsp->version) < 0) return -1; int32_t num = 0; @@ -2640,12 +2644,30 @@ int32_t tSerializeSSTbHbRsp(void *buf, int32_t bufLen, SSTbHbRsp *pRsp) { if (tStartEncode(&encoder) < 0) return -1; - int32_t numOfBatch = taosArrayGetSize(pRsp->pArray); - if (tEncodeI32(&encoder, numOfBatch) < 0) return -1; - for (int32_t i = 0; i < numOfBatch; ++i) { - STableMetaRsp *pMetaRsp = taosArrayGet(pRsp->pArray, i); + int32_t numOfMeta = taosArrayGetSize(pRsp->pMetaRsp); + if (tEncodeI32(&encoder, numOfMeta) < 0) return -1; + for (int32_t i = 0; i < numOfMeta; ++i) { + STableMetaRsp *pMetaRsp = taosArrayGet(pRsp->pMetaRsp, i); if (tEncodeSTableMetaRsp(&encoder, pMetaRsp) < 0) return -1; } + + int32_t numOfIndex = taosArrayGetSize(pRsp->pIndexRsp); + for (int32_t i = 0; i < numOfMeta; ++i) { + STableIndexRsp *pIndexRsp = taosArrayGet(pRsp->pIndexRsp, i); + if (tEncodeCStr(&encoder, pIndexRsp->tbName) < 0) return -1; + if (tEncodeCStr(&encoder, pIndexRsp->dbFName) < 0) return -1; + if (tEncodeU64(&encoder, pIndexRsp->suid) < 0) return -1; + if (tEncodeI32(&encoder, pIndexRsp->version) < 0) return -1; + int32_t num = taosArrayGetSize(pIndexRsp->pIndex); + if (tEncodeI32(&encoder, num) < 0) return -1; + if (num > 0) { + for (int32_t i = 0; i < num; ++i) { + STableIndexInfo *pInfo = (STableIndexInfo *)taosArrayGet(pIndexRsp->pIndex, i); + if (tSerializeSTableIndexInfo(&encoder, pInfo) < 0) return -1; + } + } + } + tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -2671,20 +2693,53 @@ int32_t tDeserializeSSTbHbRsp(void *buf, int32_t bufLen, SSTbHbRsp *pRsp) { if (tStartDecode(&decoder) < 0) return -1; - int32_t numOfBatch = taosArrayGetSize(pRsp->pArray); - if (tDecodeI32(&decoder, &numOfBatch) < 0) return -1; + int32_t numOfMeta = taosArrayGetSize(pRsp->pMetaRsp); + if (tDecodeI32(&decoder, &numOfMeta) < 0) return -1; - pRsp->pArray = taosArrayInit(numOfBatch, sizeof(STableMetaRsp)); - if (pRsp->pArray == NULL) { + pRsp->pMetaRsp = taosArrayInit(numOfMeta, sizeof(STableMetaRsp)); + if (pRsp->pMetaRsp == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - for (int32_t i = 0; i < numOfBatch; ++i) { + for (int32_t i = 0; i < numOfMeta; ++i) { STableMetaRsp tableMetaRsp = {0}; if (tDecodeSTableMetaRsp(&decoder, &tableMetaRsp) < 0) return -1; - taosArrayPush(pRsp->pArray, &tableMetaRsp); + taosArrayPush(pRsp->pMetaRsp, &tableMetaRsp); } + + int32_t numOfIndex = taosArrayGetSize(pRsp->pIndexRsp); + if (tDecodeI32(&decoder, &numOfIndex) < 0) return -1; + + pRsp->pIndexRsp = taosArrayInit(numOfIndex, sizeof(STableIndexRsp)); + if (pRsp->pIndexRsp == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + for (int32_t i = 0; i < numOfIndex; ++i) { + STableIndexRsp tableIndexRsp = {0}; + if (tDecodeCStrTo(&decoder, tableIndexRsp.tbName) < 0) return -1; + if (tDecodeCStrTo(&decoder, tableIndexRsp.dbFName) < 0) return -1; + if (tDecodeU64(&decoder, &tableIndexRsp.suid) < 0) return -1; + if (tDecodeI32(&decoder, &tableIndexRsp.version) < 0) return -1; + int32_t num = 0; + if (tDecodeI32(&decoder, &num) < 0) return -1; + if (num > 0) { + tableIndexRsp.pIndex = taosArrayInit(num, sizeof(STableIndexInfo)); + if (NULL == tableIndexRsp.pIndex) return -1; + STableIndexInfo info; + for (int32_t i = 0; i < num; ++i) { + if (tDeserializeSTableIndexInfo(&decoder, &info) < 0) return -1; + if (NULL == taosArrayPush(tableIndexRsp.pIndex, &info)) { + taosMemoryFree(info.expr); + return -1; + } + } + } + taosArrayPush(pRsp->pIndexRsp, &tableIndexRsp); + } + tEndDecode(&decoder); tDecoderClear(&decoder); @@ -2693,14 +2748,33 @@ int32_t tDeserializeSSTbHbRsp(void *buf, int32_t bufLen, SSTbHbRsp *pRsp) { void tFreeSTableMetaRsp(STableMetaRsp *pRsp) { taosMemoryFreeClear(pRsp->pSchemas); } +void tFreeSTableIndexRsp(void *info) { + if (NULL == info) { + return; + } + + STableIndexRsp *pInfo = (STableIndexRsp *)info; + + taosArrayDestroyEx(pInfo->pIndex, tFreeSTableIndexInfo); +} + + void tFreeSSTbHbRsp(SSTbHbRsp *pRsp) { - int32_t numOfBatch = taosArrayGetSize(pRsp->pArray); - for (int32_t i = 0; i < numOfBatch; ++i) { - STableMetaRsp *pMetaRsp = taosArrayGet(pRsp->pArray, i); + int32_t numOfMeta = taosArrayGetSize(pRsp->pMetaRsp); + for (int32_t i = 0; i < numOfMeta; ++i) { + STableMetaRsp *pMetaRsp = taosArrayGet(pRsp->pMetaRsp, i); tFreeSTableMetaRsp(pMetaRsp); } - taosArrayDestroy(pRsp->pArray); + taosArrayDestroy(pRsp->pMetaRsp); + + int32_t numOfIndex = taosArrayGetSize(pRsp->pIndexRsp); + for (int32_t i = 0; i < numOfIndex; ++i) { + STableIndexRsp *pIndexRsp = taosArrayGet(pRsp->pIndexRsp, i); + tFreeSTableIndexRsp(pIndexRsp); + } + + taosArrayDestroy(pRsp->pIndexRsp); } int32_t tSerializeSShowRsp(void *buf, int32_t bufLen, SShowRsp *pRsp) { diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index edb8a29ac0..395e2b5255 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -885,6 +885,8 @@ static int32_t mndGetTableSma(SMnode *pMnode, char *tbFName, STableIndexRsp *rsp return TSDB_CODE_SUCCESS; } + strcpy(rsp->dbFName, pStb->db); + strcpy(rsp->tbName, pStb->name); rsp->suid = pStb->uid; rsp->version = pStb->smaVer; mndReleaseStb(pMnode, pStb); diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 4b5b9e8a7b..9db0ab053f 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -1674,15 +1674,15 @@ _OVER: int32_t mndValidateStbInfo(SMnode *pMnode, SSTableVersion *pStbVersions, int32_t numOfStbs, void **ppRsp, int32_t *pRspLen) { SSTbHbRsp hbRsp = {0}; - hbRsp.pArray = taosArrayInit(numOfStbs, sizeof(STableMetaRsp)); - if (hbRsp.pArray == NULL) { + hbRsp.pMetaRsp = taosArrayInit(numOfStbs, sizeof(STableMetaRsp)); + if (hbRsp.pMetaRsp == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - hbRsp.pIndexRsp = taosMemoryCalloc(1, sizeof(STableIndexRsp)); + hbRsp.pIndexRsp = taosArrayInit(numOfStbs, sizeof(STableIndexRsp)); if (NULL == hbRsp.pIndexRsp) { - taosArrayDestroy(hbRsp.pArray); + taosArrayDestroy(hbRsp.pMetaRsp); terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } @@ -1700,12 +1700,12 @@ int32_t mndValidateStbInfo(SMnode *pMnode, SSTableVersion *pStbVersions, int32_t if (mndBuildStbSchema(pMnode, pStbVersion->dbFName, pStbVersion->stbName, &metaRsp, &smaVer) != 0) { metaRsp.numOfColumns = -1; metaRsp.suid = pStbVersion->suid; - taosArrayPush(hbRsp.pArray, &metaRsp); + taosArrayPush(hbRsp.pMetaRsp, &metaRsp); continue; } if (pStbVersion->sversion != metaRsp.sversion || pStbVersion->tversion != metaRsp.tversion) { - taosArrayPush(hbRsp.pArray, &metaRsp); + taosArrayPush(hbRsp.pMetaRsp, &metaRsp); } else { tFreeSTableMetaRsp(&metaRsp); } @@ -1713,11 +1713,19 @@ int32_t mndValidateStbInfo(SMnode *pMnode, SSTableVersion *pStbVersions, int32_t if (pStbVersion->smaVer != smaVer) { bool exist = false; char tbFName[TSDB_TABLE_FNAME_LEN]; + STableIndexRsp indexRsp = {0}; sprintf(tbFName, "%s.%s", pStbVersion->dbFName, pStbVersion->stbName); - int32_t code = mndGetTableSma(pMnode, tbFName, hbRsp.pIndexRsp, &exist); + int32_t code = mndGetTableSma(pMnode, tbFName, &indexRsp, &exist); if (code || !exist) { - taosMemoryFreeClear(hbRsp.pIndexRsp); + indexRsp.suid = pStbVersion->suid; + indexRsp.version = -1; + indexRsp.pIndex = NULL; } + + strcpy(indexRsp->dbFName, pStbVersion->dbFName); + strcpy(indexRsp->tbName, pStbVersion->stbName); + + taosArrayPush(hbRsp.pIndexRsp, &indexRsp); } } diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 82bb151d86..11ae5b3770 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -319,8 +319,6 @@ typedef struct SCtgUpdateUserMsg { typedef struct SCtgUpdateTbIndexMsg { SCatalog* pCtg; - char dbFName[TSDB_DB_FNAME_LEN]; - char tbName[TSDB_TABLE_NAME_LEN]; STableIndex* pIndex; } SCtgUpdateTbIndexMsg; @@ -503,7 +501,7 @@ int32_t ctgUpdateVgroupEnqueue(SCatalog* pCtg, const char *dbFName, int64_t dbId int32_t ctgUpdateTbMetaEnqueue(SCatalog* pCtg, STableMetaOutput *output, bool syncReq); int32_t ctgUpdateUserEnqueue(SCatalog* pCtg, SGetUserAuthRsp *pAuth, bool syncReq); int32_t ctgUpdateVgEpsetEnqueue(SCatalog* pCtg, char *dbFName, int32_t vgId, SEpSet* pEpSet); -int32_t ctgUpdateTbIndexEnqueue(SCatalog* pCtg, SName* pName, STableIndex *pIndex, bool syncOp); +int32_t ctgUpdateTbIndexEnqueue(SCatalog* pCtg, STableIndex *pIndex, bool syncOp); int32_t ctgMetaRentInit(SCtgRentMgmt *mgmt, uint32_t rentSec, int8_t type); int32_t ctgMetaRentAdd(SCtgRentMgmt *mgmt, void *meta, int64_t id, int32_t size); int32_t ctgMetaRentGet(SCtgRentMgmt *mgmt, void **res, uint32_t *num, int32_t size); diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index d01fa00878..de74dc1c68 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -751,6 +751,30 @@ _return: CTG_API_LEAVE(code); } +int32_t catalogUpdateTableIndex(SCatalog* pCtg, STableIndexRsp *pRsp) { + CTG_API_ENTER(); + + int32_t code = 0; + + if (NULL == pCtg || NULL == pRsp) { + CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); + } + + STableIndex* pIndex = taosMemoryCalloc(1, sizeof(STableIndex)); + if (NULL == pIndex) { + CTG_API_LEAVE(TSDB_CODE_OUT_OF_MEMORY); + } + + memcpy(pIndex, pRsp, sizeof(STableIndex)); + + CTG_ERR_JRET(ctgUpdateTbIndexEnqueue(pCtg, &pIndex, false)); + +_return: + + CTG_API_LEAVE(code); +} + + int32_t catalogRemoveTableMeta(SCatalog* pCtg, SName* pTableName) { CTG_API_ENTER(); @@ -1153,16 +1177,36 @@ int32_t catalogGetTableIndex(SCatalog* pCtg, SRequestConnInfo *pConn, const SNam CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } - int32_t code = 0; - CTG_ERR_JRET(ctgGetTbIndexFromMnode(pCtg, pConn, (SName*)pTableName, pRes, NULL)); + CTG_ERR_RET(ctgReadTbIndexFromCache(pCtg, pTableName, pRes)); + if (*pRes) { + CTG_API_LEAVE(TSDB_CODE_SUCCESS); + } - SArray* pInfo = NULL; - CTG_ERR_JRET(ctgCloneTableIndex(*pRes, &pInfo)); + STableIndex *pIndex = taosMemoryCalloc(1, sizeof(STableIndex)); + if (NULL == pIndex) { + CTG_API_LEAVE(TSDB_CODE_OUT_OF_MEMORY); + } - CTG_ERR_JRET(ctgUpdateTbIndexEnqueue(pCtg, pTableName, pInfo, false)); + int32_t code = 0; + CTG_ERR_JRET(ctgGetTbIndexFromMnode(pCtg, pConn, (SName*)pTableName, pIndex, NULL)); + + SArray* pInfo = NULL; + CTG_ERR_JRET(ctgCloneTableIndex(pIndex->pIndex, &pInfo)); + *pRes = pInfo; + + CTG_ERR_JRET(ctgUpdateTbIndexEnqueue(pCtg, pTableName, &pIndex, false)); + + CTG_API_LEAVE(code); + _return: + tFreeSTableIndexRsp(pIndex); + taosMemoryFree(pIndex); + + taosArrayDestroyEx(*pRes, tFreeSTableIndexInfo); + *pRes = NULL; + CTG_API_LEAVE(code); } diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index 69eddd7ba6..36e5b33521 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -876,10 +876,9 @@ int32_t ctgHandleGetTbIndexRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf SArray* pInfo = NULL; CTG_ERR_JRET(ctgCloneTableIndex(pOut->pIndex, &pInfo)); pTask->res = pInfo; - pTask->msgCtx.out = NULL; SCtgTbIndexCtx* ctx = pTask->taskCtx; - CTG_ERR_JRET(ctgUpdateTbIndexEnqueue(pTask->pJob->pCtg, ctx->pName, pOut, false)); + CTG_ERR_JRET(ctgUpdateTbIndexEnqueue(pTask->pJob->pCtg, (STableIndex**)&pTask->msgCtx.out, false)); _return: diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 49714ac2de..f2502f5943 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -838,7 +838,7 @@ _return: CTG_RET(code); } -int32_t ctgUpdateTbIndexEnqueue(SCatalog* pCtg, SName* pName, STableIndex *pIndex, bool syncOp) { +int32_t ctgUpdateTbIndexEnqueue(SCatalog* pCtg, STableIndex **pIndex, bool syncOp) { int32_t code = 0; SCtgCacheOperation *op = taosMemoryCalloc(1, sizeof(SCtgCacheOperation)); op->opId = CTG_OP_UPDATE_TB_INDEX; @@ -851,19 +851,19 @@ int32_t ctgUpdateTbIndexEnqueue(SCatalog* pCtg, SName* pName, STableIndex *pInde } msg->pCtg = pCtg; - msg->pIndex = pIndex; - tNameGetFullDbName(pName, msg->dbFName); - strcpy(msg->tbName, pName->tname); + msg->pIndex = *pIndex; op->data = msg; CTG_ERR_JRET(ctgEnqueue(pCtg, op)); - + + *pIndex = NULL; return TSDB_CODE_SUCCESS; _return: - taosArrayDestroyEx(pIndex, tFreeSTableIndexInfo); + taosArrayDestroyEx(*pIndex, tFreeSTableIndexInfo); + taosMemoryFreeClear(*pIndex); taosMemoryFreeClear(msg); CTG_RET(code); @@ -1745,12 +1745,12 @@ int32_t ctgOpUpdateTbIndex(SCtgCacheOperation *operation) { STableIndex* pIndex = msg->pIndex; SCtgDBCache *dbCache = NULL; - CTG_ERR_JRET(ctgGetAddDBCache(pCtg, msg->dbFName, 0, &dbCache)); + CTG_ERR_JRET(ctgGetAddDBCache(pCtg, pIndex->dbFName, 0, &dbCache)); if (NULL == dbCache) { CTG_ERR_JRET(code); } - CTG_ERR_JRET(ctgWriteTbIndexToCache(pCtg, dbCache, msg->dbFName, msg->tbName, &pIndex)); + CTG_ERR_JRET(ctgWriteTbIndexToCache(pCtg, dbCache, pIndex->dbFName, pIndex->tbName, &pIndex)); _return: diff --git a/source/libs/catalog/src/ctgRemote.c b/source/libs/catalog/src/ctgRemote.c index c2ebd9e210..178025704f 100644 --- a/source/libs/catalog/src/ctgRemote.c +++ b/source/libs/catalog/src/ctgRemote.c @@ -431,7 +431,7 @@ int32_t ctgGetIndexInfoFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, const return TSDB_CODE_SUCCESS; } -int32_t ctgGetTbIndexFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, SName *name, SArray** out, SCtgTask* pTask) { +int32_t ctgGetTbIndexFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, SName *name, STableIndex* out, SCtgTask* pTask) { char *msg = NULL; int32_t msgLen = 0; int32_t reqType = TDMT_MND_GET_TABLE_INDEX; @@ -448,7 +448,11 @@ int32_t ctgGetTbIndexFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, SName *n } if (pTask) { - void* pOut = NULL; + void* pOut = taosMemoryCalloc(1, sizeof(STableIndex)); + if (NULL == pOut) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, pOut, (char*)tbFName)); CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask, reqType, msg, msgLen)); diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c index edc468a6e2..b89c220519 100644 --- a/source/libs/qcom/src/querymsg.c +++ b/source/libs/qcom/src/querymsg.c @@ -484,17 +484,11 @@ int32_t queryProcessGetTbIndexRsp(void *output, char *msg, int32_t msgSize) { return TSDB_CODE_TSC_INVALID_INPUT; } - STableIndexRsp *out = taosMemoryCalloc(1, sizeof(STableIndexRsp)); - if (NULL == out) { - return TSDB_CODE_OUT_OF_MEMORY; - } - + STableIndexRsp *out = (STableIndexRsp*)output; if (tDeserializeSTableIndexRsp(msg, msgSize, out) != 0) { qError("tDeserializeSTableIndexRsp failed, msgSize:%d", msgSize); return TSDB_CODE_INVALID_MSG; } - - *(void **)output = out; return TSDB_CODE_SUCCESS; } From eb6b87b3e3609e9d3820a2707126a96d444e9613 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 13 Jun 2022 14:15:05 +0800 Subject: [PATCH 09/78] fix: invalid read/write --- source/libs/index/src/indexFilter.c | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/source/libs/index/src/indexFilter.c b/source/libs/index/src/indexFilter.c index 6197edd474..1cc2679b06 100644 --- a/source/libs/index/src/indexFilter.c +++ b/source/libs/index/src/indexFilter.c @@ -97,7 +97,14 @@ static int32_t sifGetOperParamNum(EOperatorType ty) { } return 2; } -static int32_t sifValidateColumn(SColumnNode *cn) { +static int32_t sifValidOp(EOperatorType ty) { + if ((ty >= OP_TYPE_ADD && ty <= OP_TYPE_BIT_OR) || (ty == OP_TYPE_IN || ty == OP_TYPE_NOT_IN) || + (ty == OP_TYPE_LIKE || ty == OP_TYPE_NOT_LIKE || ty == OP_TYPE_MATCH || ty == OP_TYPE_NMATCH)) { + return -1; + } + return 0; +} +static int32_t sifValidColumn(SColumnNode *cn) { // add more check if (cn == NULL) { return TSDB_CODE_QRY_INVALID_INPUT; @@ -197,7 +204,7 @@ static int32_t sifInitParam(SNode *node, SIFParam *param, SIFCtx *ctx) { case QUERY_NODE_COLUMN: { SColumnNode *cn = (SColumnNode *)node; /*only support tag column*/ - SIF_ERR_RET(sifValidateColumn(cn)); + SIF_ERR_RET(sifValidColumn(cn)); param->colId = cn->colId; param->colValType = cn->node.resType.type; @@ -505,6 +512,11 @@ static int32_t sifGetOperFn(int32_t funcId, sif_func_t *func, SIdxFltStatus *sta static int32_t sifExecOper(SOperatorNode *node, SIFCtx *ctx, SIFParam *output) { int32_t code = 0; + if (sifValidOp(node->opType) < 0) { + output->status = SFLT_NOT_INDEX; + return code; + } + int32_t nParam = sifGetOperParamNum(node->opType); if (nParam <= 1) { output->status = SFLT_NOT_INDEX; @@ -516,6 +528,12 @@ static int32_t sifExecOper(SOperatorNode *node, SIFCtx *ctx, SIFParam *output) { SIFParam *params = NULL; SIF_ERR_RET(sifInitOperParams(¶ms, node, ctx)); + + if (params[0].status == SFLT_NOT_INDEX || (nParam > 1 && params[1].status == SFLT_NOT_INDEX)) { + output->status = SFLT_NOT_INDEX; + return code; + } + // ugly code, refactor later output->arg = ctx->arg; sif_func_t operFn = sifNullFunc; From 7a78d81232317e63e6ee80408bfd3ba984ea967c Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 13 Jun 2022 15:36:28 +0800 Subject: [PATCH 10/78] fix compile issue --- include/common/tmsg.h | 1 + include/libs/catalog/catalog.h | 2 + source/common/src/tmsg.c | 1 - source/dnode/mnode/impl/inc/mndSma.h | 1 + source/dnode/mnode/impl/src/mndSma.c | 2 +- source/dnode/mnode/impl/src/mndStb.c | 9 +-- source/libs/catalog/inc/catalogInt.h | 14 +++-- source/libs/catalog/src/catalog.c | 64 ++++++++++----------- source/libs/catalog/src/ctgAsync.c | 12 ++-- source/libs/catalog/src/ctgCache.c | 85 +++++++++++++++++++++------- source/libs/catalog/src/ctgDbg.c | 18 +++--- source/libs/catalog/src/ctgUtil.c | 10 ++++ 12 files changed, 142 insertions(+), 77 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 8a2b2a31f2..b4f60c2761 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1134,6 +1134,7 @@ void tFreeSMAlterStbRsp(SMAlterStbRsp* pRsp); int32_t tSerializeSTableMetaRsp(void* buf, int32_t bufLen, STableMetaRsp* pRsp); int32_t tDeserializeSTableMetaRsp(void* buf, int32_t bufLen, STableMetaRsp* pRsp); void tFreeSTableMetaRsp(STableMetaRsp* pRsp); +void tFreeSTableIndexRsp(void *info); typedef struct { SArray* pMetaRsp; // Array of STableMetaRsp diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index 80b677dbd2..dba4ac39d2 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -282,6 +282,8 @@ int32_t catalogGetIndexMeta(SCatalog* pCtg, SRequestConnInfo* pConn, const char* int32_t catalogGetTableIndex(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, SArray** pRes); +int32_t catalogUpdateTableIndex(SCatalog* pCtg, STableIndexRsp *pRsp); + int32_t catalogGetUdfInfo(SCatalog* pCtg, SRequestConnInfo* pConn, const char* funcName, SFuncInfo* pInfo); int32_t catalogChkAuth(SCatalog* pCtg, SRequestConnInfo* pConn, const char* user, const char* dbFName, AUTH_TYPE type, bool *pass); diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 5d0cc2b28d..63153d68b4 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -2758,7 +2758,6 @@ void tFreeSTableIndexRsp(void *info) { taosArrayDestroyEx(pInfo->pIndex, tFreeSTableIndexInfo); } - void tFreeSSTbHbRsp(SSTbHbRsp *pRsp) { int32_t numOfMeta = taosArrayGetSize(pRsp->pMetaRsp); for (int32_t i = 0; i < numOfMeta; ++i) { diff --git a/source/dnode/mnode/impl/inc/mndSma.h b/source/dnode/mnode/impl/inc/mndSma.h index 4a80f619d3..63530b403d 100644 --- a/source/dnode/mnode/impl/inc/mndSma.h +++ b/source/dnode/mnode/impl/inc/mndSma.h @@ -26,6 +26,7 @@ int32_t mndInitSma(SMnode *pMnode); void mndCleanupSma(SMnode *pMnode); SSmaObj *mndAcquireSma(SMnode *pMnode, char *smaName); void mndReleaseSma(SMnode *pMnode, SSmaObj *pSma); +int32_t mndGetTableSma(SMnode *pMnode, char *tbFName, STableIndexRsp *rsp, bool *exist); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index 395e2b5255..9c28763082 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -872,7 +872,7 @@ static int32_t mndGetSma(SMnode *pMnode, SUserIndexReq *indexReq, SUserIndexRsp return code; } -static int32_t mndGetTableSma(SMnode *pMnode, char *tbFName, STableIndexRsp *rsp, bool *exist) { +int32_t mndGetTableSma(SMnode *pMnode, char *tbFName, STableIndexRsp *rsp, bool *exist) { int32_t code = 0; SSmaObj *pSma = NULL; SSdb *pSdb = pMnode->pSdb; diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 9db0ab053f..8d936e68ce 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -27,6 +27,7 @@ #include "mndTrans.h" #include "mndUser.h" #include "mndVgroup.h" +#include "mndSma.h" #include "tname.h" #define STB_VER_NUMBER 1 @@ -1638,7 +1639,7 @@ static int32_t mndProcessTableMetaReq(SRpcMsg *pReq) { } } else { mDebug("stb:%s.%s, start to retrieve meta", infoReq.dbFName, infoReq.tbName); - if (mndBuildStbSchema(pMnode, infoReq.dbFName, infoReq.tbName, &metaRsp) != 0) { + if (mndBuildStbSchema(pMnode, infoReq.dbFName, infoReq.tbName, &metaRsp, NULL) != 0) { goto _OVER; } } @@ -1710,7 +1711,7 @@ int32_t mndValidateStbInfo(SMnode *pMnode, SSTableVersion *pStbVersions, int32_t tFreeSTableMetaRsp(&metaRsp); } - if (pStbVersion->smaVer != smaVer) { + if (pStbVersion->smaVer && pStbVersion->smaVer != smaVer) { bool exist = false; char tbFName[TSDB_TABLE_FNAME_LEN]; STableIndexRsp indexRsp = {0}; @@ -1722,8 +1723,8 @@ int32_t mndValidateStbInfo(SMnode *pMnode, SSTableVersion *pStbVersions, int32_t indexRsp.pIndex = NULL; } - strcpy(indexRsp->dbFName, pStbVersion->dbFName); - strcpy(indexRsp->tbName, pStbVersion->stbName); + strcpy(indexRsp.dbFName, pStbVersion->dbFName); + strcpy(indexRsp.tbName, pStbVersion->stbName); taosArrayPush(hbRsp.pIndexRsp, &indexRsp); } diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 11ae5b3770..df4b9b6ea6 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -491,8 +491,8 @@ void ctgReleaseDBCache(SCatalog *pCtg, SCtgDBCache *dbCache); void ctgRUnlockVgInfo(SCtgDBCache *dbCache); int32_t ctgTbMetaExistInCache(SCatalog* pCtg, char *dbFName, char* tbName, int32_t *exist); int32_t ctgReadTbMetaFromCache(SCatalog* pCtg, SCtgTbMetaCtx* ctx, STableMeta** pTableMeta); -int32_t ctgReadTbVerFromCache(SCatalog *pCtg, const SName *pTableName, int32_t *sver, int32_t *tver, int32_t *tbType, uint64_t *suid, char *stbName); -int32_t ctgChkAuthFromCache(SCatalog* pCtg, const char* user, const char* dbFName, AUTH_TYPE type, bool *inCache, bool *pass); +int32_t ctgReadTbVerFromCache(SCatalog *pCtg, SName *pTableName, int32_t *sver, int32_t *tver, int32_t *tbType, uint64_t *suid, char *stbName); +int32_t ctgChkAuthFromCache(SCatalog* pCtg, char* user, char* dbFName, AUTH_TYPE type, bool *inCache, bool *pass); int32_t ctgDropDbCacheEnqueue(SCatalog* pCtg, const char *dbFName, int64_t dbId); int32_t ctgDropDbVgroupEnqueue(SCatalog* pCtg, const char *dbFName, bool syncReq); int32_t ctgDropStbMetaEnqueue(SCatalog* pCtg, const char *dbFName, int64_t dbId, const char *stbName, uint64_t suid, bool syncReq); @@ -501,13 +501,18 @@ int32_t ctgUpdateVgroupEnqueue(SCatalog* pCtg, const char *dbFName, int64_t dbId int32_t ctgUpdateTbMetaEnqueue(SCatalog* pCtg, STableMetaOutput *output, bool syncReq); int32_t ctgUpdateUserEnqueue(SCatalog* pCtg, SGetUserAuthRsp *pAuth, bool syncReq); int32_t ctgUpdateVgEpsetEnqueue(SCatalog* pCtg, char *dbFName, int32_t vgId, SEpSet* pEpSet); -int32_t ctgUpdateTbIndexEnqueue(SCatalog* pCtg, STableIndex *pIndex, bool syncOp); +int32_t ctgUpdateTbIndexEnqueue(SCatalog* pCtg, STableIndex **pIndex, bool syncOp); int32_t ctgMetaRentInit(SCtgRentMgmt *mgmt, uint32_t rentSec, int8_t type); int32_t ctgMetaRentAdd(SCtgRentMgmt *mgmt, void *meta, int64_t id, int32_t size); int32_t ctgMetaRentGet(SCtgRentMgmt *mgmt, void **res, uint32_t *num, int32_t size); int32_t ctgUpdateTbMetaToCache(SCatalog* pCtg, STableMetaOutput* pOut, bool syncReq); int32_t ctgStartUpdateThread(); int32_t ctgRelaunchGetTbMetaTask(SCtgTask *pTask); +void ctgReleaseVgInfoToCache(SCatalog* pCtg, SCtgDBCache *dbCache); +int32_t ctgReadTbIndexFromCache(SCatalog* pCtg, SName* pTableName, SArray** pRes); +int32_t ctgDropTbIndexEnqueue(SCatalog* pCtg, SName* pName, bool syncOp); +int32_t ctgOpDropTbIndex(SCtgCacheOperation *operation); +int32_t ctgOpUpdateTbIndex(SCtgCacheOperation *operation); @@ -516,7 +521,7 @@ int32_t ctgGetDBVgInfoFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, SBuildU int32_t ctgGetQnodeListFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, SArray *out, SCtgTask* pTask); int32_t ctgGetDBCfgFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, const char *dbFName, SDbCfgInfo *out, SCtgTask* pTask); int32_t ctgGetIndexInfoFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, const char *indexName, SIndexInfo *out, SCtgTask* pTask); -int32_t ctgGetTbIndexFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, SName* name, SArray** out, SCtgTask* pTask); +int32_t ctgGetTbIndexFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, SName *name, STableIndex* out, SCtgTask* pTask); int32_t ctgGetUdfInfoFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, const char *funcName, SFuncInfo *out, SCtgTask* pTask); int32_t ctgGetUserDbAuthFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, const char *user, SGetUserAuthRsp *out, SCtgTask* pTask); int32_t ctgGetTbMetaFromMnodeImpl(SCatalog* pCtg, SRequestConnInfo *pConn, char *dbFName, char* tbName, STableMetaOutput* out, SCtgTask* pTask); @@ -545,6 +550,7 @@ int32_t ctgUpdateMsgCtx(SCtgMsgCtx* pCtx, int32_t reqType, void* out, char* targ char *ctgTaskTypeStr(CTG_TASK_TYPE type); int32_t ctgUpdateSendTargetInfo(SMsgSendInfo *pMsgSendInfo, int32_t msgType, SCtgTask* pTask); int32_t ctgCloneTableIndex(SArray* pIndex, SArray** pRes); +void ctgFreeSTableIndex(void *info); extern SCatalogMgmt gCtgMgmt; diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index de74dc1c68..2ef7771c11 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -350,7 +350,7 @@ int32_t ctgChkAuth(SCatalog* pCtg, SRequestConnInfo *pConn, const char* user, co *pass = false; - CTG_ERR_RET(ctgChkAuthFromCache(pCtg, user, dbFName, type, &inCache, pass)); + CTG_ERR_RET(ctgChkAuthFromCache(pCtg, (char*)user, (char*)dbFName, type, &inCache, pass)); if (inCache) { return TSDB_CODE_SUCCESS; @@ -382,15 +382,38 @@ _return: return TSDB_CODE_SUCCESS; } -int32_t ctgGetTbIndex(SCatalog* pCtg, SRequestConnInfo *pConn, const SName* pTableName, SArray** pRes) { +int32_t ctgGetTbIndex(SCatalog* pCtg, SRequestConnInfo *pConn, SName* pTableName, SArray** pRes) { CTG_ERR_RET(ctgReadTbIndexFromCache(pCtg, pTableName, pRes)); if (*pRes) { return TSDB_CODE_SUCCESS; } - CTG_ERR_RET(ctgGetTbIndexFromMnode(pCtg, pConn, (SName*)pTableName, pRes, NULL)); + STableIndex *pIndex = taosMemoryCalloc(1, sizeof(STableIndex)); + if (NULL == pIndex) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + int32_t code = 0; + CTG_ERR_JRET(ctgGetTbIndexFromMnode(pCtg, pConn, (SName*)pTableName, pIndex, NULL)); + + SArray* pInfo = NULL; + CTG_ERR_JRET(ctgCloneTableIndex(pIndex->pIndex, &pInfo)); + + *pRes = pInfo; + + CTG_ERR_JRET(ctgUpdateTbIndexEnqueue(pCtg, &pIndex, false)); return TSDB_CODE_SUCCESS; + +_return: + + tFreeSTableIndexRsp(pIndex); + taosMemoryFree(pIndex); + + taosArrayDestroyEx(*pRes, tFreeSTableIndexInfo); + *pRes = NULL; + + CTG_RET(code); } @@ -416,7 +439,7 @@ int32_t ctgGetTbDistVgInfo(SCatalog* pCtg, SRequestConnInfo *pConn, SName* pTabl CTG_ERR_JRET(ctgGetDBVgInfo(pCtg, pConn, db, &dbCache, &vgInfo)); if (dbCache) { - vgHash = dbCache->vgInfo->vgHash; + vgHash = dbCache->vgCache.vgInfo->vgHash; } else { vgHash = vgInfo->vgHash; } @@ -640,9 +663,9 @@ int32_t catalogGetDBVgVersion(SCatalog* pCtg, const char* dbFName, int32_t* vers CTG_API_LEAVE(TSDB_CODE_SUCCESS); } - *version = dbCache->vgInfo->vgVersion; + *version = dbCache->vgCache.vgInfo->vgVersion; *dbId = dbCache->dbId; - *tableNum = dbCache->vgInfo->numOfTable; + *tableNum = dbCache->vgCache.vgInfo->numOfTable; ctgReleaseVgInfoToCache(pCtg, dbCache); @@ -969,7 +992,7 @@ int32_t catalogGetTableHashVgroup(SCatalog *pCtg, SRequestConnInfo *pConn, const SDBVgInfo *vgInfo = NULL; CTG_ERR_JRET(ctgGetDBVgInfo(pCtg, pConn, db, &dbCache, &vgInfo)); - CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, vgInfo ? vgInfo : dbCache->vgInfo, pTableName, pVgroup)); + CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, vgInfo ? vgInfo : dbCache->vgCache.vgInfo, pTableName, pVgroup)); _return: @@ -1177,36 +1200,11 @@ int32_t catalogGetTableIndex(SCatalog* pCtg, SRequestConnInfo *pConn, const SNam CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } - CTG_ERR_RET(ctgReadTbIndexFromCache(pCtg, pTableName, pRes)); - if (*pRes) { - CTG_API_LEAVE(TSDB_CODE_SUCCESS); - } - - STableIndex *pIndex = taosMemoryCalloc(1, sizeof(STableIndex)); - if (NULL == pIndex) { - CTG_API_LEAVE(TSDB_CODE_OUT_OF_MEMORY); - } - int32_t code = 0; - CTG_ERR_JRET(ctgGetTbIndexFromMnode(pCtg, pConn, (SName*)pTableName, pIndex, NULL)); - - SArray* pInfo = NULL; - CTG_ERR_JRET(ctgCloneTableIndex(pIndex->pIndex, &pInfo)); - - *pRes = pInfo; - - CTG_ERR_JRET(ctgUpdateTbIndexEnqueue(pCtg, pTableName, &pIndex, false)); - - CTG_API_LEAVE(code); + CTG_ERR_JRET(ctgGetTbIndex(pCtg, pConn, (SName*)pTableName, pRes)); _return: - tFreeSTableIndexRsp(pIndex); - taosMemoryFree(pIndex); - - taosArrayDestroyEx(*pRes, tFreeSTableIndexInfo); - *pRes = NULL; - CTG_API_LEAVE(code); } diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index 36e5b33521..b893201c53 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -685,7 +685,7 @@ int32_t ctgHandleGetTbMetaRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf * CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, dbFName, &dbCache)); if (NULL != dbCache) { SVgroupInfo vgInfo = {0}; - CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, dbCache->vgInfo, ctx->pName, &vgInfo)); + CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, dbCache->vgCache.vgInfo, ctx->pName, &vgInfo)); ctgDebug("will refresh tbmeta, not supposed to be stb, tbName:%s, flag:%d", tNameGetTableName(ctx->pName), ctx->flag); @@ -1016,7 +1016,7 @@ int32_t ctgAsyncRefreshTbMeta(SCtgTask *pTask) { CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, dbFName, &dbCache)); if (dbCache) { SVgroupInfo vgInfo = {0}; - CTG_ERR_RET(ctgGetVgInfoFromHashValue(pCtg, dbCache->vgInfo, ctx->pName, &vgInfo)); + CTG_ERR_RET(ctgGetVgInfoFromHashValue(pCtg, dbCache->vgCache.vgInfo, ctx->pName, &vgInfo)); ctgDebug("will refresh tbmeta, not supposed to be stb, tbName:%s, flag:%d", tNameGetTableName(ctx->pName), ctx->flag); @@ -1064,7 +1064,7 @@ int32_t ctgLaunchGetDbVgTask(SCtgTask *pTask) { CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, pCtx->dbFName, &dbCache)); if (NULL != dbCache) { - CTG_ERR_JRET(ctgGenerateVgList(pCtg, dbCache->vgInfo->vgHash, (SArray**)&pTask->res)); + CTG_ERR_JRET(ctgGenerateVgList(pCtg, dbCache->vgCache.vgInfo->vgHash, (SArray**)&pTask->res)); CTG_ERR_JRET(ctgHandleTaskEnd(pTask, 0)); } else { @@ -1098,7 +1098,7 @@ int32_t ctgLaunchGetTbHashTask(SCtgTask *pTask) { if (NULL == pTask->res) { CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } - CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, dbCache->vgInfo, pCtx->pName, (SVgroupInfo*)pTask->res)); + CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, dbCache->vgCache.vgInfo, pCtx->pName, (SVgroupInfo*)pTask->res)); CTG_ERR_JRET(ctgHandleTaskEnd(pTask, 0)); } else { @@ -1171,9 +1171,9 @@ int32_t ctgLaunchGetDbInfoTask(SCtgTask *pTask) { SDbInfo* pInfo = (SDbInfo*)pTask->res; CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, pCtx->dbFName, &dbCache)); if (NULL != dbCache) { - pInfo->vgVer = dbCache->vgInfo->vgVersion; + pInfo->vgVer = dbCache->vgCache.vgInfo->vgVersion; pInfo->dbId = dbCache->dbId; - pInfo->tbNum = dbCache->vgInfo->numOfTable; + pInfo->tbNum = dbCache->vgCache.vgInfo->numOfTable; } else { pInfo->vgVer = CTG_DEFAULT_INVALID_VERSION; } diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index f2502f5943..1fa61f1c59 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -64,8 +64,12 @@ SCtgOperation gCtgCacheOperation[CTG_OP_MAX] = { CTG_OP_UPDATE_TB_INDEX, "update tbIndex", ctgOpUpdateTbIndex - } - + }, + { + CTG_OP_DROP_TB_INDEX, + "drop tbIndex", + ctgOpDropTbIndex + } }; @@ -305,6 +309,7 @@ _return: int32_t ctgTbMetaExistInCache(SCatalog* pCtg, char *dbFName, char* tbName, int32_t *exist) { SCtgDBCache *dbCache = NULL; + SCtgTbCache *tbCache = NULL; ctgAcquireTbMetaFromCache(pCtg, dbFName, tbName, &dbCache, &tbCache); if (NULL == tbCache) { ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); @@ -383,7 +388,7 @@ _return: CTG_RET(code); } -int32_t ctgReadTbVerFromCache(SCatalog *pCtg, const SName *pTableName, int32_t *sver, int32_t *tver, int32_t *tbType, uint64_t *suid, +int32_t ctgReadTbVerFromCache(SCatalog *pCtg, SName *pTableName, int32_t *sver, int32_t *tver, int32_t *tbType, uint64_t *suid, char *stbName) { *sver = -1; *tver = -1; @@ -444,10 +449,10 @@ int32_t ctgReadTbVerFromCache(SCatalog *pCtg, const SName *pTableName, int32_t * } -int32_t ctgReadTbTypeFromCache(SCatalog* pCtg, const char* dbFName, const char *tableName, int32_t *tbType) { +int32_t ctgReadTbTypeFromCache(SCatalog* pCtg, char* dbFName, char *tableName, int32_t *tbType) { SCtgDBCache *dbCache = NULL; SCtgTbCache *tbCache = NULL; - ctgAcquireTbMetaFromCache(pCtg, dbFName, tableName, &dbCache, &tbCache); + CTG_ERR_RET(ctgAcquireTbMetaFromCache(pCtg, dbFName, tableName, &dbCache, &tbCache)); if (NULL == tbCache) { ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); return TSDB_CODE_SUCCESS; @@ -461,7 +466,7 @@ int32_t ctgReadTbTypeFromCache(SCatalog* pCtg, const char* dbFName, const char * return TSDB_CODE_SUCCESS; } -int32_t ctgReadTbIndexFromCache(SCatalog* pCtg, const SName* pTableName, SArray** pRes) { +int32_t ctgReadTbIndexFromCache(SCatalog* pCtg, SName* pTableName, SArray** pRes) { int32_t code = 0; SCtgDBCache *dbCache = NULL; SCtgTbCache *tbCache = NULL; @@ -485,7 +490,7 @@ _return: CTG_RET(code); } -int32_t ctgChkAuthFromCache(SCatalog* pCtg, const char* user, const char* dbFName, AUTH_TYPE type, bool *inCache, bool *pass) { +int32_t ctgChkAuthFromCache(SCatalog* pCtg, char* user, char* dbFName, AUTH_TYPE type, bool *inCache, bool *pass) { if (NULL == pCtg->userCache) { ctgDebug("empty user auth cache, user:%s", user); goto _return; @@ -862,7 +867,7 @@ int32_t ctgUpdateTbIndexEnqueue(SCatalog* pCtg, STableIndex **pIndex, bool syncO _return: - taosArrayDestroyEx(*pIndex, tFreeSTableIndexInfo); + taosArrayDestroyEx((*pIndex)->pIndex, tFreeSTableIndexInfo); taosMemoryFreeClear(*pIndex); taosMemoryFreeClear(msg); @@ -893,7 +898,6 @@ int32_t ctgDropTbIndexEnqueue(SCatalog* pCtg, SName* pName, bool syncOp) { _return: - taosArrayDestroyEx(pIndex, tFreeSTableIndexInfo); taosMemoryFreeClear(msg); CTG_RET(code); @@ -1172,7 +1176,7 @@ int32_t ctgRemoveDBFromCache(SCatalog* pCtg, SCtgDBCache *dbCache, const char* d CTG_LOCK(CTG_WRITE, &dbCache->dbLock); atomic_store_8(&dbCache->deleted, 1); - ctgRemoveStbRent(pCtg, &dbCache); + ctgRemoveStbRent(pCtg, dbCache); ctgFreeDbCache(dbCache); CTG_UNLOCK(CTG_WRITE, &dbCache->dbLock); @@ -1251,6 +1255,8 @@ int32_t ctgUpdateRentStbVersion(SCatalog *pCtg, char* dbFName, char* tbName, uin ctgDebug("db %s,%" PRIx64 " stb %s,%" PRIx64 " sver %d tver %d smaVer %d updated to stbRent", dbFName, dbId, tbName, suid, metaRent.sversion, metaRent.tversion, metaRent.smaVer); + + return TSDB_CODE_SUCCESS; } @@ -1330,32 +1336,40 @@ int32_t ctgWriteTbMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFNam int32_t ctgWriteTbIndexToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char* dbFName, char *tbName, STableIndex **index) { if (NULL == dbCache->tbCache) { + ctgFreeSTableIndex(*index); taosMemoryFreeClear(*index); ctgError("db is dropping, dbId:%"PRIx64, dbCache->dbId); CTG_ERR_RET(TSDB_CODE_CTG_DB_DROPPED); } STableIndex* pIndex = *index; + uint64_t suid = pIndex->suid; SCtgTbCache* pCache = taosHashGet(dbCache->tbCache, tbName, strlen(tbName)); if (NULL == pCache) { SCtgTbCache cache = {0}; cache.pIndex = pIndex; - if (taosHashPut(table->tbCache, tbName, strlen(tbName), &cache, sizeof(cache)) != 0) { + if (taosHashPut(dbCache->tbCache, tbName, strlen(tbName), &cache, sizeof(cache)) != 0) { + ctgFreeSTableIndex(*index); taosMemoryFreeClear(*index); ctgError("taosHashPut new tbCache failed, tbName:%s", tbName); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } *index = NULL; - ctgDebug("table %s index updated to cache, ver:%d, num:%d", tbName, pIndex->version, taosArrayGetSize(pIndex->pIndex)); - - CTG_ERR_RET(ctgUpdateRentStbVersion(pCtg, dbFName, tbName, dbCache->dbId, pIndex->suid, pCache)); + ctgDebug("table %s index updated to cache, ver:%d, num:%d", tbName, pIndex->version, (int32_t)taosArrayGetSize(pIndex->pIndex)); + if (suid) { + CTG_ERR_RET(ctgUpdateRentStbVersion(pCtg, dbFName, tbName, dbCache->dbId, pIndex->suid, pCache)); + } + return TSDB_CODE_SUCCESS; } if (pCache->pIndex) { + if (0 == suid) { + suid = pCache->pIndex->suid; + } taosArrayDestroyEx(pCache->pIndex->pIndex, tFreeSTableIndexInfo); taosMemoryFreeClear(pCache->pIndex); } @@ -1363,9 +1377,11 @@ int32_t ctgWriteTbIndexToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char* dbFNa pCache->pIndex = pIndex; *index = NULL; - ctgDebug("table %s index updated to cache, ver:%d, num:%d", tbName, pIndex->version, taosArrayGetSize(pIndex->pIndex)); + ctgDebug("table %s index updated to cache, ver:%d, num:%d", tbName, pIndex->version, (int32_t)taosArrayGetSize(pIndex->pIndex)); - CTG_ERR_RET(ctgUpdateRentStbVersion(pCtg, dbFName, tbName, dbCache->dbId, pIndex->suid, pCache)); + if (suid) { + CTG_ERR_RET(ctgUpdateRentStbVersion(pCtg, dbFName, tbName, dbCache->dbId, suid, pCache)); + } return TSDB_CODE_SUCCESS; } @@ -1390,6 +1406,7 @@ int32_t ctgOpUpdateVgroup(SCtgCacheOperation *operation) { SCtgUpdateVgMsg *msg = operation->data; SDBVgInfo* dbInfo = msg->dbInfo; char* dbFName = msg->dbFName; + SCatalog* pCtg = msg->pCtg; if (NULL == dbInfo->vgHash) { return TSDB_CODE_SUCCESS; @@ -1611,7 +1628,7 @@ int32_t ctgOpDropTbMeta(SCtgCacheOperation *operation) { } if (dbCache->dbId != msg->dbId) { - ctgDebug("dbId %" PRIx64 " not match with curId %"PRIx64", dbFName:%s, tbName:%s"msg->dbId, dbCache->dbId, msg->dbFName, msg->tbName); + ctgDebug("dbId %" PRIx64 " not match with curId %"PRIx64", dbFName:%s, tbName:%s", msg->dbId, dbCache->dbId, msg->dbFName, msg->tbName); return TSDB_CODE_SUCCESS; } @@ -1746,9 +1763,39 @@ int32_t ctgOpUpdateTbIndex(SCtgCacheOperation *operation) { SCtgDBCache *dbCache = NULL; CTG_ERR_JRET(ctgGetAddDBCache(pCtg, pIndex->dbFName, 0, &dbCache)); - if (NULL == dbCache) { - CTG_ERR_JRET(code); + + CTG_ERR_JRET(ctgWriteTbIndexToCache(pCtg, dbCache, pIndex->dbFName, pIndex->tbName, &pIndex)); + +_return: + + if (pIndex) { + taosArrayDestroyEx(pIndex->pIndex, tFreeSTableIndexInfo); + taosMemoryFreeClear(pIndex); } + + taosMemoryFreeClear(msg); + + CTG_RET(code); +} + +int32_t ctgOpDropTbIndex(SCtgCacheOperation *operation) { + int32_t code = 0; + SCtgDropTbIndexMsg *msg = operation->data; + SCatalog* pCtg = msg->pCtg; + SCtgDBCache *dbCache = NULL; + + CTG_ERR_JRET(ctgGetDBCache(pCtg, msg->dbFName, &dbCache)); + if (NULL == dbCache) { + return TSDB_CODE_SUCCESS; + } + + STableIndex* pIndex = taosMemoryCalloc(1, sizeof(STableIndex)); + if (NULL == pIndex) { + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } + strcpy(pIndex->tbName, msg->tbName); + strcpy(pIndex->dbFName, msg->dbFName); + pIndex->version = -1; CTG_ERR_JRET(ctgWriteTbIndexToCache(pCtg, dbCache, pIndex->dbFName, pIndex->tbName, &pIndex)); diff --git a/source/libs/catalog/src/ctgDbg.c b/source/libs/catalog/src/ctgDbg.c index 52f29e3acb..2403dc3c5a 100644 --- a/source/libs/catalog/src/ctgDbg.c +++ b/source/libs/catalog/src/ctgDbg.c @@ -266,11 +266,11 @@ int32_t ctgdGetStatNum(char *option, void *res) { } int32_t ctgdGetTbMetaNum(SCtgDBCache *dbCache) { - return dbCache->table.tbCache ? (int32_t)taosHashGetSize(dbCache->table.tbCache) : 0; + return dbCache->tbCache ? (int32_t)taosHashGetSize(dbCache->tbCache) : 0; } int32_t ctgdGetStbNum(SCtgDBCache *dbCache) { - return dbCache->table.stbCache ? (int32_t)taosHashGetSize(dbCache->table.stbCache) : 0; + return dbCache->stbCache ? (int32_t)taosHashGetSize(dbCache->stbCache) : 0; } int32_t ctgdGetRentNum(SCtgRentMgmt *rent) { @@ -363,17 +363,17 @@ void ctgdShowDBCache(SCatalog* pCtg, SHashObj *dbHash) { dbFName = taosHashGetKey(pIter, &len); - int32_t metaNum = dbCache->table.tbCache ? taosHashGetSize(dbCache->table.tbCache) : 0; - int32_t stbNum = dbCache->table.stbCache ? taosHashGetSize(dbCache->table.stbCache) : 0; + int32_t metaNum = dbCache->tbCache ? taosHashGetSize(dbCache->tbCache) : 0; + int32_t stbNum = dbCache->stbCache ? taosHashGetSize(dbCache->stbCache) : 0; int32_t vgVersion = CTG_DEFAULT_INVALID_VERSION; int32_t hashMethod = -1; int32_t vgNum = 0; - if (dbCache->vgInfo) { - vgVersion = dbCache->vgInfo->vgVersion; - hashMethod = dbCache->vgInfo->hashMethod; - if (dbCache->vgInfo->vgHash) { - vgNum = taosHashGetSize(dbCache->vgInfo->vgHash); + if (dbCache->vgCache.vgInfo) { + vgVersion = dbCache->vgCache.vgInfo->vgVersion; + hashMethod = dbCache->vgCache.vgInfo->hashMethod; + if (dbCache->vgCache.vgInfo->vgHash) { + vgNum = taosHashGetSize(dbCache->vgCache.vgInfo->vgHash); } } diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index f0ed60a689..1cb2b99a67 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -44,6 +44,16 @@ char *ctgTaskTypeStr(CTG_TASK_TYPE type) { } } +void ctgFreeSTableIndex(void *info) { + if (NULL == info) { + return; + } + + STableIndex *pInfo = (STableIndex *)info; + + taosArrayDestroyEx(pInfo->pIndex, tFreeSTableIndexInfo); +} + void ctgFreeSMetaData(SMetaData* pData) { taosArrayDestroy(pData->pTableMeta); pData->pTableMeta = NULL; From f47a28fe8a487329a17686f762a7e54c38f0c373 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Mon, 13 Jun 2022 15:47:43 +0800 Subject: [PATCH 11/78] enh(sync): add leader transfer --- include/libs/sync/sync.h | 1 + include/libs/sync/syncTools.h | 1 + source/libs/sync/src/syncMain.c | 68 ++++++++++++++++++++++---- source/libs/sync/test/syncTestTool.cpp | 40 +++++++++++---- 4 files changed, 92 insertions(+), 18 deletions(-) diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index 2673bc382b..a369b81d26 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -111,6 +111,7 @@ typedef struct SSyncFSM { void (*FpRestoreFinishCb)(struct SSyncFSM* pFsm); void (*FpReConfigCb)(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SReConfigCbMeta cbMeta); + void (*FpLeaderTransferCb)(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta); int32_t (*FpGetSnapshot)(struct SSyncFSM* pFsm, SSnapshot* pSnapshot); diff --git a/include/libs/sync/syncTools.h b/include/libs/sync/syncTools.h index 68a33d48cb..fbf124bf47 100644 --- a/include/libs/sync/syncTools.h +++ b/include/libs/sync/syncTools.h @@ -467,6 +467,7 @@ typedef struct SyncLeaderTransfer { SRaftId srcId; SRaftId destId; */ + SNodeInfo newNodeInfo; SRaftId newLeaderId; } SyncLeaderTransfer; diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 82fde80be2..904cdae94e 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -35,7 +35,7 @@ #include "syncVoteMgr.h" #include "tref.h" -bool gRaftDetailLog = true; +bool gRaftDetailLog = false; static int32_t tsNodeRefId = -1; @@ -183,13 +183,21 @@ int32_t syncReconfigBuild(int64_t rid, const SSyncCfg* pNewCfg, SRpcMsg* pRpcMsg int32_t ret = 0; bool IamInNew = false; for (int i = 0; i < pNewCfg->replicaNum; ++i) { - SRaftId newId; - newId.addr = syncUtilAddr2U64((pNewCfg->nodeInfo)[i].nodeFqdn, (pNewCfg->nodeInfo)[i].nodePort); - newId.vgId = pSyncNode->vgId; - if (syncUtilSameId(&(pSyncNode->myRaftId), &newId)) { + if (strcmp((pNewCfg->nodeInfo)[i].nodeFqdn, pSyncNode->myNodeInfo.nodeFqdn) == 0 && + (pNewCfg->nodeInfo)[i].nodePort == pSyncNode->myNodeInfo.nodePort) { IamInNew = true; } + + /* + SRaftId newId; + newId.addr = syncUtilAddr2U64((pNewCfg->nodeInfo)[i].nodeFqdn, (pNewCfg->nodeInfo)[i].nodePort); + newId.vgId = pSyncNode->vgId; + if (syncUtilSameId(&(pSyncNode->myRaftId), &newId)) { + IamInNew = true; + } + */ } + if (!IamInNew) { taosReleaseRef(tsNodeRefId, pSyncNode->rid); return TAOS_SYNC_NOT_IN_NEW_CONFIG; @@ -267,7 +275,7 @@ int32_t syncLeaderTransfer(int64_t rid) { } ASSERT(rid == pSyncNode->rid); - if (pSyncNode->peersNum > 0) { + if (pSyncNode->peersNum == 0) { taosReleaseRef(tsNodeRefId, pSyncNode->rid); return TAOS_SYNC_OTHER_ERROR; } @@ -296,6 +304,7 @@ int32_t syncLeaderTransferTo(int64_t rid, SNodeInfo newLeader) { SyncLeaderTransfer* pMsg = syncLeaderTransferBuild(pSyncNode->vgId); pMsg->newLeaderId.addr = syncUtilAddr2U64(newLeader.nodeFqdn, newLeader.nodePort); pMsg->newLeaderId.vgId = pSyncNode->vgId; + pMsg->newNodeInfo = newLeader; ASSERT(pMsg != NULL); SRpcMsg rpcMsg = {0}; syncLeaderTransfer2RpcMsg(pMsg, &rpcMsg); @@ -1887,10 +1896,51 @@ const char* syncStr(ESyncState state) { } static int32_t syncDoLeaderTransfer(SSyncNode* ths, SRpcMsg* pRpcMsg, SSyncRaftEntry* pEntry) { - SyncLeaderTransfer* pSyncLeaderTransfer; - if (syncUtilSameId(&(pSyncLeaderTransfer->newLeaderId), &(ths->myRaftId))) { + SyncLeaderTransfer* pSyncLeaderTransfer = syncLeaderTransferFromRpcMsg2(pRpcMsg); + +/* + char host[128]; + uint16_t port; + syncUtilU642Addr(pSyncLeaderTransfer->newLeaderId.addr, host, sizeof(host), &port); + sDebug("vgId:%d sync event, maybe leader transfer to %s:%d %lu", ths->vgId, host, port, + pSyncLeaderTransfer->newLeaderId.addr); +*/ + + sDebug("vgId:%d sync event, begin leader transfer", ths->vgId); + + if (strcmp(pSyncLeaderTransfer->newNodeInfo.nodeFqdn, ths->myNodeInfo.nodeFqdn) == 0 && pSyncLeaderTransfer->newNodeInfo.nodePort == ths->myNodeInfo.nodePort) { + sDebug("vgId:%d sync event, maybe leader transfer to %s:%d %lu", ths->vgId, pSyncLeaderTransfer->newNodeInfo.nodeFqdn, pSyncLeaderTransfer->newNodeInfo.nodePort, + pSyncLeaderTransfer->newLeaderId.addr); + + // reset elect timer now! + int32_t electMS = 1; + int32_t ret = syncNodeRestartElectTimer(ths, electMS); + ASSERT(ret == 0); } + +/* + if (syncUtilSameId(&(pSyncLeaderTransfer->newLeaderId), &(ths->myRaftId))) { + // reset elect timer now! + int32_t electMS = 1; + int32_t ret = syncNodeRestartElectTimer(ths, electMS); + ASSERT(ret == 0); + } +*/ + if (ths->pFsm->FpLeaderTransferCb != NULL) { + SFsmCbMeta cbMeta; + cbMeta.code = 0; + cbMeta.currentTerm = ths->pRaftStore->currentTerm; + cbMeta.flag = 0; + cbMeta.index = pEntry->index; + cbMeta.isWeak = pEntry->isWeak; + cbMeta.seqNum = pEntry->seqNum; + cbMeta.state = ths->state; + cbMeta.term = pEntry->term; + ths->pFsm->FpLeaderTransferCb(ths->pFsm, pRpcMsg, cbMeta); + } + + syncLeaderTransferDestroy(pSyncLeaderTransfer); return 0; } @@ -1992,7 +2042,7 @@ int32_t syncNodeCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endIndex, ASSERT(code == 0); } - // config change + // leader transfer if (pEntry->originalRpcType == TDMT_SYNC_LEADER_TRANSFER) { code = syncDoLeaderTransfer(ths, &rpcMsg, pEntry); ASSERT(code == 0); diff --git a/source/libs/sync/test/syncTestTool.cpp b/source/libs/sync/test/syncTestTool.cpp index 0c8b26e9d9..ebcd7368cc 100644 --- a/source/libs/sync/test/syncTestTool.cpp +++ b/source/libs/sync/test/syncTestTool.cpp @@ -153,6 +153,16 @@ void ReConfigCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SReConfigCbMeta cbMe taosMemoryFree(s); } +void LeaderTransferCb(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta) { + char logBuf[256] = {0}; + snprintf(logBuf, sizeof(logBuf), + "==callback== ==LeaderTransferCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s, flag:%lu, term:%lu " + "currentTerm:%lu \n", + pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state), + cbMeta.flag, cbMeta.term, cbMeta.currentTerm); + syncRpcMsgLog2(logBuf, (SRpcMsg*)pMsg); +} + SSyncFSM* createFsm() { SSyncFSM* pFsm = (SSyncFSM*)taosMemoryMalloc(sizeof(SSyncFSM)); memset(pFsm, 0, sizeof(*pFsm)); @@ -172,6 +182,8 @@ SSyncFSM* createFsm() { pFsm->FpSnapshotStopWrite = SnapshotStopWrite; pFsm->FpSnapshotDoWrite = SnapshotDoWrite; + pFsm->FpLeaderTransferCb = LeaderTransferCb; + return pFsm; } @@ -277,7 +289,8 @@ void usage(char* exe) { printf( "usage: %s replicaNum(1-5) myIndex(0-..) enableSnapshot(0/1) lastApplyIndex(>=-1) lastApplyTerm(>=0) " "writeRecordNum(>=0) " - "isStandBy(0/1) isConfigChange(0-5) iterTimes(>=0) finishLastApplyIndex(>=-1) finishLastApplyTerm(>=0) \n", + "isStandBy(0/1) isConfigChange(0-5) iterTimes(>=0) finishLastApplyIndex(>=-1) finishLastApplyTerm(>=0) " + "leaderTransfer(0/1) \n", exe); } @@ -294,9 +307,9 @@ SRpcMsg* createRpcMsg(int i, int count, int myIndex) { int main(int argc, char** argv) { sprintf(tsTempDir, "%s", "."); tsAsyncLog = 0; - sDebugFlag = DEBUG_SCREEN + DEBUG_FILE + DEBUG_TRACE + DEBUG_INFO + DEBUG_ERROR; + sDebugFlag = DEBUG_SCREEN + DEBUG_FILE + DEBUG_TRACE + DEBUG_INFO + DEBUG_ERROR + DEBUG_DEBUG; - if (argc != 12) { + if (argc != 13) { usage(argv[0]); exit(-1); } @@ -312,12 +325,14 @@ int main(int argc, char** argv) { int32_t iterTimes = atoi(argv[9]); int32_t finishLastApplyIndex = atoi(argv[10]); int32_t finishLastApplyTerm = atoi(argv[11]); + int32_t leaderTransfer = atoi(argv[12]); - sTrace( + sInfo( "args: replicaNum:%d, myIndex:%d, enableSnapshot:%d, lastApplyIndex:%d, lastApplyTerm:%d, writeRecordNum:%d, " - "isStandBy:%d, isConfigChange:%d, iterTimes:%d, finishLastApplyIndex:%d, finishLastApplyTerm:%d", + "isStandBy:%d, isConfigChange:%d, iterTimes:%d, finishLastApplyIndex:%d, finishLastApplyTerm:%d, " + "leaderTransfer:%d", replicaNum, myIndex, enableSnapshot, lastApplyIndex, lastApplyTerm, writeRecordNum, isStandBy, isConfigChange, - iterTimes, finishLastApplyIndex, finishLastApplyTerm); + iterTimes, finishLastApplyIndex, finishLastApplyTerm, leaderTransfer); // check parameter assert(replicaNum >= 1 && replicaNum <= 5); @@ -363,24 +378,31 @@ int main(int argc, char** argv) { //--------------------------- int32_t alreadySend = 0; + int32_t leaderTransferWait = 0; while (1) { char* simpleStr = syncNode2SimpleStr(pSyncNode); + leaderTransferWait++; + if (leaderTransferWait == 7) { + sTrace("begin leader transfer ..."); + int32_t ret = syncLeaderTransfer(rid); + } + if (alreadySend < writeRecordNum) { SRpcMsg* pRpcMsg = createRpcMsg(alreadySend, writeRecordNum, myIndex); int32_t ret = syncPropose(rid, pRpcMsg, false); if (ret == TAOS_SYNC_PROPOSE_NOT_LEADER) { - sTrace("%s value%d write not leader", simpleStr, alreadySend); + sTrace("%s value%d write not leader, leaderTransferWait:%d", simpleStr, alreadySend, leaderTransferWait); } else { assert(ret == 0); - sTrace("%s value%d write ok", simpleStr, alreadySend); + sTrace("%s value%d write ok, leaderTransferWait:%d", simpleStr, alreadySend, leaderTransferWait); } alreadySend++; rpcFreeCont(pRpcMsg->pCont); taosMemoryFree(pRpcMsg); } else { - sTrace("%s", simpleStr); + sTrace("%s, leaderTransferWait:%d", simpleStr, leaderTransferWait); } taosMsleep(1000); From bb89ceb6bd0bc4ee03d2c8ae1f047194ad581e06 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 13 Jun 2022 18:46:02 +0800 Subject: [PATCH 12/78] change stb hash value to stbName --- source/client/inc/clientInt.h | 2 +- source/libs/catalog/src/ctgCache.c | 127 ++++++++++++++++++++++++----- source/libs/catalog/src/ctgDbg.c | 2 +- 3 files changed, 108 insertions(+), 23 deletions(-) diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 6aa83e9575..d72aa39e4c 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -45,7 +45,7 @@ extern "C" { #define ERROR_MSG_BUF_DEFAULT_SIZE 512 #define HEARTBEAT_INTERVAL 1500 // ms -#define SYNC_ON_TOP_OF_ASYNC 1 +#define SYNC_ON_TOP_OF_ASYNC 0 enum { RES_TYPE__QUERY = 1, diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 1fa61f1c59..b938dff932 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -230,6 +230,7 @@ _return: int32_t ctgAcquireTbMetaFromCache(SCatalog* pCtg, char *dbFName, char* tbName, SCtgDBCache **pDb, SCtgTbCache** pTb) { SCtgDBCache *dbCache = NULL; + SCtgTbCache* pCache = NULL; ctgAcquireDBCache(pCtg, dbFName, &dbCache); if (NULL == dbCache) { ctgDebug("db %s not in cache", dbFName); @@ -237,7 +238,7 @@ int32_t ctgAcquireTbMetaFromCache(SCatalog* pCtg, char *dbFName, char* tbName, S } int32_t sz = 0; - SCtgTbCache* pCache = taosHashAcquire(dbCache->tbCache, tbName, strlen(tbName)); + pCache = taosHashAcquire(dbCache->tbCache, tbName, strlen(tbName)); if (NULL == pCache) { ctgDebug("tb %s not in cache, dbFName:%s", tbName, dbFName); goto _return; @@ -267,6 +268,54 @@ _return: return TSDB_CODE_SUCCESS; } +int32_t ctgAcquireStbMetaFromCache(SCatalog* pCtg, char *dbFName, uint64_t suid, SCtgDBCache **pDb, SCtgTbCache** pTb) { + SCtgDBCache* dbCache = NULL; + SCtgTbCache* pCache = NULL; + ctgAcquireDBCache(pCtg, dbFName, &dbCache); + if (NULL == dbCache) { + ctgDebug("db %s not in cache", dbFName); + goto _return; + } + + int32_t sz = 0; + char* stName = taosHashAcquire(dbCache->stbCache, &suid, sizeof(suid)); + if (NULL == stName) { + ctgDebug("stb %" PRIx64 " not in cache, dbFName:%s", suid, dbFName); + goto _return; + } + + pCache = taosHashAcquire(dbCache->tbCache, stName, strlen(stName)); + if (NULL == pCache) { + ctgDebug("stb %" PRIx64 " name %s not in cache, dbFName:%s", suid, stName, dbFName); + taosHashRelease(dbCache->stbCache, stName); + goto _return; + } + + CTG_LOCK(CTG_READ, &pCache->metaLock); + if (NULL == pCache->pMeta) { + ctgDebug("stb %" PRIx64 " meta not in cache, dbFName:%s", suid, dbFName); + goto _return; + } + + *pDb = dbCache; + *pTb = pCache; + + ctgDebug("stb %" PRIx64 " meta got in cache, dbFName:%s", suid, dbFName); + + CTG_CACHE_STAT_INC(tbMetaHitNum, 1); + + return TSDB_CODE_SUCCESS; + +_return: + + ctgReleaseTbMetaToCache(pCtg, dbCache, pCache); + + CTG_CACHE_STAT_INC(tbMetaMissNum, 1); + + return TSDB_CODE_SUCCESS; +} + + int32_t ctgAcquireTbIndexFromCache(SCatalog* pCtg, char *dbFName, char* tbName, SCtgDBCache **pDb, SCtgTbCache** pTb) { SCtgDBCache *dbCache = NULL; ctgAcquireDBCache(pCtg, dbFName, &dbCache); @@ -348,31 +397,61 @@ int32_t ctgReadTbMetaFromCache(SCatalog* pCtg, SCtgTbMetaCtx* ctx, STableMeta** ctx->tbInfo.dbId = dbCache->dbId; ctx->tbInfo.suid = tbMeta->suid; ctx->tbInfo.tbType = tbMeta->tableType; - + if (tbMeta->tableType != TSDB_CHILD_TABLE) { + int32_t metaSize = CTG_META_SIZE(tbMeta); + *pTableMeta = taosMemoryCalloc(1, metaSize); + if (NULL == *pTableMeta) { + ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } + + memcpy(*pTableMeta, tbMeta, metaSize); + ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); ctgDebug("Got tb %s meta from cache, type:%d, dbFName:%s", ctx->pName->tname, tbMeta->tableType, dbFName); return TSDB_CODE_SUCCESS; } + + // PROCESS FOR CHILD TABLE - STableMeta **stbMeta = taosHashGet(dbCache->stbCache, &tbMeta->suid, sizeof(tbMeta->suid)); - if (NULL == stbMeta || NULL == *stbMeta) { - ctgError("stb not in stbCache, suid:%"PRIx64, tbMeta->suid); - goto _return; + int32_t metaSize = sizeof(SCTableMeta); + *pTableMeta = taosMemoryCalloc(1, metaSize); + if (NULL == *pTableMeta) { + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } - if ((*stbMeta)->suid != tbMeta->suid) { - ctgError("stb suid %" PRIx64 " in stbCache mis-match, expected suid:%"PRIx64 , (*stbMeta)->suid, tbMeta->suid); + memcpy(*pTableMeta, tbMeta, metaSize); + + ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); + ctgDebug("Got ctb %s meta from cache, will continue to get its stb meta, type:%d, dbFName:%s", + ctx->pName->tname, ctx->tbInfo.tbType, dbFName); + + dbCache = NULL; + tbCache = NULL; + ctgAcquireStbMetaFromCache(pCtg, dbFName, ctx->tbInfo.suid, &dbCache, &tbCache); + if (NULL == tbCache) { + ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); + taosMemoryFreeClear(*pTableMeta); + ctgDebug("stb %" PRIx64 " meta not in cache", ctx->tbInfo.suid); + return TSDB_CODE_SUCCESS; + } + + STableMeta* stbMeta = tbCache->pMeta; + if (stbMeta->suid != ctx->tbInfo.suid) { + ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); + ctgError("stb suid %" PRIx64 " in stbCache mis-match, expected suid:%"PRIx64 , stbMeta->suid, ctx->tbInfo.suid); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } - int32_t metaSize = CTG_META_SIZE(*stbMeta); + metaSize = CTG_META_SIZE(stbMeta); *pTableMeta = taosMemoryRealloc(*pTableMeta, metaSize); if (NULL == *pTableMeta) { + ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); } - memcpy(&(*pTableMeta)->sversion, &(*stbMeta)->sversion, metaSize - sizeof(SCTableMeta)); + memcpy(&(*pTableMeta)->sversion, &stbMeta->sversion, metaSize - sizeof(SCTableMeta)); ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); @@ -419,27 +498,33 @@ int32_t ctgReadTbVerFromCache(SCatalog *pCtg, SName *pTableName, int32_t *sver, return TSDB_CODE_SUCCESS; } - STableMeta **stbMeta = taosHashGet(dbCache->stbCache, suid, sizeof(*suid)); - if (NULL == stbMeta || NULL == *stbMeta) { + // PROCESS FOR CHILD TABLE + + ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); + ctgDebug("Got ctb %s ver from cache, will continue to get its stb ver, dbFName:%s", pTableName->tname, dbFName); + + ctgAcquireStbMetaFromCache(pCtg, dbFName, *suid, &dbCache, &tbCache); + if (NULL == tbCache) { ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); - ctgDebug("stb not in stbCache, suid:%" PRIx64, *suid); + ctgDebug("stb %" PRIx64 " meta not in cache", *suid); return TSDB_CODE_SUCCESS; } - - if ((*stbMeta)->suid != *suid) { + + STableMeta* stbMeta = tbCache->pMeta; + if (stbMeta->suid != *suid) { ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); - ctgError("stb suid %" PRIx64 " in stbCache mis-match, expected suid:%" PRIx64 , (*stbMeta)->suid, *suid); + ctgError("stb suid %" PRIx64 " in stbCache mis-match, expected suid:%" PRIx64 , stbMeta->suid, *suid); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } size_t nameLen = 0; - char *name = taosHashGetKey(*stbMeta, &nameLen); + char *name = taosHashGetKey(tbCache, &nameLen); strncpy(stbName, name, nameLen); stbName[nameLen] = 0; - *sver = (*stbMeta)->sversion; - *tver = (*stbMeta)->tversion; + *sver = stbMeta->sversion; + *tver = stbMeta->tversion; ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); @@ -1320,14 +1405,14 @@ int32_t ctgWriteTbMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFNam return TSDB_CODE_SUCCESS; } - if (origSuid != meta->suid && taosHashPut(dbCache->stbCache, &meta->suid, sizeof(meta->suid), &pCache, POINTER_BYTES) != 0) { + if (origSuid != meta->suid && taosHashPut(dbCache->stbCache, &meta->suid, sizeof(meta->suid), tbName, strlen(tbName) + 1) != 0) { ctgError("taosHashPut to stable cache failed, suid:%"PRIx64, meta->suid); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } CTG_CACHE_STAT_INC(stblNum, 1); - ctgDebug("stb updated to stbCache, dbFName:%s, tbName:%s, tbType:%d", dbFName, tbName, meta->tableType); + ctgDebug("stb %" PRIx64 " updated to cache, dbFName:%s, tbName:%s, tbType:%d", meta->suid, dbFName, tbName, meta->tableType); CTG_ERR_RET(ctgUpdateRentStbVersion(pCtg, dbFName, tbName, dbId, meta->suid, pCache)); diff --git a/source/libs/catalog/src/ctgDbg.c b/source/libs/catalog/src/ctgDbg.c index 2403dc3c5a..cd61abf6cc 100644 --- a/source/libs/catalog/src/ctgDbg.c +++ b/source/libs/catalog/src/ctgDbg.c @@ -19,7 +19,7 @@ #include "catalogInt.h" extern SCatalogMgmt gCtgMgmt; -SCtgDebug gCTGDebug = {.apiEnable = true}; +SCtgDebug gCTGDebug = {.apiEnable = true, .lockEnable = true}; void ctgdUserCallback(SMetaData* pResult, void* param, int32_t code) { ASSERT(*(int32_t*)param == 1); From 6d952626882dbdc66a8c4cb1572b1ca69bf9330d Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 13 Jun 2022 19:11:28 +0800 Subject: [PATCH 13/78] enh(query): add first/last function distributed implementation --- include/libs/function/functionMgt.h | 2 + source/libs/function/inc/builtinsimpl.h | 1 + source/libs/function/src/builtins.c | 57 +++++++++++++++++++++++++ source/libs/function/src/builtinsimpl.c | 4 ++ 4 files changed, 64 insertions(+) diff --git a/include/libs/function/functionMgt.h b/include/libs/function/functionMgt.h index e86d643c0d..f01e734fde 100644 --- a/include/libs/function/functionMgt.h +++ b/include/libs/function/functionMgt.h @@ -138,6 +138,8 @@ typedef enum EFunctionType { FUNCTION_TYPE_TOP_MERGE, FUNCTION_TYPE_BOTTOM_PARTIAL, FUNCTION_TYPE_BOTTOM_MERGE, + FUNCTION_TYPE_FIRST_PARTIAL, + FUNCTION_TYPE_FIRST_MERGE, // user defined funcion FUNCTION_TYPE_UDF = 10000 diff --git a/source/libs/function/inc/builtinsimpl.h b/source/libs/function/inc/builtinsimpl.h index a528041964..f3bcad8ede 100644 --- a/source/libs/function/inc/builtinsimpl.h +++ b/source/libs/function/inc/builtinsimpl.h @@ -97,6 +97,7 @@ int32_t lastFunction(SqlFunctionCtx *pCtx); int32_t firstLastFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); int32_t firstCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx); int32_t lastCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx); +int32_t getFirstLastInfoSize(int32_t resBytes); bool getTopBotFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv); bool getTopBotMergeFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv); diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 23940a415e..a2849d601e 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -954,6 +954,41 @@ static int32_t translateFirstLast(SFunctionNode* pFunc, char* pErrBuf, int32_t l return TSDB_CODE_SUCCESS; } +static int32_t translateFirstLastImpl(SFunctionNode* pFunc, char* pErrBuf, int32_t len, bool isPartial) { + // first(col_list) will be rewritten as first(col) + if (1 != LIST_LENGTH(pFunc->pParameterList)) { + return TSDB_CODE_SUCCESS; + } + + SNode* pPara = nodesListGetNode(pFunc->pParameterList, 0); + uint8_t paraType = ((SExprNode*)pPara)->resType.type; + int32_t paraBytes = ((SExprNode*)pPara)->resType.bytes; + if (isPartial) { + if (QUERY_NODE_COLUMN != nodeType(pPara)) { + return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, + "The parameters of first/last can only be columns"); + } + + pFunc->node.resType = (SDataType){.bytes = getFirstLastInfoSize(paraBytes) + VARSTR_HEADER_SIZE, + .type = TSDB_DATA_TYPE_BINARY}; + } else { + if (TSDB_DATA_TYPE_BINARY != paraType) { + return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); + } + + pFunc->node.resType = ((SExprNode*)pPara)->resType; + } + return TSDB_CODE_SUCCESS; +} + +static int32_t translateFirstLastPartial(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { + return translateFirstLastImpl(pFunc, pErrBuf, len, true); +} + +static int32_t translateFirstLastMerge(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { + return translateFirstLastImpl(pFunc, pErrBuf, len, false); +} + static int32_t translateUnique(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { if (1 != LIST_LENGTH(pFunc->pParameterList)) { return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); @@ -1706,6 +1741,28 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .finalizeFunc = firstLastFinalize, .combineFunc = firstCombine, }, + { + .name = "_first_partial", + .type = FUNCTION_TYPE_FIRST_PARTIAL, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_TIMELINE_FUNC, + .translateFunc = translateFirstLastPartial, + .getEnvFunc = getFirstLastFuncEnv, + .initFunc = functionSetup, + .processFunc = firstFunction, + .finalizeFunc = firstLastFinalize, + .combineFunc = firstCombine, + }, + { + .name = "_first_merge", + .type = FUNCTION_TYPE_FIRST_MERGE, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_TIMELINE_FUNC, + .translateFunc = translateFirstLastMerge, + .getEnvFunc = getFirstLastFuncEnv, + .initFunc = functionSetup, + .processFunc = firstFunction, + .finalizeFunc = firstLastFinalize, + .combineFunc = firstCombine, + }, { .name = "last", .type = FUNCTION_TYPE_LAST, diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 3d64f82eba..44418685b0 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -2244,6 +2244,10 @@ int32_t apercentileCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) return TSDB_CODE_SUCCESS; } +int32_t getFirstLastInfoSize(int32_t resBytes) { + return resBytes + sizeof(int64_t); +} + bool getFirstLastFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) { SColumnNode* pNode = nodesListGetNode(pFunc->pParameterList, 0); pEnv->calcMemSize = pNode->node.resType.bytes + sizeof(int64_t); From d1b149a9f87c134ee47cc5b81d9c43e688ae43ea Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Mon, 13 Jun 2022 19:35:56 +0800 Subject: [PATCH 14/78] fix: adjust the initial array size of fs --- source/dnode/vnode/src/tsdb/tsdbCommit.c | 2 ++ source/dnode/vnode/src/tsdb/tsdbFS.c | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 40a23447b7..4dae9bdeab 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -110,6 +110,8 @@ int32_t tsdbBegin(STsdb *pTsdb) { } int32_t tsdbCommit(STsdb *pTsdb) { + if (!pTsdb) return 0; + int32_t code = 0; SCommitH commith = {0}; SDFileSet *pSet = NULL; diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index f1941a3bad..055b6c62de 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -20,6 +20,7 @@ extern const char *TSDB_LEVEL_DNAME[]; typedef enum { TSDB_TXN_TEMP_FILE = 0, TSDB_TXN_CURR_FILE } TSDB_TXN_FILE_T; static const char *tsdbTxnFname[] = {"current.t", "current"}; #define TSDB_MAX_FSETS(keep, days) ((keep) / (days) + 3) +#define TSDB_MAX_INIT_FSETS (365000) static int tsdbComparFidFSet(const void *arg1, const void *arg2); static void tsdbResetFSStatus(SFSStatus *pStatus); @@ -210,6 +211,10 @@ STsdbFS *tsdbNewFS(const STsdbKeepCfg *pCfg) { return NULL; } + if (maxFSet > TSDB_MAX_INIT_FSETS) { + maxFSet = TSDB_MAX_INIT_FSETS; + } + pfs->cstatus = tsdbNewFSStatus(maxFSet); if (pfs->cstatus == NULL) { tsdbFreeFS(pfs); From 1227d6aed222be463c9b83b2c7a27f4e7682ed28 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Mon, 13 Jun 2022 19:39:40 +0800 Subject: [PATCH 15/78] other: adjust debug logs --- source/common/src/trow.c | 15 --------------- source/dnode/vnode/src/tsdb/tsdbCommit.c | 18 ------------------ source/dnode/vnode/src/vnd/vnodeSvr.c | 6 +----- 3 files changed, 1 insertion(+), 38 deletions(-) diff --git a/source/common/src/trow.c b/source/common/src/trow.c index fb4413478d..b1ca406a4a 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -527,20 +527,6 @@ static int32_t tdAppendTpRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols return terrno; } - if (pRowCol->colId == 2) { - ASSERT(sVal.valType == 0); - int32_t val = *(int32_t *)sVal.val; - ASSERT(val == 7); - } else if (pRowCol->colId == 3) { - ASSERT(sVal.valType == 0); - int64_t val = *(int64_t *)sVal.val; - ASSERT(val == 77777); - } else if (pRowCol->colId == 4) { - ASSERT(sVal.valType == 0); - int16_t val = *(int16_t *)sVal.val; - ASSERT(val == 777); - } - tdAppendValToDataCol(pDataCol, sVal.valType, sVal.val, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode, isMerge); ++dcol; @@ -551,7 +537,6 @@ static int32_t tdAppendTpRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols tdAppendValToDataCol(pDataCol, TD_VTYPE_NULL, NULL, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode, isMerge); ++dcol; - ASSERT(0); } } #if 0 diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 4dae9bdeab..fe89321ae9 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -1350,24 +1350,6 @@ static int tsdbCommitMemData(SCommitH *pCommith, SCommitIter *pIter, TSKEY keyLi isLast = true; } - SDataCols *pCols = pCommith->pDataCols; - SDataCol pCol_1 = pCols->cols[1]; - SDataCol pCol_2 = pCols->cols[2]; - SDataCol pCol_3 = pCols->cols[3]; - SCellVal sVal = {0}; - tdGetColDataOfRow(&sVal, &pCol_1, 0, pCols->bitmapMode); - ASSERT(sVal.valType == 0); - int32_t val1 = *(int32_t *)sVal.val; - ASSERT(val1 == 7); - tdGetColDataOfRow(&sVal, &pCol_2, 0, pCols->bitmapMode); - ASSERT(sVal.valType == 0); - int64_t val2 = *(int64_t *)sVal.val; - ASSERT(val2 == 77777); - tdGetColDataOfRow(&sVal, &pCol_3, 0, pCols->bitmapMode); - ASSERT(sVal.valType == 0); - int16_t val3 = *(int16_t *)sVal.val; - ASSERT(val3 == 777); - if (tsdbWriteBlock(pCommith, pDFile, pCommith->pDataCols, &block, isLast, true) < 0) return -1; if (tsdbCommitAddBlock(pCommith, &block, NULL, 0) < 0) { diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 9bb0834547..ab2efa4791 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -15,8 +15,6 @@ #include "vnd.h" -int32_t gForceCommit = 1; - static int32_t vnodeProcessCreateStbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessAlterStbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessDropStbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); @@ -184,8 +182,7 @@ int32_t vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp } // commit if need - if ((gForceCommit == 1) || vnodeShouldCommit(pVnode)) { - gForceCommit = 0; + if (vnodeShouldCommit(pVnode)) { vInfo("vgId:%d, commit at version %" PRId64, TD_VID(pVnode), version); // commit current change vnodeCommit(pVnode); @@ -722,7 +719,6 @@ static int32_t vnodeDebugPrintSubmitMsg(SVnode *pVnode, SSubmitReq *pMsg, const } static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { - gForceCommit = 1; SSubmitReq *pSubmitReq = (SSubmitReq *)pReq; SSubmitRsp submitRsp = {0}; SSubmitMsgIter msgIter = {0}; From 8fbf5f3dba20d9089251c3783712c4eb29ae5c7d Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Mon, 13 Jun 2022 19:40:39 +0800 Subject: [PATCH 16/78] other: adjust debug logs --- source/common/src/trow.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/source/common/src/trow.c b/source/common/src/trow.c index b1ca406a4a..c8a28d7f28 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -523,10 +523,8 @@ static int32_t tdAppendTpRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols SCellVal sVal = {0}; if (pRowCol->colId == pDataCol->colId) { if (tdGetTpRowValOfCol(&sVal, pRow, pBitmap, pRowCol->type, pRowCol->offset - sizeof(TSKEY), rcol - 1) < 0) { - ASSERT(0); return terrno; } - tdAppendValToDataCol(pDataCol, sVal.valType, sVal.val, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode, isMerge); ++dcol; From b83197398eeee6e6f8d7b8506701a1ee1c675ba1 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 13 Jun 2022 19:47:14 +0800 Subject: [PATCH 17/78] fix msg decode issue --- source/common/src/tmsg.c | 18 ++++++++---------- source/dnode/mnode/impl/src/mndSma.c | 5 +++-- source/dnode/mnode/impl/src/mndStb.c | 6 ++++++ source/libs/catalog/src/ctgCache.c | 2 +- 4 files changed, 18 insertions(+), 13 deletions(-) diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 127d4ac273..31b8eb71bf 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -2652,7 +2652,8 @@ int32_t tSerializeSSTbHbRsp(void *buf, int32_t bufLen, SSTbHbRsp *pRsp) { } int32_t numOfIndex = taosArrayGetSize(pRsp->pIndexRsp); - for (int32_t i = 0; i < numOfMeta; ++i) { + if (tEncodeI32(&encoder, numOfIndex) < 0) return -1; + for (int32_t i = 0; i < numOfIndex; ++i) { STableIndexRsp *pIndexRsp = taosArrayGet(pRsp->pIndexRsp, i); if (tEncodeCStr(&encoder, pIndexRsp->tbName) < 0) return -1; if (tEncodeCStr(&encoder, pIndexRsp->dbFName) < 0) return -1; @@ -2660,12 +2661,10 @@ int32_t tSerializeSSTbHbRsp(void *buf, int32_t bufLen, SSTbHbRsp *pRsp) { if (tEncodeI32(&encoder, pIndexRsp->version) < 0) return -1; int32_t num = taosArrayGetSize(pIndexRsp->pIndex); if (tEncodeI32(&encoder, num) < 0) return -1; - if (num > 0) { - for (int32_t i = 0; i < num; ++i) { - STableIndexInfo *pInfo = (STableIndexInfo *)taosArrayGet(pIndexRsp->pIndex, i); - if (tSerializeSTableIndexInfo(&encoder, pInfo) < 0) return -1; - } - } + for (int32_t i = 0; i < num; ++i) { + STableIndexInfo *pInfo = (STableIndexInfo *)taosArrayGet(pIndexRsp->pIndex, i); + if (tSerializeSTableIndexInfo(&encoder, pInfo) < 0) return -1; + } } tEndEncode(&encoder); @@ -2693,9 +2692,8 @@ int32_t tDeserializeSSTbHbRsp(void *buf, int32_t bufLen, SSTbHbRsp *pRsp) { if (tStartDecode(&decoder) < 0) return -1; - int32_t numOfMeta = taosArrayGetSize(pRsp->pMetaRsp); + int32_t numOfMeta = 0; if (tDecodeI32(&decoder, &numOfMeta) < 0) return -1; - pRsp->pMetaRsp = taosArrayInit(numOfMeta, sizeof(STableMetaRsp)); if (pRsp->pMetaRsp == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -2708,7 +2706,7 @@ int32_t tDeserializeSSTbHbRsp(void *buf, int32_t bufLen, SSTbHbRsp *pRsp) { taosArrayPush(pRsp->pMetaRsp, &tableMetaRsp); } - int32_t numOfIndex = taosArrayGetSize(pRsp->pIndexRsp); + int32_t numOfIndex = 0; if (tDecodeI32(&decoder, &numOfIndex) < 0) return -1; pRsp->pIndexRsp = taosArrayInit(numOfIndex, sizeof(STableIndexRsp)); diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index 9cb7991d33..3f0d20348b 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -532,6 +532,7 @@ static int32_t mndCreateSma(SMnode *pMnode, SRpcMsg *pReq, SMCreateSmaReq *pCrea SStreamObj streamObj = {0}; tstrncpy(streamObj.name, pCreate->name, TSDB_STREAM_FNAME_LEN); tstrncpy(streamObj.sourceDb, pDb->name, TSDB_DB_FNAME_LEN); + tstrncpy(streamObj.targetDb, streamObj.sourceDb, TSDB_DB_FNAME_LEN); streamObj.createTime = taosGetTimestampMs(); streamObj.updateTime = streamObj.createTime; streamObj.uid = mndGenerateUid(pCreate->name, strlen(pCreate->name)); @@ -913,7 +914,7 @@ int32_t mndGetTableSma(SMnode *pMnode, char *tbFName, STableIndexRsp *rsp, bool } strcpy(rsp->dbFName, pStb->db); - strcpy(rsp->tbName, pStb->name); + strcpy(rsp->tbName, pStb->name + strlen(pStb->db) + 1); rsp->suid = pStb->uid; rsp->version = pStb->smaVer; mndReleaseStb(pMnode, pStb); @@ -1127,4 +1128,4 @@ static int32_t mndRetrieveSma(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc static void mndCancelGetNextSma(SMnode *pMnode, void *pIter) { SSdb *pSdb = pMnode->pSdb; sdbCancelFetch(pSdb, pIter); -} \ No newline at end of file +} diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index e099359d47..6f89c97f83 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -1715,6 +1715,12 @@ int32_t mndValidateStbInfo(SMnode *pMnode, SSTableVersion *pStbVersions, int32_t bool exist = false; char tbFName[TSDB_TABLE_FNAME_LEN]; STableIndexRsp indexRsp = {0}; + indexRsp.pIndex = taosArrayInit(10, sizeof(STableIndexInfo)); + if (NULL == indexRsp.pIndex) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + sprintf(tbFName, "%s.%s", pStbVersion->dbFName, pStbVersion->stbName); int32_t code = mndGetTableSma(pMnode, tbFName, &indexRsp, &exist); if (code || !exist) { diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index b938dff932..724b54d250 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -1445,7 +1445,7 @@ int32_t ctgWriteTbIndexToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char* dbFNa ctgDebug("table %s index updated to cache, ver:%d, num:%d", tbName, pIndex->version, (int32_t)taosArrayGetSize(pIndex->pIndex)); if (suid) { - CTG_ERR_RET(ctgUpdateRentStbVersion(pCtg, dbFName, tbName, dbCache->dbId, pIndex->suid, pCache)); + CTG_ERR_RET(ctgUpdateRentStbVersion(pCtg, dbFName, tbName, dbCache->dbId, pIndex->suid, &cache)); } return TSDB_CODE_SUCCESS; From fc32f716287910a0e7f3e91dd190a5fd0425adb4 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 13 Jun 2022 19:59:30 +0800 Subject: [PATCH 18/78] opt: optimize group by tag --- source/libs/executor/inc/executorInt.h | 3 - source/libs/executor/inc/executorimpl.h | 8 +-- source/libs/executor/src/executorimpl.c | 92 +++++++++++++++++++++++- source/libs/executor/src/groupoperator.c | 6 +- source/libs/executor/src/scanoperator.c | 40 ++--------- 5 files changed, 100 insertions(+), 49 deletions(-) diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h index 88f308710e..df54161720 100644 --- a/source/libs/executor/inc/executorInt.h +++ b/source/libs/executor/inc/executorInt.h @@ -27,10 +27,7 @@ typedef struct { int32_t bytes; } SGroupKeys, SStateKeys; -int32_t initGroupOptrInfo(SArray** pGroupColVals, int32_t* keyLen, char** keyBuf, const SArray* pGroupColList); uint64_t calcGroupId(char* pData, int32_t len); -void recordNewGroupKeys(SArray* pGroupCols, SArray* pGroupColVals, SSDataBlock* pBlock, int32_t rowIndex); -int32_t buildGroupKeys(void* pKey, const SArray* pGroupColVals); #ifdef __cplusplus } #endif diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 94c6512e77..3e24e43233 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -336,12 +336,6 @@ typedef struct STableScanInfo { int32_t dataBlockLoadFlag; SInterval interval; // if the upstream is an interval operator, the interval info is also kept here to get the time window to check if current data block needs to be loaded. - SArray* pGroupCols; - SArray* pGroupColVals; // current group column values, SArray - char* keyBuf; // group by keys for hash - int32_t groupKeyLen; // total group by column width - SHashObj* pGroupSet; // quick locate the window object for each result - SSampleExecInfo sample; // sample execution info int32_t curTWinIdx; } STableScanInfo; @@ -789,7 +783,7 @@ SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pR SOperatorInfo* createExchangeOperatorInfo(void* pTransporter, SExchangePhysiNode* pExNode, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, tsdbReaderT pDataReader, SReadHandle* pHandle, SArray* groupKyes, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, tsdbReaderT pDataReader, SReadHandle* pHandle, SExecTaskInfo* pTaskInfo); SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysiNode* pPhyNode, STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo); SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScanPhysiNode *pScanPhyNode, SExecTaskInfo* pTaskInfo); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 2c067bf488..13ac74199e 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -4592,6 +4592,85 @@ int32_t extractTableSchemaVersion(SReadHandle* pHandle, uint64_t uid, SExecTaskI return TSDB_CODE_SUCCESS; } +int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, SArray* groupKey){ + if(groupKey == NULL) { + return TDB_CODE_SUCCESS; + } + + pTableListInfo->map = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + if (pTableListInfo->map == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + int32_t keyLen = 0; + void *keyBuf = NULL; + int32_t numOfGroupCols = taosArrayGetSize(groupKey); + for (int32_t j = 0; j < numOfGroupCols; ++j) { + SColumn* pCol = taosArrayGet(groupKey, j); + keyLen += pCol->bytes; // actual data + null_flag + } + + int32_t nullFlagSize = sizeof(int8_t) * numOfGroupCols; + keyLen += nullFlagSize; + + keyBuf = taosMemoryCalloc(1, keyLen); + if (keyBuf == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + for(int32_t i = 0; i < taosArrayGetSize(pTableListInfo->pTableList); i++){ + STableKeyInfo *info = taosArrayGet(pTableListInfo->pTableList, i); + SMetaReader mr = {0}; + metaReaderInit(&mr, pHandle->meta, 0); + metaGetTableEntryByUid(&mr, info->uid); + + char* isNull = (char*)keyBuf; + char* pStart = (char*)keyBuf + sizeof(int8_t) * numOfGroupCols; + for (int32_t j = 0; j < numOfGroupCols; ++j) { + SColumn* pCol = taosArrayGet(groupKey, j); + + if(strcmp(pCol->name, "tbname") == 0){ + isNull[i] = 0; + memcpy(pStart, mr.me.name, strlen(mr.me.name)); + pStart += strlen(mr.me.name); + }else{ + STagVal tagVal = {0}; + tagVal.cid = pCol->colId; + const char* p = metaGetTableTagVal(&mr.me, pCol->type, &tagVal); + if(p == NULL){ + isNull[j] = 1; + continue; + } + isNull[i] = 0; + if (pCol->type == TSDB_DATA_TYPE_JSON) { +// int32_t dataLen = getJsonValueLen(pkey->pData); +// memcpy(pStart, (pkey->pData), dataLen); +// pStart += dataLen; + } else if (IS_VAR_DATA_TYPE(pCol->type)) { + memcpy(pStart, tagVal.pData, tagVal.nData); + pStart += tagVal.nData; + ASSERT(tagVal.nData <= pCol->bytes); + } else { + memcpy(pStart, &(tagVal.i64), pCol->bytes); + pStart += pCol->bytes; + } + } + } + + int32_t len = (int32_t) (pStart - (char*)keyBuf); + uint64_t* groupId = taosHashGet(pTableListInfo->map, keyBuf, len); + if (groupId) { + taosHashPut(pTableListInfo->map, &(info->uid), sizeof(uint64_t), groupId, sizeof(uint64_t)); + } else { + uint64_t tmpId = calcGroupId(keyBuf, len); + taosHashPut(pTableListInfo->map, &(info->uid), sizeof(uint64_t), &tmpId, sizeof(uint64_t)); + } + + metaReaderClear(&mr); + } + taosMemoryFree(keyBuf); + return TDB_CODE_SUCCESS; +} + SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHandle* pHandle, uint64_t queryId, uint64_t taskId, STableListInfo* pTableListInfo, SNode* pTagCond) { int32_t type = nodeType(pPhyNode); @@ -4605,15 +4684,22 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo if (pDataReader == NULL && terrno != 0) { return NULL; } - SArray* groupKyes = extractPartitionColInfo(pTableScanNode->pPartitionKeys); + int32_t code = extractTableSchemaVersion(pHandle, pTableScanNode->scan.uid, pTaskInfo); if (code) { tsdbCleanupReadHandle(pDataReader); return NULL; } - + + SArray* groupKyes = extractPartitionColInfo(pTableScanNode->pPartitionKeys); + code = generateGroupIdMap(pTableListInfo, pHandle, groupKyes); //todo for json + if (code){ + tsdbCleanupReadHandle(pDataReader); + return NULL; + } + SOperatorInfo* pOperator = - createTableScanOperatorInfo(pTableScanNode, pDataReader, pHandle, groupKyes, pTaskInfo); + createTableScanOperatorInfo(pTableScanNode, pDataReader, pHandle, pTaskInfo); STableScanInfo* pScanInfo = pOperator->info; pTaskInfo->cost.pRecoder = &pScanInfo->readRecorder; diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index 132f93a6a5..0f9879ecc9 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -37,7 +37,7 @@ static void destroyGroupOperatorInfo(void* param, int32_t numOfOutput) { taosArrayDestroy(pInfo->pGroupColVals); } -int32_t initGroupOptrInfo(SArray** pGroupColVals, int32_t* keyLen, char** keyBuf, const SArray* pGroupColList) { +static int32_t initGroupOptrInfo(SArray** pGroupColVals, int32_t* keyLen, char** keyBuf, const SArray* pGroupColList) { *pGroupColVals = taosArrayInit(4, sizeof(SGroupKeys)); if ((*pGroupColVals) == NULL) { return TSDB_CODE_OUT_OF_MEMORY; @@ -118,7 +118,7 @@ static bool groupKeyCompare(SArray* pGroupCols, SArray* pGroupColVals, SSDataBlo return true; } -void recordNewGroupKeys(SArray* pGroupCols, SArray* pGroupColVals, SSDataBlock* pBlock, int32_t rowIndex) { +static void recordNewGroupKeys(SArray* pGroupCols, SArray* pGroupColVals, SSDataBlock* pBlock, int32_t rowIndex) { SColumnDataAgg* pColAgg = NULL; size_t numOfGroupCols = taosArrayGetSize(pGroupCols); @@ -150,7 +150,7 @@ void recordNewGroupKeys(SArray* pGroupCols, SArray* pGroupColVals, SSDataBlock* } } -int32_t buildGroupKeys(void* pKey, const SArray* pGroupColVals) { +static int32_t buildGroupKeys(void* pKey, const SArray* pGroupColVals) { ASSERT(pKey != NULL); size_t numOfGroupCols = taosArrayGetSize(pGroupColVals); diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 8fb4878cd9..b6ee9843b4 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -391,22 +391,16 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { longjmp(pOperator->pTaskInfo->env, code); } - recordNewGroupKeys(pTableScanInfo->pGroupCols, pTableScanInfo->pGroupColVals, pBlock, 0); - int32_t len = buildGroupKeys(pTableScanInfo->keyBuf, pTableScanInfo->pGroupColVals); - - uint64_t* groupId = taosHashGet(pTableScanInfo->pGroupSet, pTableScanInfo->keyBuf, len); - if (groupId) { - pBlock->info.groupId = *groupId; - } else if (len != 0) { - pBlock->info.groupId = calcGroupId(pTableScanInfo->keyBuf, len); - taosHashPut(pTableScanInfo->pGroupSet, pTableScanInfo->keyBuf, len, &pBlock->info.groupId, sizeof(uint64_t)); - } - // current block is filter out according to filter condition, continue load the next block if (status == FUNC_DATA_REQUIRED_FILTEROUT || pBlock->info.rows == 0) { continue; } + uint64_t* groupId = taosHashGet(pOperator->pTaskInfo->tableqinfoList.map, &pBlock->info.uid, sizeof(int64_t)); + if (groupId) { + pBlock->info.groupId = *groupId; + } + pOperator->resultInfo.totalRows = pTableScanInfo->readRecorder.totalRows; pTableScanInfo->readRecorder.elapsedTime += (taosGetTimestampUs() - st) / 1000.0; @@ -530,21 +524,13 @@ static void destroyTableScanOperatorInfo(void* param, int32_t numOfOutput) { tsdbCleanupReadHandle(pTableScanInfo->dataReader); - taosArrayDestroy(pTableScanInfo->pGroupCols); - for (int i = 0; i < taosArrayGetSize(pTableScanInfo->pGroupColVals); i++) { - SGroupKeys key = *(SGroupKeys*)taosArrayGet(pTableScanInfo->pGroupColVals, i); - taosMemoryFree(key.pData); - } - taosArrayDestroy(pTableScanInfo->pGroupColVals); - taosMemoryFree(pTableScanInfo->keyBuf); - taosHashCleanup(pTableScanInfo->pGroupSet); if (pTableScanInfo->pColMatchInfo != NULL) { taosArrayDestroy(pTableScanInfo->pColMatchInfo); } } SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, tsdbReaderT pDataReader, - SReadHandle* readHandle, SArray* groupKyes, SExecTaskInfo* pTaskInfo) { + SReadHandle* readHandle, SExecTaskInfo* pTaskInfo) { STableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableScanInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { @@ -591,18 +577,6 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, pOperator->numOfExprs = numOfCols; pOperator->pTaskInfo = pTaskInfo; - // for table group - pInfo->pGroupCols = groupKyes; - _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); - pInfo->pGroupSet = taosHashInit(100, hashFn, false, HASH_NO_LOCK); - if (pInfo->pGroupSet == NULL) { - goto _error; - } - code = initGroupOptrInfo(&pInfo->pGroupColVals, &pInfo->groupKeyLen, &pInfo->keyBuf, groupKyes); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTableScan, NULL, NULL, destroyTableScanOperatorInfo, NULL, NULL, getTableScannerExecInfo); @@ -992,7 +966,7 @@ SOperatorInfo* createStreamScanOperatorInfo(void* pDataReader, SReadHandle* pHan SScanPhysiNode* pScanPhyNode = &pTableScanNode->scan; SDataBlockDescNode* pDescNode = pScanPhyNode->node.pOutputDataBlockDesc; - SOperatorInfo* pTableScanDummy = createTableScanOperatorInfo(pTableScanNode, pDataReader, pHandle, NULL, pTaskInfo); + SOperatorInfo* pTableScanDummy = createTableScanOperatorInfo(pTableScanNode, pDataReader, pHandle, pTaskInfo); STableScanInfo* pSTInfo = (STableScanInfo*)pTableScanDummy->info; From 2488382dd6de0f5c45a7c7df99dee4019cd6c730 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 13 Jun 2022 20:49:54 +0800 Subject: [PATCH 19/78] fix: avoid invalid value --- source/libs/index/src/indexFilter.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/source/libs/index/src/indexFilter.c b/source/libs/index/src/indexFilter.c index 1cc2679b06..90aafb1097 100644 --- a/source/libs/index/src/indexFilter.c +++ b/source/libs/index/src/indexFilter.c @@ -183,6 +183,7 @@ static int32_t sifInitJsonParam(SNode *node, SIFParam *param, SIFCtx *ctx) { memcpy(param->colName, r->literal, strlen(r->literal)); // sprintf(param->colName, "%s_%s", l->colName, r->literal); param->colValType = r->typeData; + param->status = SFLT_COARSE_INDEX; return 0; // memcpy(param->colName, l->colName, sizeof(l->colName)); } @@ -254,11 +255,13 @@ static int32_t sifInitOperParams(SIFParam **params, SOperatorNode *node, SIFCtx return code; } SIFParam *paramList = taosMemoryCalloc(nParam, sizeof(SIFParam)); + if (NULL == paramList) { SIF_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - if (nodeType(node->pLeft) == QUERY_NODE_OPERATOR) { + if (nodeType(node->pLeft) == QUERY_NODE_OPERATOR && + (((SOperatorNode *)(node->pLeft))->opType == OP_TYPE_JSON_GET_VALUE)) { SNode *interNode = (node->pLeft); SIF_ERR_JRET(sifInitJsonParam(interNode, ¶mList[0], ctx)); if (nParam > 1) { @@ -525,8 +528,8 @@ static int32_t sifExecOper(SOperatorNode *node, SIFCtx *ctx, SIFParam *output) { if (node->opType == OP_TYPE_JSON_GET_VALUE) { return code; } - SIFParam *params = NULL; + SIFParam *params = NULL; SIF_ERR_RET(sifInitOperParams(¶ms, node, ctx)); if (params[0].status == SFLT_NOT_INDEX || (nParam > 1 && params[1].status == SFLT_NOT_INDEX)) { From 45ef3db78abd908abbc57a7bfbc56871381413f2 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 13 Jun 2022 20:53:42 +0800 Subject: [PATCH 20/78] add first function partial/merge --- source/libs/function/inc/builtinsimpl.h | 2 + source/libs/function/src/builtins.c | 4 +- source/libs/function/src/builtinsimpl.c | 82 +++++++++++++++++++++---- 3 files changed, 75 insertions(+), 13 deletions(-) diff --git a/source/libs/function/inc/builtinsimpl.h b/source/libs/function/inc/builtinsimpl.h index f3bcad8ede..af9b8d4a10 100644 --- a/source/libs/function/inc/builtinsimpl.h +++ b/source/libs/function/inc/builtinsimpl.h @@ -93,8 +93,10 @@ int32_t diffFunction(SqlFunctionCtx *pCtx); bool getFirstLastFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); int32_t firstFunction(SqlFunctionCtx *pCtx); +int32_t firstFunctionMerge(SqlFunctionCtx *pCtx); int32_t lastFunction(SqlFunctionCtx *pCtx); int32_t firstLastFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); +int32_t firstLastPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); int32_t firstCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx); int32_t lastCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx); int32_t getFirstLastInfoSize(int32_t resBytes); diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index a2849d601e..180ffd1848 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -1749,7 +1749,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .getEnvFunc = getFirstLastFuncEnv, .initFunc = functionSetup, .processFunc = firstFunction, - .finalizeFunc = firstLastFinalize, + .finalizeFunc = firstLastPartialFinalize, .combineFunc = firstCombine, }, { @@ -1759,7 +1759,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .translateFunc = translateFirstLastMerge, .getEnvFunc = getFirstLastFuncEnv, .initFunc = functionSetup, - .processFunc = firstFunction, + .processFunc = firstFunctionMerge, .finalizeFunc = firstLastFinalize, .combineFunc = firstCombine, }, diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 44418685b0..1bb27780bc 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -72,6 +72,12 @@ typedef struct STopBotRes { STopBotResItem* pItems; } STopBotRes; +typedef struct SFirstLastRes { + bool hasResult; + int32_t bytes; + char buf[]; +} SFirstLastRes; + typedef struct SStddevRes { double result; int64_t count; @@ -2245,12 +2251,12 @@ int32_t apercentileCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) } int32_t getFirstLastInfoSize(int32_t resBytes) { - return resBytes + sizeof(int64_t); + return sizeof(SFirstLastRes) + resBytes + sizeof(int64_t); } bool getFirstLastFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) { SColumnNode* pNode = nodesListGetNode(pFunc->pParameterList, 0); - pEnv->calcMemSize = pNode->node.resType.bytes + sizeof(int64_t); + pEnv->calcMemSize = sizeof(SFirstLastRes) + pNode->node.resType.bytes + sizeof(int64_t); return true; } @@ -2274,12 +2280,13 @@ int32_t firstFunction(SqlFunctionCtx* pCtx) { int32_t numOfElems = 0; SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); - char* buf = GET_ROWCELL_INTERBUF(pResInfo); + SFirstLastRes *pInfo = GET_ROWCELL_INTERBUF(pResInfo); SInputColumnInfoData* pInput = &pCtx->input; SColumnInfoData* pInputCol = pInput->pData[0]; int32_t bytes = pInputCol->info.bytes; + pInfo->bytes = bytes; // All null data column, return directly. if (pInput->colDataAggIsSet && (pInput->pColumnDataAgg[0]->numOfNull == pInput->totalRows)) { @@ -2297,7 +2304,7 @@ int32_t firstFunction(SqlFunctionCtx* pCtx) { if (blockDataOrder == TSDB_ORDER_ASC) { // filter according to current result firstly if (pResInfo->numOfRes > 0) { - TSKEY ts = *(TSKEY*)(buf + bytes); + TSKEY ts = *(TSKEY*)(pInfo->buf + bytes); if (ts < startKey) { return TSDB_CODE_SUCCESS; } @@ -2313,9 +2320,10 @@ int32_t firstFunction(SqlFunctionCtx* pCtx) { char* data = colDataGetData(pInputCol, i); TSKEY cts = getRowPTs(pInput->pPTS, i); - if (pResInfo->numOfRes == 0 || *(TSKEY*)(buf + bytes) > cts) { - memcpy(buf, data, bytes); - *(TSKEY*)(buf + bytes) = cts; + if (pResInfo->numOfRes == 0 || *(TSKEY*)(pInfo->buf + bytes) > cts) { + memcpy(pInfo->buf, data, bytes); + *(TSKEY*)(pInfo->buf + bytes) = cts; + pInfo->hasResult = true; // DO_UPDATE_TAG_COLUMNS(pCtx, ts); pResInfo->numOfRes = 1; @@ -2326,7 +2334,7 @@ int32_t firstFunction(SqlFunctionCtx* pCtx) { // in case of descending order time stamp serial, which usually happens as the results of the nest query, // all data needs to be check. if (pResInfo->numOfRes > 0) { - TSKEY ts = *(TSKEY*)(buf + bytes); + TSKEY ts = *(TSKEY*)(pInfo->buf + bytes); if (ts < endKey) { return TSDB_CODE_SUCCESS; } @@ -2342,9 +2350,10 @@ int32_t firstFunction(SqlFunctionCtx* pCtx) { char* data = colDataGetData(pInputCol, i); TSKEY cts = getRowPTs(pInput->pPTS, i); - if (pResInfo->numOfRes == 0 || *(TSKEY*)(buf + bytes) > cts) { - memcpy(buf, data, bytes); - *(TSKEY*)(buf + bytes) = cts; + if (pResInfo->numOfRes == 0 || *(TSKEY*)(pInfo->buf + bytes) > cts) { + memcpy(pInfo->buf, data, bytes); + *(TSKEY*)(pInfo->buf + bytes) = cts; + pInfo->hasResult = true; // DO_UPDATE_TAG_COLUMNS(pCtx, ts); pResInfo->numOfRes = 1; break; @@ -2422,6 +2431,39 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { return TSDB_CODE_SUCCESS; } +static void firstLastTransferInfo(SFirstLastRes* pInput, SFirstLastRes* pOutput) { + pOutput->bytes = pInput->bytes; + TSKEY *tsIn = (TSKEY*)(pInput->buf + pInput->bytes); + TSKEY *tsOut = (TSKEY*)(pOutput->buf + pInput->bytes); + if (pOutput->hasResult) { + if (*tsIn > *tsOut) { + return; + } + } + *tsOut = *tsIn; + memcpy(pOutput->buf, pInput->buf, pOutput->bytes); + pOutput->hasResult = true; + return; +} + +int32_t firstFunctionMerge(SqlFunctionCtx *pCtx) { + SInputColumnInfoData* pInput = &pCtx->input; + SColumnInfoData* pCol = pInput->pData[0]; + ASSERT(pCol->info.type == TSDB_DATA_TYPE_BINARY); + + SFirstLastRes* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); + + int32_t start = pInput->startRowIndex; + char* data = colDataGetData(pCol, start); + SFirstLastRes* pInputInfo = (SFirstLastRes *)varDataVal(data); + + firstLastTransferInfo(pInputInfo, pInfo); + + SET_VAL(GET_RES_INFO(pCtx), 1, 1); + + return TSDB_CODE_SUCCESS; +} + int32_t firstLastFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { int32_t slotId = pCtx->pExpr->base.resSchema.slotId; SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); @@ -2435,6 +2477,24 @@ int32_t firstLastFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { return pResInfo->numOfRes; } +int32_t firstLastPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { + SResultRowEntryInfo* pEntryInfo = GET_RES_INFO(pCtx); + SFirstLastRes* pRes = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); + int32_t resultBytes = getFirstLastInfoSize(pRes->bytes); + char *res = taosMemoryCalloc(resultBytes + VARSTR_HEADER_SIZE, sizeof(char)); + + memcpy(varDataVal(res), pRes, resultBytes); + varDataSetLen(res, resultBytes); + + int32_t slotId = pCtx->pExpr->base.resSchema.slotId; + SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); + + colDataAppend(pCol, pBlock->info.rows, res, false); + + taosMemoryFree(res); + return 1; +} + int32_t lastCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) { SResultRowEntryInfo* pDResInfo = GET_RES_INFO(pDestCtx); char* pDBuf = GET_ROWCELL_INTERBUF(pDResInfo); From 4240a38c1783804d4ee7aa0404fca0e305e8ccdc Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 13 Jun 2022 20:56:33 +0800 Subject: [PATCH 21/78] fix bugs --- source/libs/function/src/builtinsimpl.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 1bb27780bc..280f0217b9 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -2471,8 +2471,8 @@ int32_t firstLastFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); pResInfo->isNullRes = (pResInfo->numOfRes == 0) ? 1 : 0; - char* in = GET_ROWCELL_INTERBUF(pResInfo); - colDataAppend(pCol, pBlock->info.rows, in, pResInfo->isNullRes); + SFirstLastRes* pRes = GET_ROWCELL_INTERBUF(pResInfo); + colDataAppend(pCol, pBlock->info.rows, pRes->buf, pResInfo->isNullRes); return pResInfo->numOfRes; } From 763e8bd6aef99264dc6608df5881738be11d9790 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 13 Jun 2022 21:17:55 +0800 Subject: [PATCH 22/78] enable first function split --- source/libs/function/src/builtins.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 180ffd1848..3bfb60ec7c 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -956,7 +956,7 @@ static int32_t translateFirstLast(SFunctionNode* pFunc, char* pErrBuf, int32_t l static int32_t translateFirstLastImpl(SFunctionNode* pFunc, char* pErrBuf, int32_t len, bool isPartial) { // first(col_list) will be rewritten as first(col) - if (1 != LIST_LENGTH(pFunc->pParameterList)) { + if (2 != LIST_LENGTH(pFunc->pParameterList)) { //input has two params c0,ts, is this a bug? return TSDB_CODE_SUCCESS; } @@ -1739,6 +1739,8 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .initFunc = functionSetup, .processFunc = firstFunction, .finalizeFunc = firstLastFinalize, + .pPartialFunc = "_first_partial", + .pMergeFunc = "_first_merge", .combineFunc = firstCombine, }, { From d4a38245a20d6296716200b253a7b8ec5a51abf3 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 13 Jun 2022 21:21:46 +0800 Subject: [PATCH 23/78] add some check --- source/libs/function/src/builtinsimpl.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 280f0217b9..032333228a 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -2432,6 +2432,9 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { } static void firstLastTransferInfo(SFirstLastRes* pInput, SFirstLastRes* pOutput) { + if (!pInput->hasResult) { + return; + } pOutput->bytes = pInput->bytes; TSKEY *tsIn = (TSKEY*)(pInput->buf + pInput->bytes); TSKEY *tsOut = (TSKEY*)(pOutput->buf + pInput->bytes); From 0111549155f08d1c03bd04674c9bda5290a09a4f Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 14 Jun 2022 08:41:52 +0800 Subject: [PATCH 24/78] index not exist as success --- source/libs/catalog/src/catalog.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 6aa86aa4ba..a3576f8738 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -392,8 +392,12 @@ int32_t ctgGetTbIndex(SCatalog* pCtg, SRequestConnInfo *pConn, SName* pTableName CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - int32_t code = 0; - CTG_ERR_JRET(ctgGetTbIndexFromMnode(pCtg, pConn, (SName*)pTableName, pIndex, NULL)); + int32_t code = ctgGetTbIndexFromMnode(pCtg, pConn, (SName*)pTableName, pIndex, NULL); + if (TSDB_CODE_MND_DB_INDEX_NOT_EXIST == code) { + code = 0; + goto _return; + } + CTG_ERR_JRET(code); SArray* pInfo = NULL; CTG_ERR_JRET(ctgCloneTableIndex(pIndex->pIndex, &pInfo)); From e8ab28bee17e379634764012ecc091b67f3245ed Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 14 Jun 2022 09:02:57 +0800 Subject: [PATCH 25/78] fix release crash issue --- source/libs/catalog/src/ctgCache.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 724b54d250..62890b8326 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -311,6 +311,9 @@ _return: ctgReleaseTbMetaToCache(pCtg, dbCache, pCache); CTG_CACHE_STAT_INC(tbMetaMissNum, 1); + + *pDb = NULL; + *pTb = NULL; return TSDB_CODE_SUCCESS; } @@ -427,8 +430,6 @@ int32_t ctgReadTbMetaFromCache(SCatalog* pCtg, SCtgTbMetaCtx* ctx, STableMeta** ctgDebug("Got ctb %s meta from cache, will continue to get its stb meta, type:%d, dbFName:%s", ctx->pName->tname, ctx->tbInfo.tbType, dbFName); - dbCache = NULL; - tbCache = NULL; ctgAcquireStbMetaFromCache(pCtg, dbFName, ctx->tbInfo.suid, &dbCache, &tbCache); if (NULL == tbCache) { ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); From 877cb761f1b4fdc316078b47ffa0abbee1e17783 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 14 Jun 2022 09:43:22 +0800 Subject: [PATCH 26/78] refactor: mnode worker --- include/dnode/mnode/mnode.h | 1 + source/dnode/mgmt/mgmt_mnode/inc/mmInt.h | 14 +- source/dnode/mgmt/mgmt_mnode/src/mmHandle.c | 218 ++++++++++---------- source/dnode/mgmt/mgmt_mnode/src/mmInt.c | 5 +- source/dnode/mgmt/mgmt_mnode/src/mmWorker.c | 88 ++++---- 5 files changed, 161 insertions(+), 165 deletions(-) diff --git a/include/dnode/mnode/mnode.h b/include/dnode/mnode/mnode.h index ab090940f2..83183422c2 100644 --- a/include/dnode/mnode/mnode.h +++ b/include/dnode/mnode/mnode.h @@ -83,6 +83,7 @@ int32_t mndGetLoad(SMnode *pMnode, SMnodeLoad *pLoad); */ int32_t mndProcessRpcMsg(SRpcMsg *pMsg); int32_t mndProcessSyncMsg(SRpcMsg *pMsg); +int32_t mndPreprocessQueryMsg(SMnode * pMnode, SRpcMsg * pMsg); /** * @brief Generate machine code diff --git a/source/dnode/mgmt/mgmt_mnode/inc/mmInt.h b/source/dnode/mgmt/mgmt_mnode/inc/mmInt.h index 22691300bc..2443493d0b 100644 --- a/source/dnode/mgmt/mgmt_mnode/inc/mmInt.h +++ b/source/dnode/mgmt/mgmt_mnode/inc/mmInt.h @@ -53,20 +53,18 @@ void mmRelease(SMnodeMgmt *pMgmt); SArray *mmGetMsgHandles(); int32_t mmProcessCreateReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg); int32_t mmProcessDropReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg); -int32_t mmProcessAlterReq(SMnodeMgmt *pMgmt, SRpcMsg *pMsg); int32_t mmProcessGetMonitorInfoReq(SMnodeMgmt *pMgmt, SRpcMsg *pMsg); int32_t mmProcessGetLoadsReq(SMnodeMgmt *pMgmt, SRpcMsg *pMsg); -int32_t mndPreprocessQueryMsg(SMnode * pMnode, SRpcMsg * pMsg); // mmWorker.c int32_t mmStartWorker(SMnodeMgmt *pMgmt); void mmStopWorker(SMnodeMgmt *pMgmt); -int32_t mmPutNodeMsgToWriteQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg); -int32_t mmPutNodeMsgToSyncQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg); -int32_t mmPutNodeMsgToReadQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg); -int32_t mmPutNodeMsgToQueryQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg); -int32_t mmPutNodeMsgToMonitorQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg); -int32_t mmPutRpcMsgToQueue(SMnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc); +int32_t mmPutMsgToWriteQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg); +int32_t mmPutMsgToSyncQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg); +int32_t mmPutMsgToReadQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg); +int32_t mmPutMsgToQueryQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg); +int32_t mmPutMsgToMonitorQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg); +int32_t mmPutMsgToQueue(SMnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc); #ifdef __cplusplus } diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c index c47988e1ef..f151a09b94 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c @@ -20,11 +20,6 @@ void mmGetMonitorInfo(SMnodeMgmt *pMgmt, SMonMmInfo *pInfo) { mndGetMonitorInfo(pMgmt->pMnode, &pInfo->cluster, &pInfo->vgroup, &pInfo->grant); } -void mmGetMnodeLoads(SMnodeMgmt *pMgmt, SMonMloadInfo *pInfo) { - pInfo->isMnode = 1; - mndGetLoad(pMgmt->pMnode, &pInfo->load); -} - int32_t mmProcessGetMonitorInfoReq(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { SMonMmInfo mmInfo = {0}; mmGetMonitorInfo(pMgmt, &mmInfo); @@ -50,6 +45,11 @@ int32_t mmProcessGetMonitorInfoReq(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { return 0; } +void mmGetMnodeLoads(SMnodeMgmt *pMgmt, SMonMloadInfo *pInfo) { + pInfo->isMnode = 1; + mndGetLoad(pMgmt->pMnode, &pInfo->load); +} + int32_t mmProcessGetLoadsReq(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { SMonMloadInfo mloads = {0}; mmGetMnodeLoads(pMgmt, &mloads); @@ -129,114 +129,114 @@ SArray *mmGetMsgHandles() { SArray *pArray = taosArrayInit(64, sizeof(SMgmtHandle)); if (pArray == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_MNODE_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_MNODE_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_QNODE_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_QNODE_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_SNODE_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_SNODE_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_BNODE_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_BNODE_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_VNODE_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_VNODE_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_DND_CONFIG_DNODE_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_MNODE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_MNODE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_QNODE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_QNODE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_SNODE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_SNODE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_BNODE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_BNODE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_VNODE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_VNODE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_DND_CONFIG_DNODE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_CONNECT, mmPutNodeMsgToReadQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_ACCT, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_ALTER_ACCT, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_ACCT, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_USER, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_ALTER_USER, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_USER, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_GET_USER_AUTH, mmPutNodeMsgToReadQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_DNODE, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_CONFIG_DNODE, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_DNODE, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_MNODE, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_ALTER_MNODE, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_ALTER_MNODE_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_SET_STANDBY_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_MNODE, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_QNODE, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_QNODE, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_QNODE_LIST, mmPutNodeMsgToReadQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_SNODE, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_SNODE, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_BNODE, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_BNODE, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_DB, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_DB, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_USE_DB, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_ALTER_DB, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_COMPACT_DB, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_GET_DB_CFG, mmPutNodeMsgToReadQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_VGROUP_LIST, mmPutNodeMsgToReadQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_REDISTRIBUTE_VGROUP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_MERGE_VGROUP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_BALANCE_VGROUP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_FUNC, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_RETRIEVE_FUNC, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_FUNC, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_STB, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_ALTER_STB, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_STB, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_TABLE_META, mmPutNodeMsgToReadQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_SMA, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_SMA, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_STREAM, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_GET_INDEX, mmPutNodeMsgToReadQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_GET_TABLE_INDEX, mmPutNodeMsgToReadQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_TOPIC, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_ALTER_TOPIC, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_TOPIC, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_SUBSCRIBE, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_MQ_ASK_EP, mmPutNodeMsgToReadQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_MQ_DROP_CGROUP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_MQ_DROP_CGROUP_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_MQ_COMMIT_OFFSET, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_KILL_TRANS, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_KILL_QUERY, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_KILL_CONN, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_HEARTBEAT, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_STATUS, mmPutNodeMsgToReadQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_SYSTABLE_RETRIEVE, mmPutNodeMsgToReadQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_GRANT, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_AUTH, mmPutNodeMsgToReadQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_CONNECT, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_ACCT, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_ALTER_ACCT, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_ACCT, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_USER, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_ALTER_USER, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_USER, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_GET_USER_AUTH, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_DNODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_CONFIG_DNODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_DNODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_MNODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_ALTER_MNODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_ALTER_MNODE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_SET_STANDBY_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_MNODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_QNODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_QNODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_QNODE_LIST, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_SNODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_SNODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_BNODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_BNODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_DB, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_DB, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_USE_DB, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_ALTER_DB, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_COMPACT_DB, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_GET_DB_CFG, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_VGROUP_LIST, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_REDISTRIBUTE_VGROUP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_MERGE_VGROUP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_BALANCE_VGROUP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_FUNC, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_RETRIEVE_FUNC, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_FUNC, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_STB, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_ALTER_STB, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_STB, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_TABLE_META, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_SMA, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_SMA, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_STREAM, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_GET_INDEX, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_GET_TABLE_INDEX, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_TOPIC, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_ALTER_TOPIC, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_TOPIC, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_SUBSCRIBE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_MQ_ASK_EP, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_MQ_DROP_CGROUP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_MQ_DROP_CGROUP_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_MQ_COMMIT_OFFSET, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_KILL_TRANS, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_KILL_QUERY, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_KILL_CONN, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_HEARTBEAT, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_STATUS, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_SYSTABLE_RETRIEVE, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_GRANT, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_AUTH, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_QUERY, mmPutNodeMsgToQueryQueue, 1) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_QUERY_CONTINUE, mmPutNodeMsgToQueryQueue, 1) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_QUERY_HEARTBEAT, mmPutNodeMsgToQueryQueue, 1) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_FETCH, mmPutNodeMsgToQueryQueue, 1) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_CREATE_STB_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_STB_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_STB_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_CREATE_SMA_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_SMA_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_MQ_VG_CHANGE_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_MQ_VG_DELETE_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_TASK, mmPutNodeMsgToQueryQueue, 1) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DEPLOY_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_CONFIG_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_REPLICA_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_CONFIRM_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_HASHRANGE_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_COMPACT_RSP, mmPutNodeMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_QUERY, mmPutMsgToQueryQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_QUERY_CONTINUE, mmPutMsgToQueryQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_QUERY_HEARTBEAT, mmPutMsgToQueryQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_FETCH, mmPutMsgToQueryQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_CREATE_STB_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_STB_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_STB_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_CREATE_SMA_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_SMA_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_MQ_VG_CHANGE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_MQ_VG_DELETE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_TASK, mmPutMsgToQueryQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DEPLOY_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_CONFIG_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_REPLICA_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_CONFIRM_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_HASHRANGE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_COMPACT_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MON_MM_INFO, mmPutNodeMsgToMonitorQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MON_MM_LOAD, mmPutNodeMsgToMonitorQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MON_MM_INFO, mmPutMsgToMonitorQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MON_MM_LOAD, mmPutMsgToMonitorQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_SYNC_TIMEOUT, mmPutNodeMsgToSyncQueue, 1) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_SYNC_PING, mmPutNodeMsgToSyncQueue, 1) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_SYNC_PING_REPLY, mmPutNodeMsgToSyncQueue, 1) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST, mmPutNodeMsgToSyncQueue, 1) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST_REPLY, mmPutNodeMsgToSyncQueue, 1) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_SYNC_REQUEST_VOTE, mmPutNodeMsgToSyncQueue, 1) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_SYNC_REQUEST_VOTE_REPLY, mmPutNodeMsgToSyncQueue, 1) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES, mmPutNodeMsgToSyncQueue, 1) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES_REPLY, mmPutNodeMsgToSyncQueue, 1) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_SYNC_SNAPSHOT_SEND, mmPutNodeMsgToSyncQueue, 1) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_SYNC_SNAPSHOT_RSP, mmPutNodeMsgToSyncQueue, 1) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_SET_STANDBY, mmPutNodeMsgToSyncQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_TIMEOUT, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_PING, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_PING_REPLY, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST_REPLY, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_REQUEST_VOTE, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_REQUEST_VOTE_REPLY, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES_REPLY, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_SNAPSHOT_SEND, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_SNAPSHOT_RSP, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_SET_STANDBY, mmPutMsgToSyncQueue, 0) == NULL) goto _OVER; code = 0; diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmInt.c b/source/dnode/mgmt/mgmt_mnode/src/mmInt.c index 0f3c06cb3a..cb942e8d1c 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmInt.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmInt.c @@ -43,7 +43,6 @@ static void mmBuildOptionForDeploy(SMnodeMgmt *pMgmt, const SMgmtInputOpt *pInpu pOption->deploy = true; pOption->msgCb = pMgmt->msgCb; pOption->dnodeId = pMgmt->pData->dnodeId; - pOption->replica = 1; pOption->selfIndex = 0; @@ -54,8 +53,8 @@ static void mmBuildOptionForDeploy(SMnodeMgmt *pMgmt, const SMgmtInputOpt *pInpu } static void mmBuildOptionForOpen(SMnodeMgmt *pMgmt, SMnodeOpt *pOption) { - pOption->deploy = false; pOption->standby = false; + pOption->deploy = false; pOption->msgCb = pMgmt->msgCb; pOption->dnodeId = pMgmt->pData->dnodeId; @@ -105,7 +104,7 @@ static int32_t mmOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) { pMgmt->path = pInput->path; pMgmt->name = pInput->name; pMgmt->msgCb = pInput->msgCb; - pMgmt->msgCb.putToQueueFp = (PutToQueueFp)mmPutRpcMsgToQueue; + pMgmt->msgCb.putToQueueFp = (PutToQueueFp)mmPutMsgToQueue; pMgmt->msgCb.mgmt = pMgmt; taosThreadRwlockInit(&pMgmt->lock, NULL); diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c index 53943b61b0..6da9bfc528 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c @@ -26,7 +26,7 @@ static inline void mmSendRsp(SRpcMsg *pMsg, int32_t code) { tmsgSendRsp(&rsp); } -static void mmProcessQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { +static void mmProcessRpcMsg(SQueueInfo *pInfo, SRpcMsg *pMsg) { SMnodeMgmt *pMgmt = pInfo->ahandle; int32_t code = -1; dTrace("msg:%p, get from mnode queue", pMsg); @@ -53,11 +53,10 @@ static void mmProcessQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { taosFreeQitem(pMsg); } -static void mmProcessSyncQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { +static void mmProcessSyncMsg(SQueueInfo *pInfo, SRpcMsg *pMsg) { SMnodeMgmt *pMgmt = pInfo->ahandle; - dTrace("msg:%p, get from mnode-sync queue", pMsg); - pMsg->info.node = pMgmt->pMnode; + dTrace("msg:%p, get from mnode-sync queue", pMsg); SMsgHead *pHead = pMsg->pCont; pHead->contLen = ntohl(pHead->contLen); @@ -70,64 +69,63 @@ static void mmProcessSyncQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { taosFreeQitem(pMsg); } -static int32_t mmPutNodeMsgToWorker(SSingleWorker *pWorker, SRpcMsg *pMsg) { - dTrace("msg:%p, put into %s queue, type:%s", pMsg, pWorker->name, TMSG_INFO(pMsg->msgType)); - taosWriteQitem(pWorker->queue, pMsg); - return 0; +static inline int32_t mmPutMsgToWorker(SMnodeMgmt *pMgmt, SSingleWorker *pWorker, SRpcMsg *pMsg) { + if (mmAcquire(pMgmt) == 0) { + dTrace("msg:%p, put into %s queue, type:%s", pMsg, pWorker->name, TMSG_INFO(pMsg->msgType)); + taosWriteQitem(pWorker->queue, pMsg); + mmRelease(pMgmt); + return 0; + } else { + dTrace("msg:%p, failed to put into %s queue since %s, type:%s", pMsg, pWorker->name, terrstr(), + TMSG_INFO(pMsg->msgType)); + return -1; + } } -int32_t mmPutNodeMsgToWriteQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { - return mmPutNodeMsgToWorker(&pMgmt->writeWorker, pMsg); +inline int32_t mmPutMsgToWriteQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { + return mmPutMsgToWorker(pMgmt, &pMgmt->writeWorker, pMsg); } -int32_t mmPutNodeMsgToSyncQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { - return mmPutNodeMsgToWorker(&pMgmt->syncWorker, pMsg); +inline int32_t mmPutMsgToSyncQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { + return mmPutMsgToWorker(pMgmt, &pMgmt->syncWorker, pMsg); } -int32_t mmPutNodeMsgToReadQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { - return mmPutNodeMsgToWorker(&pMgmt->readWorker, pMsg); +inline int32_t mmPutMsgToReadQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { + return mmPutMsgToWorker(pMgmt, &pMgmt->readWorker, pMsg); } -int32_t mmPutNodeMsgToQueryQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { - mndPreprocessQueryMsg(pMgmt->pMnode, pMsg); - - return mmPutNodeMsgToWorker(&pMgmt->queryWorker, pMsg); +inline int32_t mmPutMsgToQueryQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { + if (mndPreprocessQueryMsg(pMgmt->pMnode, pMsg) != 0) { + dError("msg:%p, failed to pre-process in mnode since %s, type:%s", pMsg, terrstr(), TMSG_INFO(pMsg->msgType)); + return -1; + } + return mmPutMsgToWorker(pMgmt, &pMgmt->queryWorker, pMsg); } -int32_t mmPutNodeMsgToMonitorQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { - return mmPutNodeMsgToWorker(&pMgmt->monitorWorker, pMsg); +inline int32_t mmPutMsgToMonitorQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { + return mmPutMsgToWorker(pMgmt, &pMgmt->monitorWorker, pMsg); } -int32_t mmPutRpcMsgToQueue(SMnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) { +inline int32_t mmPutMsgToQueue(SMnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) { SRpcMsg *pMsg = taosAllocateQitem(sizeof(SRpcMsg), RPC_QITEM); if (pMsg == NULL) return -1; - memcpy(pMsg, pRpc, sizeof(SRpcMsg)); + dTrace("msg:%p, is created", pMsg); + memcpy(pMsg, pRpc, sizeof(SRpcMsg)); switch (qtype) { case WRITE_QUEUE: - dTrace("msg:%p, is created and will put into vnode-write queue", pMsg); - taosWriteQitem(pMgmt->writeWorker.queue, pMsg); - return 0; + return mmPutMsgToWorker(pMgmt, &pMgmt->writeWorker, pMsg); case QUERY_QUEUE: - dTrace("msg:%p, is created and will put into vnode-query queue", pMsg); - taosWriteQitem(pMgmt->queryWorker.queue, pMsg); - return 0; - + return mmPutMsgToWorker(pMgmt, &pMgmt->queryWorker, pMsg); case READ_QUEUE: - dTrace("msg:%p, is created and will put into vnode-read queue", pMsg); - taosWriteQitem(pMgmt->readWorker.queue, pMsg); - return 0; + return mmPutMsgToWorker(pMgmt, &pMgmt->readWorker, pMsg); case SYNC_QUEUE: - if (mmAcquire(pMgmt) == 0) { - dTrace("msg:%p, is created and will put into vnode-sync queue", pMsg); - taosWriteQitem(pMgmt->syncWorker.queue, pMsg); - mmRelease(pMgmt); - return 0; - } else { - return -1; - } + return mmPutMsgToWorker(pMgmt, &pMgmt->syncWorker, pMsg); default: terrno = TSDB_CODE_INVALID_PARA; + dTrace("msg:%p, is freed", pMsg); + taosFreeQitem(pMsg); + rpcFreeCont(pMsg->pCont); return -1; } } @@ -137,7 +135,7 @@ int32_t mmStartWorker(SMnodeMgmt *pMgmt) { .min = tsNumOfMnodeQueryThreads, .max = tsNumOfMnodeQueryThreads, .name = "mnode-query", - .fp = (FItem)mmProcessQueue, + .fp = (FItem)mmProcessRpcMsg, .param = pMgmt, }; if (tSingleWorkerInit(&pMgmt->queryWorker, &qCfg) != 0) { @@ -149,7 +147,7 @@ int32_t mmStartWorker(SMnodeMgmt *pMgmt) { .min = tsNumOfMnodeReadThreads, .max = tsNumOfMnodeReadThreads, .name = "mnode-read", - .fp = (FItem)mmProcessQueue, + .fp = (FItem)mmProcessRpcMsg, .param = pMgmt, }; if (tSingleWorkerInit(&pMgmt->readWorker, &rCfg) != 0) { @@ -161,7 +159,7 @@ int32_t mmStartWorker(SMnodeMgmt *pMgmt) { .min = 1, .max = 1, .name = "mnode-write", - .fp = (FItem)mmProcessQueue, + .fp = (FItem)mmProcessRpcMsg, .param = pMgmt, }; if (tSingleWorkerInit(&pMgmt->writeWorker, &wCfg) != 0) { @@ -173,7 +171,7 @@ int32_t mmStartWorker(SMnodeMgmt *pMgmt) { .min = 1, .max = 1, .name = "mnode-sync", - .fp = (FItem)mmProcessSyncQueue, + .fp = (FItem)mmProcessSyncMsg, .param = pMgmt, }; if (tSingleWorkerInit(&pMgmt->syncWorker, &sCfg) != 0) { @@ -185,7 +183,7 @@ int32_t mmStartWorker(SMnodeMgmt *pMgmt) { .min = 1, .max = 1, .name = "mnode-monitor", - .fp = (FItem)mmProcessQueue, + .fp = (FItem)mmProcessRpcMsg, .param = pMgmt, }; if (tSingleWorkerInit(&pMgmt->monitorWorker, &mCfg) != 0) { From a04065769564d50b2d43e3df31fb566e638af790 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Tue, 14 Jun 2022 09:45:15 +0800 Subject: [PATCH 27/78] test: reopen one case --- tests/system-test/fulltest.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 5149994228..b40d9a7fbb 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -103,7 +103,7 @@ python3 ./test.py -f 6-cluster/5dnode3mnodeStop.py python3 ./test.py -f 7-tmq/basic5.py python3 ./test.py -f 7-tmq/subscribeDb.py python3 ./test.py -f 7-tmq/subscribeDb0.py -# python3 ./test.py -f 7-tmq/subscribeDb1.py +python3 ./test.py -f 7-tmq/subscribeDb1.py python3 ./test.py -f 7-tmq/subscribeStb.py python3 ./test.py -f 7-tmq/subscribeStb0.py python3 ./test.py -f 7-tmq/subscribeStb1.py From 3887e940f14c12e77f556f2fd04654417e8652be Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Tue, 14 Jun 2022 10:24:05 +0800 Subject: [PATCH 28/78] fix(sync): when receive snapshot rsp from a dropped replica --- source/libs/sync/src/syncMain.c | 21 ++++++++++++--------- source/libs/sync/src/syncSnapshot.c | 6 ++++++ 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 94bc755f4e..12d09ce26c 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -1968,8 +1968,9 @@ static int32_t syncNodeConfigChange(SSyncNode* ths, SRpcMsg* pRpcMsg, SSyncRaftE bool isDrop; if (IamInNew || (!IamInNew && ths->state != TAOS_SYNC_STATE_LEADER)) { - syncNodeUpdateConfig(ths, &newSyncCfg, pEntry->index, &isDrop); + syncNodeUpdateConfig(ths, &newSyncCfg, pEntry->index, &isDrop); + // change isStandBy to normal if (!isDrop) { if (ths->state == TAOS_SYNC_STATE_LEADER) { @@ -1978,14 +1979,16 @@ static int32_t syncNodeConfigChange(SSyncNode* ths, SRpcMsg* pRpcMsg, SSyncRaftE syncNodeBecomeFollower(ths, "config change"); } } - - if (gRaftDetailLog) { - char* sOld = syncCfg2Str(&oldSyncCfg); - char* sNew = syncCfg2Str(&newSyncCfg); - sInfo("==config change== 0x11 old:%s new:%s isDrop:%d \n", sOld, sNew, isDrop); - taosMemoryFree(sOld); - taosMemoryFree(sNew); - } + } else { + syncNodeBecomeFollower(ths, "config change"); + } + + if (gRaftDetailLog) { + char* sOld = syncCfg2Str(&oldSyncCfg); + char* sNew = syncCfg2Str(&newSyncCfg); + sInfo("==config change== 0x11 old:%s new:%s isDrop:%d index:%ld IamInNew:%d \n", sOld, sNew, isDrop, pEntry->index, IamInNew); + taosMemoryFree(sOld); + taosMemoryFree(sNew); } // always call FpReConfigCb diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index 38742220fe..9c43c36af3 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -755,6 +755,12 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { // sender receives ack, set seq = ack + 1, send msg from seq // if ack == SYNC_SNAPSHOT_SEQ_END, stop sender int32_t syncNodeOnSnapshotRspCb(SSyncNode *pSyncNode, SyncSnapshotRsp *pMsg) { + // if already drop replica, do not process + if (!syncNodeInRaftGroup(pSyncNode, &(pMsg->srcId)) && pSyncNode->state == TAOS_SYNC_STATE_LEADER) { + sInfo("recv SyncSnapshotRsp maybe replica already dropped"); + return 0; + } + // get sender SSyncSnapshotSender *pSender = syncNodeGetSnapshotSender(pSyncNode, &(pMsg->srcId)); ASSERT(pSender != NULL); From 01521a2541af1aa7cd62eb790163ce89407f5807 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 14 Jun 2022 10:41:15 +0800 Subject: [PATCH 29/78] refactor: mnode file --- source/dnode/mgmt/mgmt_mnode/inc/mmInt.h | 6 +- source/dnode/mgmt/mgmt_mnode/src/mmFile.c | 84 ++++++++------------- source/dnode/mgmt/mgmt_mnode/src/mmHandle.c | 2 +- source/dnode/mgmt/mgmt_mnode/src/mmInt.c | 24 +++--- source/dnode/mgmt/node_mgmt/src/dmEnv.c | 8 +- 5 files changed, 47 insertions(+), 77 deletions(-) diff --git a/source/dnode/mgmt/mgmt_mnode/inc/mmInt.h b/source/dnode/mgmt/mgmt_mnode/inc/mmInt.h index 2443493d0b..5dd1a5b3b8 100644 --- a/source/dnode/mgmt/mgmt_mnode/inc/mmInt.h +++ b/source/dnode/mgmt/mgmt_mnode/inc/mmInt.h @@ -34,16 +34,14 @@ typedef struct SMnodeMgmt { SSingleWorker writeWorker; SSingleWorker syncWorker; SSingleWorker monitorWorker; - SReplica replicas[TSDB_MAX_REPLICA]; - int8_t replica; bool stopped; int32_t refCount; TdThreadRwlock lock; } SMnodeMgmt; // mmFile.c -int32_t mmReadFile(SMnodeMgmt *pMgmt, bool *pDeployed); -int32_t mmWriteFile(SMnodeMgmt *pMgmt, SDCreateMnodeReq *pMsg, bool deployed); +int32_t mmReadFile(SMnodeMgmt *pMgmt, SReplica *pReplica, bool *pDeployed); +int32_t mmWriteFile(SMnodeMgmt *pMgmt, const SReplica *pReplica, bool deployed); // mmInt.c int32_t mmAcquire(SMnodeMgmt *pMgmt); diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmFile.c b/source/dnode/mgmt/mgmt_mnode/src/mmFile.c index 478d6abd52..e24ebcb9e2 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmFile.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmFile.c @@ -16,7 +16,7 @@ #define _DEFAULT_SOURCE #include "mmInt.h" -int32_t mmReadFile(SMnodeMgmt *pMgmt, bool *pDeployed) { +int32_t mmReadFile(SMnodeMgmt *pMgmt, SReplica *pReplica, bool *pDeployed) { int32_t code = TSDB_CODE_INVALID_JSON_FORMAT; int32_t len = 0; int32_t maxLen = 4096; @@ -52,45 +52,36 @@ int32_t mmReadFile(SMnodeMgmt *pMgmt, bool *pDeployed) { } *pDeployed = deployed->valueint; - cJSON *mnodes = cJSON_GetObjectItem(root, "mnodes"); - if (mnodes != NULL) { - if (!mnodes || mnodes->type != cJSON_Array) { - dError("failed to read %s since nodes not found", file); + cJSON *id = cJSON_GetObjectItem(root, "id"); + if (id) { + if (id->type != cJSON_Number) { + dError("failed to read %s since id not found", file); goto _OVER; } - - pMgmt->replica = cJSON_GetArraySize(mnodes); - if (pMgmt->replica <= 0 || pMgmt->replica > TSDB_MAX_REPLICA) { - dError("failed to read %s since mnodes size %d invalid", file, pMgmt->replica); - goto _OVER; - } - - for (int32_t i = 0; i < pMgmt->replica; ++i) { - cJSON *node = cJSON_GetArrayItem(mnodes, i); - if (node == NULL) break; - - SReplica *pReplica = &pMgmt->replicas[i]; - - cJSON *id = cJSON_GetObjectItem(node, "id"); - if (!id || id->type != cJSON_Number) { - dError("failed to read %s since id not found", file); - goto _OVER; - } + if (pReplica) { pReplica->id = id->valueint; + } + } - cJSON *fqdn = cJSON_GetObjectItem(node, "fqdn"); - if (!fqdn || fqdn->type != cJSON_String || fqdn->valuestring == NULL) { - dError("failed to read %s since fqdn not found", file); - goto _OVER; - } + cJSON *fqdn = cJSON_GetObjectItem(root, "fqdn"); + if (fqdn) { + if (fqdn->type != cJSON_String || fqdn->valuestring == NULL) { + dError("failed to read %s since fqdn not found", file); + goto _OVER; + } + if (pReplica) { tstrncpy(pReplica->fqdn, fqdn->valuestring, TSDB_FQDN_LEN); + } + } - cJSON *port = cJSON_GetObjectItem(node, "port"); - if (!port || port->type != cJSON_Number) { - dError("failed to read %s since port not found", file); - goto _OVER; - } - pReplica->port = port->valueint; + cJSON *port = cJSON_GetObjectItem(root, "port"); + if (port) { + if (port->type != cJSON_Number) { + dError("failed to read %s since port not found", file); + goto _OVER; + } + if (pReplica) { + pReplica->port = (uint16_t)port->valueint; } } @@ -106,7 +97,7 @@ _OVER: return code; } -int32_t mmWriteFile(SMnodeMgmt *pMgmt, SDCreateMnodeReq *pMsg, bool deployed) { +int32_t mmWriteFile(SMnodeMgmt *pMgmt, const SReplica *pReplica, bool deployed) { char file[PATH_MAX] = {0}; char realfile[PATH_MAX] = {0}; snprintf(file, sizeof(file), "%s%smnode.json.bak", pMgmt->path, TD_DIRSEP); @@ -124,26 +115,11 @@ int32_t mmWriteFile(SMnodeMgmt *pMgmt, SDCreateMnodeReq *pMsg, bool deployed) { char *content = taosMemoryCalloc(1, maxLen + 1); len += snprintf(content + len, maxLen - len, "{\n"); - - int8_t replica = (pMsg != NULL ? pMsg->replica : pMgmt->replica); - if (replica > 0) { - len += snprintf(content + len, maxLen - len, " \"mnodes\": [{\n"); - for (int32_t i = 0; i < replica; ++i) { - SReplica *pReplica = &pMgmt->replicas[i]; - if (pMsg != NULL) { - pReplica = &pMsg->replicas[i]; - } - len += snprintf(content + len, maxLen - len, " \"id\": %d,\n", pReplica->id); - len += snprintf(content + len, maxLen - len, " \"fqdn\": \"%s\",\n", pReplica->fqdn); - len += snprintf(content + len, maxLen - len, " \"port\": %u\n", pReplica->port); - if (i < replica - 1) { - len += snprintf(content + len, maxLen - len, " },{\n"); - } else { - len += snprintf(content + len, maxLen - len, " }],\n"); - } - } + if (pReplica != NULL && pReplica->id > 0) { + len += snprintf(content + len, maxLen - len, " \"id\": %d,\n", pReplica->id); + len += snprintf(content + len, maxLen - len, " \"fqdn\": \"%s\",\n", pReplica->fqdn); + len += snprintf(content + len, maxLen - len, " \"port\": %u\n,", pReplica->port); } - len += snprintf(content + len, maxLen - len, " \"deployed\": %d\n", deployed); len += snprintf(content + len, maxLen - len, "}\n"); diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c index f151a09b94..124876d718 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c @@ -90,7 +90,7 @@ int32_t mmProcessCreateReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) { SMnodeMgmt mgmt = {0}; mgmt.path = pInput->path; mgmt.name = pInput->name; - if (mmWriteFile(&mgmt, &createReq, deployed) != 0) { + if (mmWriteFile(&mgmt, &createReq.replicas[0], deployed) != 0) { dError("failed to write mnode file since %s", terrstr()); return -1; } diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmInt.c b/source/dnode/mgmt/mgmt_mnode/src/mmInt.c index cb942e8d1c..7e73e53481 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmInt.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmInt.c @@ -27,7 +27,7 @@ static bool mmDeployRequired(const SMgmtInputOpt *pInput) { static int32_t mmRequire(const SMgmtInputOpt *pInput, bool *required) { SMnodeMgmt mgmt = {0}; mgmt.path = pInput->path; - if (mmReadFile(&mgmt, required) != 0) { + if (mmReadFile(&mgmt, NULL, required) != 0) { return -1; } @@ -52,23 +52,17 @@ static void mmBuildOptionForDeploy(SMnodeMgmt *pMgmt, const SMgmtInputOpt *pInpu tstrncpy(pReplica->fqdn, tsLocalFqdn, TSDB_FQDN_LEN); } -static void mmBuildOptionForOpen(SMnodeMgmt *pMgmt, SMnodeOpt *pOption) { +static void mmBuildOptionForOpen(SMnodeMgmt *pMgmt, const SReplica *pReplica, SMnodeOpt *pOption) { pOption->standby = false; pOption->deploy = false; pOption->msgCb = pMgmt->msgCb; pOption->dnodeId = pMgmt->pData->dnodeId; - if (pMgmt->replica > 0) { + if (pReplica->id > 0) { pOption->standby = true; pOption->replica = 1; pOption->selfIndex = 0; - SReplica *pReplica = &pOption->replicas[0]; - for (int32_t i = 0; i < pMgmt->replica; ++i) { - if (pMgmt->replicas[i].id != pMgmt->pData->dnodeId) continue; - pReplica->id = pMgmt->replicas[i].id; - pReplica->port = pMgmt->replicas[i].port; - memcpy(pReplica->fqdn, pMgmt->replicas[i].fqdn, TSDB_FQDN_LEN); - } + memcpy(&pOption->replicas[0], pReplica, sizeof(SReplica)); } } @@ -108,8 +102,9 @@ static int32_t mmOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) { pMgmt->msgCb.mgmt = pMgmt; taosThreadRwlockInit(&pMgmt->lock, NULL); - bool deployed = false; - if (mmReadFile(pMgmt, &deployed) != 0) { + bool deployed = false; + SReplica replica = {0}; + if (mmReadFile(pMgmt, &replica, &deployed) != 0) { dError("failed to read file since %s", terrstr()); mmClose(pMgmt); return -1; @@ -122,7 +117,7 @@ static int32_t mmOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) { mmBuildOptionForDeploy(pMgmt, pInput, &option); } else { dInfo("mnode start to open"); - mmBuildOptionForOpen(pMgmt, &option); + mmBuildOptionForOpen(pMgmt, &replica, &option); } pMgmt->pMnode = mndOpen(pMgmt->path, &option); @@ -140,8 +135,7 @@ static int32_t mmOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) { } tmsgReportStartup("mnode-worker", "initialized"); - if (!deployed || pMgmt->replica > 0) { - pMgmt->replica = 0; + if (!deployed || replica.id > 0) { deployed = true; if (mmWriteFile(pMgmt, NULL, deployed) != 0) { dError("failed to write mnode file since %s", terrstr()); diff --git a/source/dnode/mgmt/node_mgmt/src/dmEnv.c b/source/dnode/mgmt/node_mgmt/src/dmEnv.c index 528beb280b..498af36786 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmEnv.c +++ b/source/dnode/mgmt/node_mgmt/src/dmEnv.c @@ -123,10 +123,12 @@ static int32_t dmProcessCreateNodeReq(EDndNodeType ntype, SRpcMsg *pMsg) { dError("node:%s, failed to create since %s", pWrapper->name, terrstr()); } else { dInfo("node:%s, has been created", pWrapper->name); - (void)dmOpenNode(pWrapper); - (void)dmStartNode(pWrapper); - pWrapper->required = true; + code = dmOpenNode(pWrapper); + if (code != 0) { + code = dmStartNode(pWrapper); + } pWrapper->deployed = true; + pWrapper->required = true; pWrapper->proc.ptype = pDnode->ptype; } From b5bc9aa2bae4dc9df60577b0b6d0cacec53f0107 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 14 Jun 2022 10:44:34 +0800 Subject: [PATCH 30/78] fix rpc retry error --- source/libs/transport/src/transCli.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index d1debc6af6..4852dcfb54 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -983,6 +983,9 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { SEpSet epSet = {0}; tDeserializeSEpSet(pResp->pCont, pResp->contLen, &epSet); pCtx->epSet = epSet; + if (!transEpSetIsEqual(&epSet, &pCtx->epSet)) { + pCtx->retryCount = 0; + } } addConnToPool(pThrd->pool, pConn); tTrace("use remote epset, current in use: %d, retry count:%d, try limit: %d", pEpSet->inUse, pCtx->retryCount + 1, From 589b30422f180cc990a371cc4d0fdd36843d26d4 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 14 Jun 2022 10:51:00 +0800 Subject: [PATCH 31/78] add last function splitting --- include/libs/function/functionMgt.h | 2 ++ source/libs/function/inc/builtinsimpl.h | 1 + source/libs/function/src/builtins.c | 22 ++++++++++++++++++++++ 3 files changed, 25 insertions(+) diff --git a/include/libs/function/functionMgt.h b/include/libs/function/functionMgt.h index e787639897..b5641f2d07 100644 --- a/include/libs/function/functionMgt.h +++ b/include/libs/function/functionMgt.h @@ -140,6 +140,8 @@ typedef enum EFunctionType { FUNCTION_TYPE_BOTTOM_MERGE, FUNCTION_TYPE_FIRST_PARTIAL, FUNCTION_TYPE_FIRST_MERGE, + FUNCTION_TYPE_LAST_PARTIAL, + FUNCTION_TYPE_LAST_MERGE, // user defined funcion FUNCTION_TYPE_UDF = 10000 diff --git a/source/libs/function/inc/builtinsimpl.h b/source/libs/function/inc/builtinsimpl.h index af9b8d4a10..7db1396603 100644 --- a/source/libs/function/inc/builtinsimpl.h +++ b/source/libs/function/inc/builtinsimpl.h @@ -95,6 +95,7 @@ bool getFirstLastFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); int32_t firstFunction(SqlFunctionCtx *pCtx); int32_t firstFunctionMerge(SqlFunctionCtx *pCtx); int32_t lastFunction(SqlFunctionCtx *pCtx); +int32_t lastFunctionMerge(SqlFunctionCtx *pCtx); int32_t firstLastFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); int32_t firstLastPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); int32_t firstCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx); diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 0a6d99c52d..7a06304487 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -1776,6 +1776,28 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .finalizeFunc = firstLastFinalize, .combineFunc = lastCombine, }, + { + .name = "_last_partial", + .type = FUNCTION_TYPE_LAST_PARTIAL, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_TIMELINE_FUNC, + .translateFunc = translateFirstLastPartial, + .getEnvFunc = getFirstLastFuncEnv, + .initFunc = functionSetup, + .processFunc = lastFunction, + .finalizeFunc = firstLastPartialFinalize, + .combineFunc = lastCombine, + }, + { + .name = "_last_merge", + .type = FUNCTION_TYPE_LAST_MERGE, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_TIMELINE_FUNC, + .translateFunc = translateFirstLastMerge, + .getEnvFunc = getFirstLastFuncEnv, + .initFunc = functionSetup, + .processFunc = lastFunctionMerge, + .finalizeFunc = firstLastFinalize, + .combineFunc = lastCombine, + }, { .name = "twa", .type = FUNCTION_TYPE_TWA, From a3f12736407af1bbe8ed75435fde928b70167ef9 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 14 Jun 2022 10:58:36 +0800 Subject: [PATCH 32/78] refactor: mnode sync --- include/dnode/mnode/mnode.h | 4 +--- source/dnode/mgmt/mgmt_mnode/src/mmFile.c | 4 +++- source/dnode/mgmt/mgmt_mnode/src/mmHandle.c | 2 +- source/dnode/mgmt/mgmt_mnode/src/mmInt.c | 15 ++++----------- source/dnode/mnode/impl/inc/mndInt.h | 14 ++++++-------- source/dnode/mnode/impl/src/mndDnode.c | 4 ++-- source/dnode/mnode/impl/src/mndMain.c | 4 +--- source/dnode/mnode/impl/src/mndSync.c | 18 +++++++++--------- 8 files changed, 27 insertions(+), 38 deletions(-) diff --git a/include/dnode/mnode/mnode.h b/include/dnode/mnode/mnode.h index 83183422c2..99b704826d 100644 --- a/include/dnode/mnode/mnode.h +++ b/include/dnode/mnode/mnode.h @@ -32,9 +32,7 @@ typedef struct { int32_t dnodeId; bool standby; bool deploy; - int8_t replica; - int8_t selfIndex; - SReplica replicas[TSDB_MAX_REPLICA]; + SReplica replica; SMsgCb msgCb; } SMnodeOpt; diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmFile.c b/source/dnode/mgmt/mgmt_mnode/src/mmFile.c index e24ebcb9e2..27a35ae17a 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmFile.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmFile.c @@ -86,12 +86,14 @@ int32_t mmReadFile(SMnodeMgmt *pMgmt, SReplica *pReplica, bool *pDeployed) { } code = 0; - dDebug("succcessed to read file %s, deployed:%d", file, *pDeployed); _OVER: if (content != NULL) taosMemoryFree(content); if (root != NULL) cJSON_Delete(root); if (pFile != NULL) taosCloseFile(&pFile); + if (code == 0) { + dDebug("succcessed to read file %s, deployed:%d", file, *pDeployed); + } terrno = code; return code; diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c index 124876d718..e91f4b8cf4 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c @@ -126,7 +126,7 @@ int32_t mmProcessDropReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) { SArray *mmGetMsgHandles() { int32_t code = -1; - SArray *pArray = taosArrayInit(64, sizeof(SMgmtHandle)); + SArray *pArray = taosArrayInit(128, sizeof(SMgmtHandle)); if (pArray == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_MNODE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmInt.c b/source/dnode/mgmt/mgmt_mnode/src/mmInt.c index 7e73e53481..bc67b8442e 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmInt.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmInt.c @@ -43,13 +43,9 @@ static void mmBuildOptionForDeploy(SMnodeMgmt *pMgmt, const SMgmtInputOpt *pInpu pOption->deploy = true; pOption->msgCb = pMgmt->msgCb; pOption->dnodeId = pMgmt->pData->dnodeId; - pOption->replica = 1; - pOption->selfIndex = 0; - - SReplica *pReplica = &pOption->replicas[0]; - pReplica->id = 1; - pReplica->port = tsServerPort; - tstrncpy(pReplica->fqdn, tsLocalFqdn, TSDB_FQDN_LEN); + pOption->replica.id = 1; + pOption->replica.port = tsServerPort; + tstrncpy(pOption->replica.fqdn, tsLocalFqdn, TSDB_FQDN_LEN); } static void mmBuildOptionForOpen(SMnodeMgmt *pMgmt, const SReplica *pReplica, SMnodeOpt *pOption) { @@ -57,12 +53,9 @@ static void mmBuildOptionForOpen(SMnodeMgmt *pMgmt, const SReplica *pReplica, SM pOption->deploy = false; pOption->msgCb = pMgmt->msgCb; pOption->dnodeId = pMgmt->pData->dnodeId; - if (pReplica->id > 0) { pOption->standby = true; - pOption->replica = 1; - pOption->selfIndex = 0; - memcpy(&pOption->replicas[0], pReplica, sizeof(SReplica)); + pOption->replica = *pReplica; } } diff --git a/source/dnode/mnode/impl/inc/mndInt.h b/source/dnode/mnode/impl/inc/mndInt.h index 4869a19856..16220f101a 100644 --- a/source/dnode/mnode/impl/inc/mndInt.h +++ b/source/dnode/mnode/impl/inc/mndInt.h @@ -76,11 +76,12 @@ typedef struct { } STelemMgmt; typedef struct { - sem_t syncSem; - int64_t sync; - bool standby; - int32_t errCode; - int32_t transId; + sem_t syncSem; + int64_t sync; + bool standby; + SReplica replica; + int32_t errCode; + int32_t transId; } SSyncMgmt; typedef struct { @@ -98,9 +99,6 @@ typedef struct SMnode { bool stopped; bool restored; bool deploy; - int8_t replica; - int8_t selfIndex; - SReplica replicas[TSDB_MAX_REPLICA]; char *path; int64_t checkTime; SSdb *pSdb; diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 73eea70195..c936c0f93d 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -95,8 +95,8 @@ static int32_t mndCreateDefaultDnode(SMnode *pMnode) { dnodeObj.id = 1; dnodeObj.createdTime = taosGetTimestampMs(); dnodeObj.updateTime = dnodeObj.createdTime; - dnodeObj.port = pMnode->replicas[0].port; - memcpy(&dnodeObj.fqdn, pMnode->replicas[0].fqdn, TSDB_FQDN_LEN); + dnodeObj.port = tsServerPort; + memcpy(&dnodeObj.fqdn, tsLocalFqdn, TSDB_FQDN_LEN); snprintf(dnodeObj.ep, TSDB_EP_LEN, "%s:%u", dnodeObj.fqdn, dnodeObj.port); pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, NULL); diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index 7a73d3c360..892650fa18 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -289,11 +289,9 @@ static int32_t mndExecSteps(SMnode *pMnode) { } static void mndSetOptions(SMnode *pMnode, const SMnodeOpt *pOption) { - pMnode->replica = pOption->replica; - pMnode->selfIndex = pOption->selfIndex; - memcpy(&pMnode->replicas, pOption->replicas, sizeof(SReplica) * TSDB_MAX_REPLICA); pMnode->msgCb = pOption->msgCb; pMnode->selfDnodeId = pOption->dnodeId; + pMnode->syncMgmt.replica = pOption->replica; pMnode->syncMgmt.standby = pOption->standby; } diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index e0b4cc6a57..e58012b1b7 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -188,15 +188,15 @@ int32_t mndInitSync(SMnode *pMnode) { syncInfo.isStandBy = pMgmt->standby; syncInfo.snapshotEnable = true; - SSyncCfg *pCfg = &syncInfo.syncCfg; - pCfg->replicaNum = pMnode->replica; - pCfg->myIndex = pMnode->selfIndex; - mInfo("start to open mnode sync, replica:%d myindex:%d standby:%d", pCfg->replicaNum, pCfg->myIndex, pMgmt->standby); - for (int32_t i = 0; i < pMnode->replica; ++i) { - SNodeInfo *pNode = &pCfg->nodeInfo[i]; - tstrncpy(pNode->nodeFqdn, pMnode->replicas[i].fqdn, sizeof(pNode->nodeFqdn)); - pNode->nodePort = pMnode->replicas[i].port; - mInfo("index:%d, fqdn:%s port:%d", i, pNode->nodeFqdn, pNode->nodePort); + mInfo("start to open mnode sync, standby:%d", pMgmt->standby); + if (pMgmt->standby || pMgmt->replica.id > 0) { + SSyncCfg *pCfg = &syncInfo.syncCfg; + pCfg->replicaNum = 1; + pCfg->myIndex = 0; + SNodeInfo *pNode = &pCfg->nodeInfo[0]; + tstrncpy(pNode->nodeFqdn, pMgmt->replica.fqdn, sizeof(pNode->nodeFqdn)); + pNode->nodePort = pMgmt->replica.port; + mInfo("fqdn:%s port:%u", pNode->nodeFqdn, pNode->nodePort); } tsem_init(&pMgmt->syncSem, 0, 0); From 6a4fc4577126119b4ffc3d2f0285862a8b656647 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 14 Jun 2022 11:00:22 +0800 Subject: [PATCH 33/78] fix bugs --- source/libs/function/src/builtinsimpl.c | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index f2c5df8fc2..fe52e9c7c5 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -2355,12 +2355,13 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { int32_t numOfElems = 0; SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); - char* buf = GET_ROWCELL_INTERBUF(pResInfo); + SFirstLastRes *pInfo = GET_ROWCELL_INTERBUF(pResInfo); SInputColumnInfoData* pInput = &pCtx->input; SColumnInfoData* pInputCol = pInput->pData[0]; int32_t bytes = pInputCol->info.bytes; + pInfo->bytes = bytes; // All null data column, return directly. if (pInput->colDataAggIsSet && (pInput->pColumnDataAgg[0]->numOfNull == pInput->totalRows)) { @@ -2385,9 +2386,9 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { char* data = colDataGetData(pInputCol, i); TSKEY cts = getRowPTs(pInput->pPTS, i); - if (pResInfo->numOfRes == 0 || *(TSKEY*)(buf + bytes) < cts) { - memcpy(buf, data, bytes); - *(TSKEY*)(buf + bytes) = cts; + if (pResInfo->numOfRes == 0 || *(TSKEY*)(pInfo->buf + bytes) < cts) { + memcpy(pInfo->buf, data, bytes); + *(TSKEY*)(pInfo->buf + bytes) = cts; // DO_UPDATE_TAG_COLUMNS(pCtx, ts); pResInfo->numOfRes = 1; } @@ -2403,9 +2404,9 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { char* data = colDataGetData(pInputCol, i); TSKEY cts = getRowPTs(pInput->pPTS, i); - if (pResInfo->numOfRes == 0 || *(TSKEY*)(buf + bytes) < cts) { - memcpy(buf, data, bytes); - *(TSKEY*)(buf + bytes) = cts; + if (pResInfo->numOfRes == 0 || *(TSKEY*)(pInfo->buf + bytes) < cts) { + memcpy(pInfo->buf, data, bytes); + *(TSKEY*)(pInfo->buf + bytes) = cts; pResInfo->numOfRes = 1; // DO_UPDATE_TAG_COLUMNS(pCtx, ts); } From 6f27ee975e758cf4af935b0d554c740412f9c2b7 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 14 Jun 2022 11:04:36 +0800 Subject: [PATCH 34/78] enable last splitting --- source/libs/function/src/builtins.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 7a06304487..1c3f274ee9 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -1774,6 +1774,8 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .initFunc = functionSetup, .processFunc = lastFunction, .finalizeFunc = firstLastFinalize, + .pPartialFunc = "_last_partial", + .pMergeFunc = "_last_merge", .combineFunc = lastCombine, }, { From d93f4c4a83d906e729cbf8fc344cdfa4c5b14254 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 14 Jun 2022 11:10:36 +0800 Subject: [PATCH 35/78] refactor: mnode worker --- source/dnode/mgmt/mgmt_mnode/inc/mmInt.h | 4 -- source/dnode/mgmt/mgmt_mnode/src/mmInt.c | 19 -------- source/dnode/mgmt/mgmt_mnode/src/mmWorker.c | 48 +++++++++++++++------ 3 files changed, 35 insertions(+), 36 deletions(-) diff --git a/source/dnode/mgmt/mgmt_mnode/inc/mmInt.h b/source/dnode/mgmt/mgmt_mnode/inc/mmInt.h index 5dd1a5b3b8..c5c3d76f1e 100644 --- a/source/dnode/mgmt/mgmt_mnode/inc/mmInt.h +++ b/source/dnode/mgmt/mgmt_mnode/inc/mmInt.h @@ -43,10 +43,6 @@ typedef struct SMnodeMgmt { int32_t mmReadFile(SMnodeMgmt *pMgmt, SReplica *pReplica, bool *pDeployed); int32_t mmWriteFile(SMnodeMgmt *pMgmt, const SReplica *pReplica, bool deployed); -// mmInt.c -int32_t mmAcquire(SMnodeMgmt *pMgmt); -void mmRelease(SMnodeMgmt *pMgmt); - // mmHandle.c SArray *mmGetMsgHandles(); int32_t mmProcessCreateReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg); diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmInt.c b/source/dnode/mgmt/mgmt_mnode/src/mmInt.c index bc67b8442e..b7124dfaa5 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmInt.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmInt.c @@ -164,22 +164,3 @@ SMgmtFunc mmGetMgmtFunc() { return mgmtFunc; } - -int32_t mmAcquire(SMnodeMgmt *pMgmt) { - int32_t code = 0; - - taosThreadRwlockRdlock(&pMgmt->lock); - if (pMgmt->stopped) { - code = -1; - } else { - atomic_add_fetch_32(&pMgmt->refCount, 1); - } - taosThreadRwlockUnlock(&pMgmt->lock); - return code; -} - -void mmRelease(SMnodeMgmt *pMgmt) { - taosThreadRwlockRdlock(&pMgmt->lock); - atomic_sub_fetch_32(&pMgmt->refCount, 1); - taosThreadRwlockUnlock(&pMgmt->lock); -} \ No newline at end of file diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c index 6da9bfc528..17581fcef7 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c @@ -16,6 +16,25 @@ #define _DEFAULT_SOURCE #include "mmInt.h" +static inline int32_t mmAcquire(SMnodeMgmt *pMgmt) { + int32_t code = 0; + + taosThreadRwlockRdlock(&pMgmt->lock); + if (pMgmt->stopped) { + code = -1; + } else { + atomic_add_fetch_32(&pMgmt->refCount, 1); + } + taosThreadRwlockUnlock(&pMgmt->lock); + return code; +} + +static inline void mmRelease(SMnodeMgmt *pMgmt) { + taosThreadRwlockRdlock(&pMgmt->lock); + atomic_sub_fetch_32(&pMgmt->refCount, 1); + taosThreadRwlockUnlock(&pMgmt->lock); +} + static inline void mmSendRsp(SRpcMsg *pMsg, int32_t code) { SRpcMsg rsp = { .code = code, @@ -107,27 +126,30 @@ inline int32_t mmPutMsgToMonitorQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { } inline int32_t mmPutMsgToQueue(SMnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) { - SRpcMsg *pMsg = taosAllocateQitem(sizeof(SRpcMsg), RPC_QITEM); - if (pMsg == NULL) return -1; - dTrace("msg:%p, is created", pMsg); - - memcpy(pMsg, pRpc, sizeof(SRpcMsg)); + SSingleWorker *pWorker = NULL; switch (qtype) { case WRITE_QUEUE: - return mmPutMsgToWorker(pMgmt, &pMgmt->writeWorker, pMsg); + pWorker = &pMgmt->writeWorker; + break; case QUERY_QUEUE: - return mmPutMsgToWorker(pMgmt, &pMgmt->queryWorker, pMsg); + pWorker = &pMgmt->queryWorker; + break; case READ_QUEUE: - return mmPutMsgToWorker(pMgmt, &pMgmt->readWorker, pMsg); + pWorker = &pMgmt->readWorker; + break; case SYNC_QUEUE: - return mmPutMsgToWorker(pMgmt, &pMgmt->syncWorker, pMsg); + pWorker = &pMgmt->syncWorker; + break; default: terrno = TSDB_CODE_INVALID_PARA; - dTrace("msg:%p, is freed", pMsg); - taosFreeQitem(pMsg); - rpcFreeCont(pMsg->pCont); - return -1; } + + if (pWorker == NULL) return -1; + SRpcMsg *pMsg = taosAllocateQitem(sizeof(SRpcMsg), RPC_QITEM); + if (pMsg == NULL) return -1; + dTrace("msg:%p, is created and will put int %s queue", pMsg, pWorker->name); + + return mmPutMsgToWorker(pMgmt, pWorker, pMsg); } int32_t mmStartWorker(SMnodeMgmt *pMgmt) { From 22f86ee52978c26d91ff137c1b552f7a4d7fa389 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Tue, 14 Jun 2022 11:15:10 +0800 Subject: [PATCH 36/78] feat(stream): partition by --- source/libs/executor/src/groupoperator.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index 132f93a6a5..f663f5798a 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -582,6 +582,15 @@ int32_t* setupColumnOffset(const SSDataBlock* pBlock, int32_t rowCapacity) { return offset; } +static void clearPartitionOperator(SPartitionOperatorInfo* pInfo) { + void *ite = NULL; + while( (ite = taosHashIterate(pInfo->pGroupSet, ite)) != NULL ) { + taosArrayDestroy( ((SDataGroupInfo *)ite)->pPageList); + } + taosHashClear(pInfo->pGroupSet); + clearDiskbasedBuf(pInfo->pBuf); +} + static SSDataBlock* buildPartitionResult(SOperatorInfo* pOperator) { SPartitionOperatorInfo* pInfo = pOperator->info; @@ -591,6 +600,7 @@ static SSDataBlock* buildPartitionResult(SOperatorInfo* pOperator) { pInfo->pGroupIter = taosHashIterate(pInfo->pGroupSet, pInfo->pGroupIter); if (pInfo->pGroupIter == NULL) { doSetOperatorCompleted(pOperator); + clearPartitionOperator(pInfo); return NULL; } From 63927d15fab259f84e78e710ce1011fb3aff8ad6 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 14 Jun 2022 11:15:17 +0800 Subject: [PATCH 37/78] fix: compile err in windows --- source/dnode/mgmt/mgmt_mnode/src/mmWorker.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c index 17581fcef7..f02f90962b 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c @@ -101,19 +101,19 @@ static inline int32_t mmPutMsgToWorker(SMnodeMgmt *pMgmt, SSingleWorker *pWorker } } -inline int32_t mmPutMsgToWriteQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { +int32_t mmPutMsgToWriteQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { return mmPutMsgToWorker(pMgmt, &pMgmt->writeWorker, pMsg); } -inline int32_t mmPutMsgToSyncQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { +int32_t mmPutMsgToSyncQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { return mmPutMsgToWorker(pMgmt, &pMgmt->syncWorker, pMsg); } -inline int32_t mmPutMsgToReadQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { +int32_t mmPutMsgToReadQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { return mmPutMsgToWorker(pMgmt, &pMgmt->readWorker, pMsg); } -inline int32_t mmPutMsgToQueryQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { +int32_t mmPutMsgToQueryQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { if (mndPreprocessQueryMsg(pMgmt->pMnode, pMsg) != 0) { dError("msg:%p, failed to pre-process in mnode since %s, type:%s", pMsg, terrstr(), TMSG_INFO(pMsg->msgType)); return -1; @@ -121,11 +121,11 @@ inline int32_t mmPutMsgToQueryQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { return mmPutMsgToWorker(pMgmt, &pMgmt->queryWorker, pMsg); } -inline int32_t mmPutMsgToMonitorQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { +int32_t mmPutMsgToMonitorQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { return mmPutMsgToWorker(pMgmt, &pMgmt->monitorWorker, pMsg); } -inline int32_t mmPutMsgToQueue(SMnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) { +int32_t mmPutMsgToQueue(SMnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) { SSingleWorker *pWorker = NULL; switch (qtype) { case WRITE_QUEUE: From 313d2f0e4dbcd7d647d9a8bbd511e2151f9c08bc Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 14 Jun 2022 11:18:30 +0800 Subject: [PATCH 38/78] test: enable mnode/basic3.sim --- tests/script/jenkins/basic.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 1c694bf7fb..db8a055362 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -58,7 +58,7 @@ # ---- mnode ./test.sh -f tsim/mnode/basic1.sim ./test.sh -f tsim/mnode/basic2.sim -#./test.sh -f tsim/mnode/basic3.sim +./test.sh -f tsim/mnode/basic3.sim ./test.sh -f tsim/mnode/basic4.sim ./test.sh -f tsim/mnode/basic5.sim From 0dde9cc9f95c9d6aebabc8a3b4b2feb2be12ffc3 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 14 Jun 2022 11:25:11 +0800 Subject: [PATCH 39/78] fix bugs --- source/libs/function/src/builtinsimpl.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index fe52e9c7c5..b5009b7d41 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -2390,6 +2390,7 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { memcpy(pInfo->buf, data, bytes); *(TSKEY*)(pInfo->buf + bytes) = cts; // DO_UPDATE_TAG_COLUMNS(pCtx, ts); + pInfo->hasResult = true; pResInfo->numOfRes = 1; } break; @@ -2407,6 +2408,7 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { if (pResInfo->numOfRes == 0 || *(TSKEY*)(pInfo->buf + bytes) < cts) { memcpy(pInfo->buf, data, bytes); *(TSKEY*)(pInfo->buf + bytes) = cts; + pInfo->hasResult = true; pResInfo->numOfRes = 1; // DO_UPDATE_TAG_COLUMNS(pCtx, ts); } From 4e7aac8171a99b41c3c020e54046bbe95dc1d0d0 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 14 Jun 2022 11:34:51 +0800 Subject: [PATCH 40/78] fix: crash in mnode --- source/dnode/mgmt/mgmt_mnode/src/mmWorker.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c index f02f90962b..ee2df5f089 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c @@ -147,8 +147,9 @@ int32_t mmPutMsgToQueue(SMnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) { if (pWorker == NULL) return -1; SRpcMsg *pMsg = taosAllocateQitem(sizeof(SRpcMsg), RPC_QITEM); if (pMsg == NULL) return -1; - dTrace("msg:%p, is created and will put int %s queue", pMsg, pWorker->name); + memcpy(pMsg, pRpc, sizeof(SRpcMsg)); + dTrace("msg:%p, is created and will put int %s queue", pMsg, pWorker->name); return mmPutMsgToWorker(pMgmt, pWorker, pMsg); } From d44c6a810bb288839d18fb68d88070d807a2a2ed Mon Sep 17 00:00:00 2001 From: gccgdb1234 Date: Tue, 14 Jun 2022 11:54:55 +0800 Subject: [PATCH 41/78] docs: refine keywords section and add delete keywords --- docs/en/12-taos-sql/12-keywords.md | 311 +++++++++++++++++++++++++---- docs/zh/12-taos-sql/12-keywords.md | 310 ++++++++++++++++++++++++---- 2 files changed, 536 insertions(+), 85 deletions(-) diff --git a/docs/en/12-taos-sql/12-keywords.md b/docs/en/12-taos-sql/12-keywords.md index ed0c96b4e4..6d40deb5a6 100644 --- a/docs/en/12-taos-sql/12-keywords.md +++ b/docs/en/12-taos-sql/12-keywords.md @@ -4,50 +4,275 @@ title: Keywords There are about 200 keywords reserved by TDengine, they can't be used as the name of database, STable or table with either upper case, lower case or mixed case. -**Keywords List** +## Keywords List -| | | | | | -| ----------- | ---------- | --------- | ---------- | ------------ | -| ABORT | CREATE | IGNORE | NULL | STAR | -| ACCOUNT | CTIME | IMMEDIATE | OF | STATE | -| ACCOUNTS | DATABASE | IMPORT | OFFSET | STATEMENT | -| ADD | DATABASES | IN | OR | STATE_WINDOW | -| AFTER | DAYS | INITIALLY | ORDER | STORAGE | -| ALL | DBS | INSERT | PARTITIONS | STREAM | -| ALTER | DEFERRED | INSTEAD | PASS | STREAMS | -| AND | DELIMITERS | INT | PLUS | STRING | -| AS | DESC | INTEGER | PPS | SYNCDB | -| ASC | DESCRIBE | INTERVAL | PRECISION | TABLE | -| ATTACH | DETACH | INTO | PREV | TABLES | -| BEFORE | DISTINCT | IS | PRIVILEGE | TAG | -| BEGIN | DIVIDE | ISNULL | QTIME | TAGS | -| BETWEEN | DNODE | JOIN | QUERIES | TBNAME | -| BIGINT | DNODES | KEEP | QUERY | TIMES | -| BINARY | DOT | KEY | QUORUM | TIMESTAMP | -| BITAND | DOUBLE | KILL | RAISE | TINYINT | -| BITNOT | DROP | LE | REM | TOPIC | -| BITOR | EACH | LIKE | REPLACE | TOPICS | -| BLOCKS | END | LIMIT | REPLICA | TRIGGER | -| BOOL | EQ | LINEAR | RESET | TSERIES | -| BY | EXISTS | LOCAL | RESTRICT | UMINUS | -| CACHE | EXPLAIN | LP | ROW | UNION | -| CACHELAST | FAIL | LSHIFT | RP | UNSIGNED | -| CASCADE | FILE | LT | RSHIFT | UPDATE | -| CHANGE | FILL | MATCH | SCORES | UPLUS | -| CLUSTER | FLOAT | MAXROWS | SELECT | USE | -| COLON | FOR | MINROWS | SEMI | USER | -| COLUMN | FROM | MINUS | SESSION | USERS | -| COMMA | FSYNC | MNODES | SET | USING | -| COMP | GE | MODIFY | SHOW | VALUES | -| COMPACT | GLOB | MODULES | SLASH | VARIABLE | -| CONCAT | GRANTS | NCHAR | SLIDING | VARIABLES | -| CONFLICT | GROUP | NE | SLIMIT | VGROUPS | -| CONNECTION | GT | NONE | SMALLINT | VIEW | -| CONNECTIONS | HAVING | NOT | SOFFSET | VNODES | -| CONNS | ID | NOTNULL | STable | WAL | -| COPY | IF | NOW | STableS | WHERE | -| _C0 | _QSTART | _QSTOP | _QDURATION | _WSTART | -| _WSTOP | _WDURATION | _ROWTS | +### A + +- ABORT +- ACCOUNT +- ACCOUNTS +- ADD +- AFTER +- ALL +- ALTER +- AND +- AS +- ASC +- ATTACH + +### B + +- BEFORE +- BEGIN +- BETWEEN +- BIGINT +- BINARY +- BITAND +- BITNOT +- BITOR +- BLOCKS +- BOOL +- BY + +### C + +- CACHE +- CACHELAST +- CASCADE +- CHANGE +- CLUSTER +- COLON +- COLUMN +- COMMA +- COMP +- COMPACT +- CONCAT +- CONFLICT +- CONNECTION +- CONNECTIONS +- CONNS +- COPY +- CREATE +- CTIME + +### D + +- DATABASE +- DATABASES +- DAYS +- DBS +- DEFERRED +- DELETE +- DELIMITERS +- DESC +- DESCRIBE +- DETACH +- DISTINCT +- DIVIDE +- DNODE +- DNODES +- DOT +- DOUBLE +- DROP + +### E + +- END +- EQ +- EXISTS +- EXPLAIN + +### F + +- FAIL +- FILE +- FILL +- FLOAT +- FOR +- FROM +- FSYNC + +### G + +- GE +- GLOB +- GRANTS +- GROUP +- GT + +### H + +- HAVING + +### I + +- ID +- IF +- IGNORE +- IMMEDIA +- IMPORT +- IN +- INITIAL +- INSERT +- INSTEAD +- INT +- INTEGER +- INTERVA +- INTO +- IS +- ISNULL + +### J + +- JOIN + +### K + +- KEEP +- KEY +- KILL + +### L + +- LE +- LIKE +- LIMIT +- LINEAR +- LOCAL +- LP +- LSHIFT +- LT + +### M + +- MATCH +- MAXROWS +- MINROWS +- MINUS +- MNODES +- MODIFY +- MODULES + +### N + +- NE +- NONE +- NOT +- NOTNULL +- NOW +- NULL + +### O + +- OF +- OFFSET +- OR +- ORDER + +### P + +- PARTITION +- PASS +- PLUS +- PPS +- PRECISION +- PREV +- PRIVILEGE + +### Q + +- QTIME +- QUERIE +- QUERY +- QUORUM + +### R + +- RAISE +- REM +- REPLACE +- REPLICA +- RESET +- RESTRIC +- ROW +- RP +- RSHIFT + +### S + +- SCORES +- SELECT +- SEMI +- SESSION +- SET +- SHOW +- SLASH +- SLIDING +- SLIMIT +- SMALLIN +- SOFFSET +- STable +- STableS +- STAR +- STATE +- STATEMEN +- STATE_WI +- STORAGE +- STREAM +- STREAMS +- STRING +- SYNCDB + +### T + +- TABLE +- TABLES +- TAG +- TAGS +- TBNAME +- TIMES +- TIMESTAMP +- TINYINT +- TOPIC +- TOPICS +- TRIGGER +- TSERIES + +### U + +- UMINUS +- UNION +- UNSIGNED +- UPDATE +- UPLUS +- USE +- USER +- USERS +- USING + +### V + +- VALUES +- VARIABLE +- VARIABLES +- VGROUPS +- VIEW +- VNODES + +### W + +- WAL +- WHERE + +### _ + +- _C0 +- _QSTART +- _QSTOP +- _QDURATION +- _WSTART +- _WSTOP +- _WDURATION ## Explanations ### TBNAME diff --git a/docs/zh/12-taos-sql/12-keywords.md b/docs/zh/12-taos-sql/12-keywords.md index 5c68e5da7e..202f223b45 100644 --- a/docs/zh/12-taos-sql/12-keywords.md +++ b/docs/zh/12-taos-sql/12-keywords.md @@ -45,48 +45,274 @@ title: TDengine 参数限制与保留关键字 目前 TDengine 有将近 200 个内部保留关键字，这些关键字无论大小写均不可以用作库名、表名、STable 名、数据列名及标签列名等。这些关键字列表如下： -| 关键字列表 | | | | | -| ----------- | ---------- | --------- | ---------- | ------------ | -| ABORT | CREATE | IGNORE | NULL | STAR | -| ACCOUNT | CTIME | IMMEDIATE | OF | STATE | -| ACCOUNTS | DATABASE | IMPORT | OFFSET | STATEMENT | -| ADD | DATABASES | IN | OR | STATE_WINDOW | -| AFTER | DAYS | INITIALLY | ORDER | STORAGE | -| ALL | DBS | INSERT | PARTITIONS | STREAM | -| ALTER | DEFERRED | INSTEAD | PASS | STREAMS | -| AND | DELIMITERS | INT | PLUS | STRING | -| AS | DESC | INTEGER | PPS | SYNCDB | -| ASC | DESCRIBE | INTERVAL | PRECISION | TABLE | -| ATTACH | DETACH | INTO | PREV | TABLES | -| BEFORE | DISTINCT | IS | PRIVILEGE | TAG | -| BEGIN | DIVIDE | ISNULL | QTIME | TAGS | -| BETWEEN | DNODE | JOIN | QUERIES | TBNAME | -| BIGINT | DNODES | KEEP | QUERY | TIMES | -| BINARY | DOT | KEY | QUORUM | TIMESTAMP | -| BITAND | DOUBLE | KILL | RAISE | TINYINT | -| BITNOT | DROP | LE | REM | TOPIC | -| BITOR | EACH | LIKE | REPLACE | TOPICS | -| BLOCKS | END | LIMIT | REPLICA | TRIGGER | -| BOOL | EQ | LINEAR | RESET | TSERIES | -| BY | EXISTS | LOCAL | RESTRICT | UMINUS | -| CACHE | EXPLAIN | LP | ROW | UNION | -| CACHELAST | FAIL | LSHIFT | RP | UNSIGNED | -| CASCADE | FILE | LT | RSHIFT | UPDATE | -| CHANGE | FILL | MATCH | SCORES | UPLUS | -| CLUSTER | FLOAT | MAXROWS | SELECT | USE | -| COLON | FOR | MINROWS | SEMI | USER | -| COLUMN | FROM | MINUS | SESSION | USERS | -| COMMA | FSYNC | MNODES | SET | USING | -| COMP | GE | MODIFY | SHOW | VALUES | -| COMPACT | GLOB | MODULES | SLASH | VARIABLE | -| CONCAT | GRANTS | NCHAR | SLIDING | VARIABLES | -| CONFLICT | GROUP | NE | SLIMIT | VGROUPS | -| CONNECTION | GT | NONE | SMALLINT | VIEW | -| CONNECTIONS | HAVING | NOT | SOFFSET | VNODES | -| CONNS | ID | NOTNULL | STABLE | WAL | -| COPY | IF | NOW | STABLES | WHERE | -| _C0 | _QSTART | _QSTOP | _QDURATION | _WSTART | -| _WSTOP | _WDURATION | _ROWTS | +### A + +- ABORT +- ACCOUNT +- ACCOUNTS +- ADD +- AFTER +- ALL +- ALTER +- AND +- AS +- ASC +- ATTACH + +### B + +- BEFORE +- BEGIN +- BETWEEN +- BIGINT +- BINARY +- BITAND +- BITNOT +- BITOR +- BLOCKS +- BOOL +- BY + +### C + +- CACHE +- CACHELAST +- CASCADE +- CHANGE +- CLUSTER +- COLON +- COLUMN +- COMMA +- COMP +- COMPACT +- CONCAT +- CONFLICT +- CONNECTION +- CONNECTIONS +- CONNS +- COPY +- CREATE +- CTIME + +### D + +- DATABASE +- DATABASES +- DAYS +- DBS +- DEFERRED +- DELETE +- DELIMITERS +- DESC +- DESCRIBE +- DETACH +- DISTINCT +- DIVIDE +- DNODE +- DNODES +- DOT +- DOUBLE +- DROP + +### E + +- END +- EQ +- EXISTS +- EXPLAIN + +### F + +- FAIL +- FILE +- FILL +- FLOAT +- FOR +- FROM +- FSYNC + +### G + +- GE +- GLOB +- GRANTS +- GROUP +- GT + +### H + +- HAVING + +### I + +- ID +- IF +- IGNORE +- IMMEDIA +- IMPORT +- IN +- INITIAL +- INSERT +- INSTEAD +- INT +- INTEGER +- INTERVA +- INTO +- IS +- ISNULL + +### J + +- JOIN + +### K + +- KEEP +- KEY +- KILL + +### L + +- LE +- LIKE +- LIMIT +- LINEAR +- LOCAL +- LP +- LSHIFT +- LT + +### M + +- MATCH +- MAXROWS +- MINROWS +- MINUS +- MNODES +- MODIFY +- MODULES + +### N + +- NE +- NONE +- NOT +- NOTNULL +- NOW +- NULL + +### O + +- OF +- OFFSET +- OR +- ORDER + +### P + +- PARTITION +- PASS +- PLUS +- PPS +- PRECISION +- PREV +- PRIVILEGE + +### Q + +- QTIME +- QUERIE +- QUERY +- QUORUM + +### R + +- RAISE +- REM +- REPLACE +- REPLICA +- RESET +- RESTRIC +- ROW +- RP +- RSHIFT + +### S + +- SCORES +- SELECT +- SEMI +- SESSION +- SET +- SHOW +- SLASH +- SLIDING +- SLIMIT +- SMALLIN +- SOFFSET +- STable +- STableS +- STAR +- STATE +- STATEMEN +- STATE_WI +- STORAGE +- STREAM +- STREAMS +- STRING +- SYNCDB + +### T + +- TABLE +- TABLES +- TAG +- TAGS +- TBNAME +- TIMES +- TIMESTAMP +- TINYINT +- TOPIC +- TOPICS +- TRIGGER +- TSERIES + +### U + +- UMINUS +- UNION +- UNSIGNED +- UPDATE +- UPLUS +- USE +- USER +- USERS +- USING + +### V + +- VALUES +- VARIABLE +- VARIABLES +- VGROUPS +- VIEW +- VNODES + +### W + +- WAL +- WHERE + +### _ + +- _C0 +- _QSTART +- _QSTOP +- _QDURATION +- _WSTART +- _WSTOP +- _WDURATION + ## 特殊说明 ### TBNAME From fe6d46d2c0a94d236897aeb1977e81e82cc66047 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 14 Jun 2022 12:41:33 +0800 Subject: [PATCH 42/78] test: disable mnode basic3 --- tests/script/jenkins/basic.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index db8a055362..1c694bf7fb 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -58,7 +58,7 @@ # ---- mnode ./test.sh -f tsim/mnode/basic1.sim ./test.sh -f tsim/mnode/basic2.sim -./test.sh -f tsim/mnode/basic3.sim +#./test.sh -f tsim/mnode/basic3.sim ./test.sh -f tsim/mnode/basic4.sim ./test.sh -f tsim/mnode/basic5.sim From 0b58c2b850a8e5896a6532763978d0e441bf2956 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Tue, 14 Jun 2022 13:03:12 +0800 Subject: [PATCH 43/78] test: modify sleep length --- tests/system-test/7-tmq/subscribeDb1.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/system-test/7-tmq/subscribeDb1.py b/tests/system-test/7-tmq/subscribeDb1.py index 44c24eb616..ed92a429ae 100644 --- a/tests/system-test/7-tmq/subscribeDb1.py +++ b/tests/system-test/7-tmq/subscribeDb1.py @@ -437,12 +437,12 @@ class TDTestCase: event.wait() tdLog.info("start consume processor") - pollDelay = 100 + pollDelay = 20 showMsg = 1 showRow = 1 self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) - time.sleep(5) + time.sleep(3) tdLog.info("pkill consume processor") if (platform.system().lower() == 'windows'): os.system("TASKKILL /F /IM tmq_sim.exe") @@ -465,7 +465,7 @@ class TDTestCase: totalConsumeRows += resultList[i] if totalConsumeRows >= expectrowcnt or totalConsumeRows <= 0: - tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt)) + tdLog.info("act consume rows: %d, expect consume rows between %d and 0"%(totalConsumeRows, expectrowcnt)) tdLog.exit("tmq consume rows error!") time.sleep(15) From 979420801eb0acda4cb3449b82a6332eadf9c87e Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Tue, 14 Jun 2022 13:07:27 +0800 Subject: [PATCH 44/78] fix(sync) if FpEqMsg return error, do not enqueue msg --- source/libs/sync/src/syncMain.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index f3faa60d34..7934087ff3 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -1658,7 +1658,13 @@ static void syncNodeEqHeartbeatTimer(void* param, void* tmrId) { syncTimeout2RpcMsg(pSyncMsg, &rpcMsg); syncRpcMsgLog2((char*)"==syncNodeEqHeartbeatTimer==", &rpcMsg); if (pSyncNode->FpEqMsg != NULL) { - pSyncNode->FpEqMsg(pSyncNode->msgcb, &rpcMsg); + int32_t code = pSyncNode->FpEqMsg(pSyncNode->msgcb, &rpcMsg); + if (code != 0) { + sError("vgId:%d sync enqueue timer msg error, code:%d", pSyncNode->vgId, code); + rpcFreeCont(rpcMsg.pCont); + return; + } + } else { sTrace("syncNodeEqHeartbeatTimer pSyncNode->FpEqMsg is NULL"); } From a2039d8bb2dda45763804be5a267ebb1cd0d2693 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Tue, 14 Jun 2022 13:16:49 +0800 Subject: [PATCH 45/78] fix(sync) if FpEqMsg return error, do not enqueue msg --- source/libs/sync/src/syncMain.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 7934087ff3..e7f11b9a7d 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -1670,8 +1670,12 @@ static void syncNodeEqHeartbeatTimer(void* param, void* tmrId) { } syncTimeoutDestroy(pSyncMsg); + if (gSyncEnv != NULL) { taosTmrReset(syncNodeEqHeartbeatTimer, pSyncNode->heartbeatTimerMS, pSyncNode, gSyncEnv->pTimerManager, &pSyncNode->pHeartbeatTimer); + } else { + sError("sync env is already stop"); + } } else { sTrace("==syncNodeEqHeartbeatTimer== heartbeatTimerLogicClock:%" PRIu64 ", heartbeatTimerLogicClockUser:%" PRIu64 "", From 110bf979044f8aa012ad57a93bfbd07049c23506 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 14 Jun 2022 13:22:24 +0800 Subject: [PATCH 46/78] fix: mnode not start --- source/dnode/mgmt/node_mgmt/src/dmEnv.c | 2 +- tests/script/jenkins/basic.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/mgmt/node_mgmt/src/dmEnv.c b/source/dnode/mgmt/node_mgmt/src/dmEnv.c index 498af36786..119d521827 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmEnv.c +++ b/source/dnode/mgmt/node_mgmt/src/dmEnv.c @@ -124,7 +124,7 @@ static int32_t dmProcessCreateNodeReq(EDndNodeType ntype, SRpcMsg *pMsg) { } else { dInfo("node:%s, has been created", pWrapper->name); code = dmOpenNode(pWrapper); - if (code != 0) { + if (code == 0) { code = dmStartNode(pWrapper); } pWrapper->deployed = true; diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 1c694bf7fb..db8a055362 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -58,7 +58,7 @@ # ---- mnode ./test.sh -f tsim/mnode/basic1.sim ./test.sh -f tsim/mnode/basic2.sim -#./test.sh -f tsim/mnode/basic3.sim +./test.sh -f tsim/mnode/basic3.sim ./test.sh -f tsim/mnode/basic4.sim ./test.sh -f tsim/mnode/basic5.sim From 545345de7b7e67dce5c3135dfc5424619dd334c1 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 14 Jun 2022 13:23:47 +0800 Subject: [PATCH 47/78] remove debug flag --- source/client/inc/clientInt.h | 2 +- source/libs/catalog/src/ctgDbg.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index d72aa39e4c..6aa83e9575 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -45,7 +45,7 @@ extern "C" { #define ERROR_MSG_BUF_DEFAULT_SIZE 512 #define HEARTBEAT_INTERVAL 1500 // ms -#define SYNC_ON_TOP_OF_ASYNC 0 +#define SYNC_ON_TOP_OF_ASYNC 1 enum { RES_TYPE__QUERY = 1, diff --git a/source/libs/catalog/src/ctgDbg.c b/source/libs/catalog/src/ctgDbg.c index cd61abf6cc..9de1ea22be 100644 --- a/source/libs/catalog/src/ctgDbg.c +++ b/source/libs/catalog/src/ctgDbg.c @@ -19,7 +19,7 @@ #include "catalogInt.h" extern SCatalogMgmt gCtgMgmt; -SCtgDebug gCTGDebug = {.apiEnable = true, .lockEnable = true}; +SCtgDebug gCTGDebug = {0}; void ctgdUserCallback(SMetaData* pResult, void* param, int32_t code) { ASSERT(*(int32_t*)param == 1); From cb85ee6d3c944148f0616757dbf304e95098eff0 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 14 Jun 2022 13:36:54 +0800 Subject: [PATCH 48/78] opt: optimize generating groupid in partition/group by tag --- source/libs/executor/inc/executorimpl.h | 2 +- source/libs/executor/src/executorimpl.c | 24 ++++++++---------------- source/libs/executor/src/scanoperator.c | 25 +++++++++++++++++++++++-- 3 files changed, 32 insertions(+), 19 deletions(-) diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 3e24e43233..ab60acab53 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -821,7 +821,7 @@ SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pEx SExprInfo* pScalarExprInfo, int32_t numOfScalarExpr, SExecTaskInfo* pTaskInfo); SOperatorInfo* createDataBlockInfoScanOperator(void* dataReader, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createStreamScanOperatorInfo(void* pDataReader, SReadHandle* pHandle, SArray* pTableIdList, +SOperatorInfo* createStreamScanOperatorInfo(void* pDataReader, SReadHandle* pHandle, STableScanPhysiNode* pTableScanNode, SExecTaskInfo* pTaskInfo, STimeWindowAggSupp* pTwSup); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 13ac74199e..e4c072a396 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -4558,7 +4558,6 @@ static tsdbReaderT doCreateDataReader(STableScanPhysiNode* pTableScanNode, SRead static int32_t getTableList(void* metaHandle, int32_t tableType, uint64_t tableUid, STableListInfo* pListInfo, SNode* pTagCond); -static SArray* extractTableIdList(const STableListInfo* pTableGroupInfo); static SArray* extractColumnInfo(SNodeList* pNodeList); static SArray* createSortInfo(SNodeList* pNodeList); @@ -4725,12 +4724,17 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo } else { qDebug("%s pDataReader is not NULL", GET_TASKID(pTaskInfo)); } - SArray* tableIdList = extractTableIdList(pTableListInfo); + + SArray* groupKyes = extractPartitionColInfo(pTableScanNode->pPartitionKeys); + int32_t code = generateGroupIdMap(pTableListInfo, pHandle, groupKyes); //todo for json + if (code){ + tsdbCleanupReadHandle(pDataReader); + return NULL; + } SOperatorInfo* pOperator = - createStreamScanOperatorInfo(pDataReader, pHandle, tableIdList, pTableScanNode, pTaskInfo, &twSup); + createStreamScanOperatorInfo(pDataReader, pHandle, pTableScanNode, pTaskInfo, &twSup); - taosArrayDestroy(tableIdList); return pOperator; } else if (QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN == type) { SSystemTableScanPhysiNode* pSysScanPhyNode = (SSystemTableScanPhysiNode*)pPhyNode; @@ -5183,18 +5187,6 @@ int32_t getTableList(void* metaHandle, int32_t tableType, uint64_t tableUid, STa return code; } -SArray* extractTableIdList(const STableListInfo* pTableGroupInfo) { - SArray* tableIdList = taosArrayInit(4, sizeof(uint64_t)); - - // Transfer the Array of STableKeyInfo into uid list. - for (int32_t i = 0; i < taosArrayGetSize(pTableGroupInfo->pTableList); ++i) { - STableKeyInfo* pkeyInfo = taosArrayGet(pTableGroupInfo->pTableList, i); - taosArrayPush(tableIdList, &pkeyInfo->uid); - } - - return tableIdList; -} - tsdbReaderT doCreateDataReader(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle, STableListInfo* pTableListInfo, uint64_t queryId, uint64_t taskId, SNode* pTagCond) { int32_t code = diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index b6ee9843b4..d30e4ef6db 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -886,6 +886,11 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { pInfo->pRes->info.groupId = groupId; } + uint64_t* groupIdPre = taosHashGet(pOperator->pTaskInfo->tableqinfoList.map, &uid, sizeof(int64_t)); + if (groupIdPre) { + pInfo->pRes->info.groupId = *groupIdPre; + } + for (int32_t i = 0; i < taosArrayGetSize(pInfo->pColMatchInfo); ++i) { SColMatchInfo* pColMatchInfo = taosArrayGet(pInfo->pColMatchInfo, i); if (!pColMatchInfo->output) { @@ -953,11 +958,24 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { } } -SOperatorInfo* createStreamScanOperatorInfo(void* pDataReader, SReadHandle* pHandle, SArray* pTableIdList, +static SArray* extractTableIdList(const STableListInfo* pTableGroupInfo) { + SArray* tableIdList = taosArrayInit(4, sizeof(uint64_t)); + + // Transfer the Array of STableKeyInfo into uid list. + for (int32_t i = 0; i < taosArrayGetSize(pTableGroupInfo->pTableList); ++i) { + STableKeyInfo* pkeyInfo = taosArrayGet(pTableGroupInfo->pTableList, i); + taosArrayPush(tableIdList, &pkeyInfo->uid); + } + + return tableIdList; +} + +SOperatorInfo* createStreamScanOperatorInfo(void* pDataReader, SReadHandle* pHandle, STableScanPhysiNode* pTableScanNode, SExecTaskInfo* pTaskInfo, STimeWindowAggSupp* pTwSup) { SStreamBlockScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamBlockScanInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + if (pInfo == NULL || pOperator == NULL) { terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; goto _error; @@ -988,10 +1006,13 @@ SOperatorInfo* createStreamScanOperatorInfo(void* pDataReader, SReadHandle* pHan // set the extract column id to streamHandle tqReadHandleSetColIdList((STqReadHandle*)pHandle->reader, pColIds); - int32_t code = tqReadHandleSetTbUidList(pHandle->reader, pTableIdList); + SArray* tableIdList = extractTableIdList(&pTaskInfo->tableqinfoList); + int32_t code = tqReadHandleSetTbUidList(pHandle->reader, tableIdList); if (code != 0) { + taosArrayDestroy(tableIdList); goto _error; } + taosArrayDestroy(tableIdList); pInfo->pBlockLists = taosArrayInit(4, POINTER_BYTES); if (pInfo->pBlockLists == NULL) { From 0237c68e272da34a38439b5c2707a69e2ece361a Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 14 Jun 2022 13:44:13 +0800 Subject: [PATCH 49/78] fix: invalid read/write --- source/dnode/vnode/src/meta/metaQuery.c | 11 ++++ tests/script/tsim/stable/tag_filter.sim | 75 +++++++++++++++++++++++++ 2 files changed, 86 insertions(+) diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index 1d8082084f..a6339125c4 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -663,12 +663,23 @@ int32_t metaFilteTableIds(SMeta *pMeta, SMetaFltParam *param, SArray *pUids) { void * entryKey = NULL, *entryVal = NULL; int32_t nEntryKey, nEntryVal; + bool first = true; while (1) { valid = tdbTbcGet(pCursor->pCur, (const void **)&entryKey, &nEntryKey, (const void **)&entryVal, &nEntryVal); if (valid < 0) { break; } STagIdxKey *p = entryKey; + if (p->type != pCursor->type) { + if (first) { + valid = param->reverse ? tdbTbcMoveToPrev(pCursor->pCur) : tdbTbcMoveToNext(pCursor->pCur); + if (valid < 0) break; + continue; + } else { + break; + } + } + first = false; if (p != NULL) { int32_t cmp = (*param->filterFunc)(p->data, pKey->data, pKey->type); if (cmp == 0) { diff --git a/tests/script/tsim/stable/tag_filter.sim b/tests/script/tsim/stable/tag_filter.sim index 04b8f21de9..1f400eb803 100644 --- a/tests/script/tsim/stable/tag_filter.sim +++ b/tests/script/tsim/stable/tag_filter.sim @@ -73,5 +73,80 @@ if $rows != 6 then endi +print ========== prepare stbBin and ctbBin +sql create table db.stbBin (ts timestamp, c1 int, c2 binary(4)) tags(t1 binary(16)) + +sql create table db.ctbBin using db.stbBin tags("a") +sql insert into db.ctbBin values(now, 1, "2") + +sql create table db.ctbBin1 using db.stbBin tags("b") +sql insert into db.ctbBin1 values(now, 2, "2") + +sql create table db.ctbBin2 using db.stbBin tags("c") +sql insert into db.ctbBin2 values(now, 3, "2") + +sql create table db.ctbBin3 using db.stbBin tags("d") +sql insert into db.ctbBin3 values(now, 4, "2") + +sql select * from db.stbBin where t1 = "a" +if $rows != 1 then + return -1 +endi + + +sql select * from db.stbBin where t1 < "a" +if $rows != 0 then + return -=1 +endi + +sql select * from db.stbBin where t1 < "b" +if $rows != 1 then + return -1 +endi + + +sql select * from db.stbBin where t1 between "a" and "e" +if $rows != 4 then + return -1 +endi + + +print ========== prepare stbNc and ctbNc +sql create table db.stbNc (ts timestamp, c1 int, c2 binary(4)) tags(t1 nchar(16)) + + +sql create table db.ctbNc using db.stbNc tags("a") +sql insert into db.ctbNc values(now, 1, "2") + +sql create table db.ctbNc1 using db.stbNc tags("b") +sql insert into db.ctbNc1 values(now, 2, "2") + +sql create table db.ctbNc2 using db.stbNc tags("c") +sql insert into db.ctbNc2 values(now, 3, "2") + +sql create table db.ctbNc3 using db.stbNc tags("d") +sql insert into db.ctbNc3 values(now, 4, "2") + +sql select * from db.stbNc where t1 = "a" +if $rows != 1 then + return -1 +endi + + +sql select * from db.stbNc where t1 < "a" +if $rows != 0 then + return -=1 +endi + +sql select * from db.stbNc where t1 < "b" +if $rows != 1 then + return -1 +endi + + +sql select * from db.stbNc where t1 between "a" and "e" +if $rows != 4 then + return -1 +endi system sh/exec.sh -n dnode1 -s stop -x SIGINT From 2a6dfbb0d7185d53eb1519ce9e66d583346d775c Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 14 Jun 2022 13:48:33 +0800 Subject: [PATCH 50/78] enh(query): avg function use pre-agg results --- source/libs/function/src/builtinsimpl.c | 154 +++++++++++++----------- 1 file changed, 83 insertions(+), 71 deletions(-) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index b5009b7d41..3510a59254 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -642,8 +642,8 @@ bool avgFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo) { int32_t avgFunction(SqlFunctionCtx* pCtx) { int32_t numOfElem = 0; - // Only the pre-computing information loaded and actual data does not loaded SInputColumnInfoData* pInput = &pCtx->input; + SColumnDataAgg* pAgg = pInput->pColumnDataAgg[0]; int32_t type = pInput->pData[0]->info.type; SAvgRes* pAvgRes = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); @@ -660,95 +660,107 @@ int32_t avgFunction(SqlFunctionCtx* pCtx) { goto _avg_over; } - switch (type) { - case TSDB_DATA_TYPE_TINYINT: { - int8_t* plist = (int8_t*)pCol->pData; - for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) { - if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) { - continue; + if (pInput->colDataAggIsSet) { + numOfElem = numOfRows - pAgg->numOfNull; + ASSERT(numOfElem >= 0); + + pAvgRes->count += numOfElem; + if (IS_INTEGER_TYPE(type)) { + pAvgRes->sum.isum += pAgg->sum; + } else if (IS_FLOAT_TYPE(type)) { + pAvgRes->sum.dsum += GET_DOUBLE_VAL((const char*)&(pAgg->sum)); + } + } else { // computing based on the true data block + switch (type) { + case TSDB_DATA_TYPE_TINYINT: { + int8_t* plist = (int8_t*)pCol->pData; + for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) { + if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) { + continue; + } + + numOfElem += 1; + pAvgRes->count += 1; + pAvgRes->sum.isum += plist[i]; } - numOfElem += 1; - pAvgRes->count += 1; - pAvgRes->sum.isum += plist[i]; + break; } - break; - } + case TSDB_DATA_TYPE_SMALLINT: { + int16_t* plist = (int16_t*)pCol->pData; + for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) { + if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) { + continue; + } - case TSDB_DATA_TYPE_SMALLINT: { - int16_t* plist = (int16_t*)pCol->pData; - for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) { - if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) { - continue; + numOfElem += 1; + pAvgRes->count += 1; + pAvgRes->sum.isum += plist[i]; + } + break; + } + + case TSDB_DATA_TYPE_INT: { + int32_t* plist = (int32_t*)pCol->pData; + for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) { + if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) { + continue; + } + + numOfElem += 1; + pAvgRes->count += 1; + pAvgRes->sum.isum += plist[i]; } - numOfElem += 1; - pAvgRes->count += 1; - pAvgRes->sum.isum += plist[i]; + break; } - break; - } - case TSDB_DATA_TYPE_INT: { - int32_t* plist = (int32_t*)pCol->pData; - for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) { - if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) { - continue; + case TSDB_DATA_TYPE_BIGINT: { + int64_t* plist = (int64_t*)pCol->pData; + for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) { + if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) { + continue; + } + + numOfElem += 1; + pAvgRes->count += 1; + pAvgRes->sum.isum += plist[i]; } - - numOfElem += 1; - pAvgRes->count += 1; - pAvgRes->sum.isum += plist[i]; + break; } - break; - } + case TSDB_DATA_TYPE_FLOAT: { + float* plist = (float*)pCol->pData; + for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) { + if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) { + continue; + } - case TSDB_DATA_TYPE_BIGINT: { - int64_t* plist = (int64_t*)pCol->pData; - for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) { - if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) { - continue; + numOfElem += 1; + pAvgRes->count += 1; + pAvgRes->sum.dsum += plist[i]; } - - numOfElem += 1; - pAvgRes->count += 1; - pAvgRes->sum.isum += plist[i]; + break; } - break; - } - case TSDB_DATA_TYPE_FLOAT: { - float* plist = (float*)pCol->pData; - for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) { - if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) { - continue; + case TSDB_DATA_TYPE_DOUBLE: { + double* plist = (double*)pCol->pData; + for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) { + if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) { + continue; + } + + numOfElem += 1; + pAvgRes->count += 1; + pAvgRes->sum.dsum += plist[i]; } - - numOfElem += 1; - pAvgRes->count += 1; - pAvgRes->sum.dsum += plist[i]; + break; } - break; + + default: + break; } - - case TSDB_DATA_TYPE_DOUBLE: { - double* plist = (double*)pCol->pData; - for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) { - if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) { - continue; - } - - numOfElem += 1; - pAvgRes->count += 1; - pAvgRes->sum.dsum += plist[i]; - } - break; - } - - default: - break; } _avg_over: From ad11eef979b82ad517e8154e9ba7e8006fff9f51 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 14 Jun 2022 13:52:07 +0800 Subject: [PATCH 51/78] enh: show db status such as creating and dropping --- source/dnode/mnode/impl/src/mndDb.c | 25 ++++++++++++++----------- source/dnode/mnode/impl/src/mndMnode.c | 2 +- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index b4237466d4..cfe363fdf0 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -1366,7 +1366,7 @@ char *buildRetension(SArray *pRetension) { } static void dumpDbInfoData(SSDataBlock *pBlock, SDbObj *pDb, SShowObj *pShow, int32_t rows, int64_t numOfTables, - bool sysDb) { + bool sysDb, ESdbStatus objStatus) { int32_t cols = 0; int32_t bytes = pShow->pMeta->pSchemas[cols].bytes; @@ -1379,6 +1379,8 @@ static void dumpDbInfoData(SSDataBlock *pBlock, SDbObj *pDb, SShowObj *pShow, in } char *status = "ready"; + if (objStatus == SDB_STATUS_CREATING) status = "creating"; + if (objStatus == SDB_STATUS_DROPPING) status = "dropping"; char statusB[24] = {0}; STR_WITH_SIZE_TO_VARSTR(statusB, status, strlen(status)); @@ -1503,8 +1505,8 @@ static void dumpDbInfoData(SSDataBlock *pBlock, SDbObj *pDb, SShowObj *pShow, in pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, rows, (const char *)statusB, false); -// pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); -// colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.schemaless, false); + // pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + // colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.schemaless, false); char *p = buildRetension(pDb->cfg.pRetensions); @@ -1548,29 +1550,30 @@ static bool mndGetTablesOfDbFp(SMnode *pMnode, void *pObj, void *p1, void *p2, v } static int32_t mndRetrieveDbs(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rowsCapacity) { - SMnode *pMnode = pReq->info.node; - SSdb *pSdb = pMnode->pSdb; - int32_t numOfRows = 0; - SDbObj *pDb = NULL; + SMnode *pMnode = pReq->info.node; + SSdb *pSdb = pMnode->pSdb; + int32_t numOfRows = 0; + SDbObj *pDb = NULL; + ESdbStatus objStatus = 0; // Append the information_schema database into the result. if (!pShow->sysDbRsp) { SDbObj infoschemaDb = {0}; setInformationSchemaDbCfg(&infoschemaDb); - dumpDbInfoData(pBlock, &infoschemaDb, pShow, numOfRows, 14, true); + dumpDbInfoData(pBlock, &infoschemaDb, pShow, numOfRows, 14, true, 0); numOfRows += 1; SDbObj perfschemaDb = {0}; setPerfSchemaDbCfg(&perfschemaDb); - dumpDbInfoData(pBlock, &perfschemaDb, pShow, numOfRows, 3, true); + dumpDbInfoData(pBlock, &perfschemaDb, pShow, numOfRows, 3, true, 0); numOfRows += 1; pShow->sysDbRsp = true; } while (numOfRows < rowsCapacity) { - pShow->pIter = sdbFetch(pSdb, SDB_DB, pShow->pIter, (void **)&pDb); + pShow->pIter = sdbFetchAll(pSdb, SDB_DB, pShow->pIter, (void **)&pDb, &objStatus); if (pShow->pIter == NULL) { break; } @@ -1578,7 +1581,7 @@ static int32_t mndRetrieveDbs(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc int32_t numOfTables = 0; sdbTraverse(pSdb, SDB_VGROUP, mndGetTablesOfDbFp, &numOfTables, NULL, NULL); - dumpDbInfoData(pBlock, pDb, pShow, numOfRows, numOfTables, false); + dumpDbInfoData(pBlock, pDb, pShow, numOfRows, numOfTables, false, objStatus); numOfRows++; sdbRelease(pSdb, pDb); } diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index 95a2c3d935..f9749a969d 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -659,7 +659,7 @@ static int32_t mndRetrieveMnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB int32_t numOfRows = 0; int32_t cols = 0; SMnodeObj *pObj = NULL; - ESdbStatus objStatus; + ESdbStatus objStatus = 0; char *pWrite; int64_t curMs = taosGetTimestampMs(); From 8dea8dbbebeca2cf8b61d739d3cadbfe7b8c6b7f Mon Sep 17 00:00:00 2001 From: gccgdb1234 Date: Tue, 14 Jun 2022 13:40:48 +0800 Subject: [PATCH 52/78] docs: move delete data from developer guide to sql reference guide --- .../05-delete-data.mdx => 12-taos-sql/08-delete-data.mdx} | 0 docs/en/12-taos-sql/{07-function.md => 10-function.md} | 0 docs/en/12-taos-sql/{08-interval.md => 12-interval.md} | 0 docs/en/12-taos-sql/{09-limit.md => 14-limit.md} | 0 docs/en/12-taos-sql/{10-json.md => 16-json.md} | 0 docs/en/12-taos-sql/{11-escape.md => 18-escape.md} | 0 docs/en/12-taos-sql/{12-keywords.md => 20-keywords.md} | 0 .../05-delete-data.mdx => 12-taos-sql/08-delete-data.mdx} | 0 docs/zh/12-taos-sql/{07-function.md => 10-function.md} | 0 docs/zh/12-taos-sql/{08-interval.md => 12-interval.md} | 0 docs/zh/12-taos-sql/{09-limit.md => 14-limit.md} | 0 docs/zh/12-taos-sql/{10-json.md => 16-json.md} | 0 docs/zh/12-taos-sql/{11-escape.md => 18-escape.md} | 0 docs/zh/12-taos-sql/{12-keywords.md => 20-keywords.md} | 0 14 files changed, 0 insertions(+), 0 deletions(-) rename docs/en/{07-develop/05-delete-data.mdx => 12-taos-sql/08-delete-data.mdx} (100%) rename docs/en/12-taos-sql/{07-function.md => 10-function.md} (100%) rename docs/en/12-taos-sql/{08-interval.md => 12-interval.md} (100%) rename docs/en/12-taos-sql/{09-limit.md => 14-limit.md} (100%) rename docs/en/12-taos-sql/{10-json.md => 16-json.md} (100%) rename docs/en/12-taos-sql/{11-escape.md => 18-escape.md} (100%) rename docs/en/12-taos-sql/{12-keywords.md => 20-keywords.md} (100%) rename docs/zh/{07-develop/05-delete-data.mdx => 12-taos-sql/08-delete-data.mdx} (100%) rename docs/zh/12-taos-sql/{07-function.md => 10-function.md} (100%) rename docs/zh/12-taos-sql/{08-interval.md => 12-interval.md} (100%) rename docs/zh/12-taos-sql/{09-limit.md => 14-limit.md} (100%) rename docs/zh/12-taos-sql/{10-json.md => 16-json.md} (100%) rename docs/zh/12-taos-sql/{11-escape.md => 18-escape.md} (100%) rename docs/zh/12-taos-sql/{12-keywords.md => 20-keywords.md} (100%) diff --git a/docs/en/07-develop/05-delete-data.mdx b/docs/en/12-taos-sql/08-delete-data.mdx similarity index 100% rename from docs/en/07-develop/05-delete-data.mdx rename to docs/en/12-taos-sql/08-delete-data.mdx diff --git a/docs/en/12-taos-sql/07-function.md b/docs/en/12-taos-sql/10-function.md similarity index 100% rename from docs/en/12-taos-sql/07-function.md rename to docs/en/12-taos-sql/10-function.md diff --git a/docs/en/12-taos-sql/08-interval.md b/docs/en/12-taos-sql/12-interval.md similarity index 100% rename from docs/en/12-taos-sql/08-interval.md rename to docs/en/12-taos-sql/12-interval.md diff --git a/docs/en/12-taos-sql/09-limit.md b/docs/en/12-taos-sql/14-limit.md similarity index 100% rename from docs/en/12-taos-sql/09-limit.md rename to docs/en/12-taos-sql/14-limit.md diff --git a/docs/en/12-taos-sql/10-json.md b/docs/en/12-taos-sql/16-json.md similarity index 100% rename from docs/en/12-taos-sql/10-json.md rename to docs/en/12-taos-sql/16-json.md diff --git a/docs/en/12-taos-sql/11-escape.md b/docs/en/12-taos-sql/18-escape.md similarity index 100% rename from docs/en/12-taos-sql/11-escape.md rename to docs/en/12-taos-sql/18-escape.md diff --git a/docs/en/12-taos-sql/12-keywords.md b/docs/en/12-taos-sql/20-keywords.md similarity index 100% rename from docs/en/12-taos-sql/12-keywords.md rename to docs/en/12-taos-sql/20-keywords.md diff --git a/docs/zh/07-develop/05-delete-data.mdx b/docs/zh/12-taos-sql/08-delete-data.mdx similarity index 100% rename from docs/zh/07-develop/05-delete-data.mdx rename to docs/zh/12-taos-sql/08-delete-data.mdx diff --git a/docs/zh/12-taos-sql/07-function.md b/docs/zh/12-taos-sql/10-function.md similarity index 100% rename from docs/zh/12-taos-sql/07-function.md rename to docs/zh/12-taos-sql/10-function.md diff --git a/docs/zh/12-taos-sql/08-interval.md b/docs/zh/12-taos-sql/12-interval.md similarity index 100% rename from docs/zh/12-taos-sql/08-interval.md rename to docs/zh/12-taos-sql/12-interval.md diff --git a/docs/zh/12-taos-sql/09-limit.md b/docs/zh/12-taos-sql/14-limit.md similarity index 100% rename from docs/zh/12-taos-sql/09-limit.md rename to docs/zh/12-taos-sql/14-limit.md diff --git a/docs/zh/12-taos-sql/10-json.md b/docs/zh/12-taos-sql/16-json.md similarity index 100% rename from docs/zh/12-taos-sql/10-json.md rename to docs/zh/12-taos-sql/16-json.md diff --git a/docs/zh/12-taos-sql/11-escape.md b/docs/zh/12-taos-sql/18-escape.md similarity index 100% rename from docs/zh/12-taos-sql/11-escape.md rename to docs/zh/12-taos-sql/18-escape.md diff --git a/docs/zh/12-taos-sql/12-keywords.md b/docs/zh/12-taos-sql/20-keywords.md similarity index 100% rename from docs/zh/12-taos-sql/12-keywords.md rename to docs/zh/12-taos-sql/20-keywords.md From 601e454a242bd511f2e2914f3f946490715cb45f Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Tue, 14 Jun 2022 13:58:40 +0800 Subject: [PATCH 53/78] enh(tmq): put offset store into vnode --- include/common/tmsg.h | 29 +++- include/common/tmsgdef.h | 3 +- source/client/src/clientImpl.c | 54 +++--- source/client/src/tmq.c | 172 +++++++++++++++++++- source/common/src/tdatablock.c | 4 +- source/common/src/tmsg.c | 29 ++++ source/dnode/mgmt/mgmt_vnode/src/vmHandle.c | 1 + source/dnode/vnode/src/inc/tq.h | 36 ++-- source/dnode/vnode/src/inc/vnodeInt.h | 1 + source/dnode/vnode/src/tq/tq.c | 35 +++- source/dnode/vnode/src/tq/tqOffset.c | 115 +++++++++++-- source/dnode/vnode/src/vnd/vnodeSvr.c | 15 +- 12 files changed, 415 insertions(+), 79 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index d69849349c..be9eda3825 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1494,9 +1494,9 @@ typedef struct { int32_t code; } STaskDropRsp; -#define STREAM_TRIGGER_AT_ONCE 1 -#define STREAM_TRIGGER_WINDOW_CLOSE 2 -#define STREAM_TRIGGER_MAX_DELAY 3 +#define STREAM_TRIGGER_AT_ONCE 1 +#define STREAM_TRIGGER_WINDOW_CLOSE 2 +#define STREAM_TRIGGER_MAX_DELAY 3 typedef struct { char name[TSDB_TABLE_FNAME_LEN]; @@ -2297,6 +2297,29 @@ int32_t tDecodeSMqOffset(SDecoder* decoder, SMqOffset* pOffset); int32_t tEncodeSMqCMCommitOffsetReq(SEncoder* encoder, const SMqCMCommitOffsetReq* pReq); int32_t tDecodeSMqCMCommitOffsetReq(SDecoder* decoder, SMqCMCommitOffsetReq* pReq); +// tqOffset +enum { + TMQ_OFFSET__SNAPSHOT = 1, + TMQ_OFFSET__LOG, +}; + +typedef struct { + int8_t type; + union { + struct { + int64_t uid; + int64_t ts; + }; + struct { + int64_t version; + }; + }; + char subKey[TSDB_SUBSCRIBE_KEY_LEN]; +} STqOffset; + +int32_t tEncodeSTqOffset(SEncoder* pEncoder, const STqOffset* pOffset); +int32_t tDecodeSTqOffset(SDecoder* pDecoder, STqOffset* pOffset); + typedef struct { char name[TSDB_TABLE_FNAME_LEN]; char stb[TSDB_TABLE_FNAME_LEN]; diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 743f10bd55..534194e1cf 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -140,7 +140,7 @@ enum { TD_DEF_MSG_TYPE(TDMT_MND_MQ_CONSUMER_RECOVER, "consumer-recover", SMqConsumerRecoverMsg, NULL) TD_DEF_MSG_TYPE(TDMT_MND_MQ_DO_REBALANCE, "do-rebalance", SMqDoRebalanceMsg, NULL) TD_DEF_MSG_TYPE(TDMT_MND_MQ_DROP_CGROUP, "drop-cgroup", SMqDropCGroupReq, SMqDropCGroupRsp) - TD_DEF_MSG_TYPE(TDMT_MND_MQ_COMMIT_OFFSET, "commit-offset", SMqCMCommitOffsetReq, SMqCMCommitOffsetRsp) + TD_DEF_MSG_TYPE(TDMT_MND_MQ_COMMIT_OFFSET, "mnode-commit-offset", SMqCMCommitOffsetReq, SMqCMCommitOffsetRsp) TD_DEF_MSG_TYPE(TDMT_MND_MQ_TIMER, "mq-tmr", SMTimerReq, NULL) TD_DEF_MSG_TYPE(TDMT_MND_TELEM_TIMER, "telem-tmr", SMTimerReq, SMTimerReq) TD_DEF_MSG_TYPE(TDMT_MND_TRANS_TIMER, "trans-tmr", NULL, NULL) @@ -176,6 +176,7 @@ enum { TD_DEF_MSG_TYPE(TDMT_VND_DROP_STB, "vnode-drop-stb", SVDropStbReq, NULL) TD_DEF_MSG_TYPE(TDMT_VND_MQ_VG_CHANGE, "vnode-mq-vg-change", SMqRebVgReq, SMqRebVgRsp) TD_DEF_MSG_TYPE(TDMT_VND_MQ_VG_DELETE, "vnode-mq-vg-delete", SMqVDeleteReq, SMqVDeleteRsp) + TD_DEF_MSG_TYPE(TDMT_VND_MQ_COMMIT_OFFSET, "vnode-commit-offset", STqOffset, STqOffset) TD_DEF_MSG_TYPE(TDMT_VND_CANCEL_TASK, "vnode-cancel-task", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_DROP_TASK, "vnode-drop-task", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_CREATE_TOPIC, "vnode-create-topic", NULL, NULL) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index dc83f7bc43..f48383c43a 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -205,7 +205,7 @@ int32_t execLocalCmd(SRequestObj* pRequest, SQuery* pQuery) { SRetrieveTableRsp* pRsp = NULL; int32_t code = qExecCommand(pQuery->pRoot, &pRsp); if (TSDB_CODE_SUCCESS == code && NULL != pRsp) { - code = setQueryResultFromRsp(&pRequest->body.resInfo, pRsp, false, false); + code = setQueryResultFromRsp(&pRequest->body.resInfo, pRsp, false, true); } return code; @@ -232,9 +232,7 @@ int32_t execDdlQuery(SRequestObj* pRequest, SQuery* pQuery) { return TSDB_CODE_SUCCESS; } -static SAppInstInfo* getAppInfo(SRequestObj* pRequest) { - return pRequest->pTscObj->pAppInfo; -} +static SAppInstInfo* getAppInfo(SRequestObj* pRequest) { return pRequest->pTscObj->pAppInfo; } void asyncExecLocalCmd(SRequestObj* pRequest, SQuery* pQuery) { SRetrieveTableRsp* pRsp = NULL; @@ -258,7 +256,7 @@ void asyncExecLocalCmd(SRequestObj* pRequest, SQuery* pQuery) { } pRequest->body.queryFp(pRequest->body.param, pRequest, 0); -// pRequest->body.fetchFp(pRequest->body.param, pRequest, pResultInfo->numOfRows); + // pRequest->body.fetchFp(pRequest->body.param, pRequest, pResultInfo->numOfRows); } int32_t asyncExecDdlQuery(SRequestObj* pRequest, SQuery* pQuery) { @@ -401,7 +399,7 @@ int32_t scheduleAsyncQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNod SQueryResult res = {.code = 0, .numOfRows = 0}; int32_t code = schedulerAsyncExecJob(pTransporter, pNodeList, pDag, &pRequest->body.queryJob, pRequest->sqlstr, - pRequest->metric.start, schdExecCallback, &res); + pRequest->metric.start, schdExecCallback, &res); pRequest->body.resInfo.execRes = res.res; @@ -457,7 +455,8 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList return pRequest->code; } - if (TDMT_VND_SUBMIT == pRequest->type || TDMT_VND_DELETE == pRequest->type || TDMT_VND_CREATE_TABLE == pRequest->type) { + if (TDMT_VND_SUBMIT == pRequest->type || TDMT_VND_DELETE == pRequest->type || + TDMT_VND_CREATE_TABLE == pRequest->type) { pRequest->body.resInfo.numOfRows = res.numOfRows; if (pRequest->body.queryJob != 0) { @@ -470,9 +469,9 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList return pRequest->code; } -int32_t handleSubmitExecRes(SRequestObj* pRequest, void* res, SCatalog* pCatalog, SEpSet *epset) { - int32_t code = 0; - SArray* pArray = NULL; +int32_t handleSubmitExecRes(SRequestObj* pRequest, void* res, SCatalog* pCatalog, SEpSet* epset) { + int32_t code = 0; + SArray* pArray = NULL; SSubmitRsp* pRsp = (SSubmitRsp*)res; if (pRsp->nBlocks <= 0) { return TSDB_CODE_SUCCESS; @@ -502,7 +501,7 @@ _return: return code; } -int32_t handleQueryExecRes(SRequestObj* pRequest, void* res, SCatalog* pCatalog, SEpSet *epset) { +int32_t handleQueryExecRes(SRequestObj* pRequest, void* res, SCatalog* pCatalog, SEpSet* epset) { int32_t code = 0; SArray* pArray = NULL; SArray* pTbArray = (SArray*)res; @@ -540,15 +539,15 @@ int32_t handleQueryExecRsp(SRequestObj* pRequest) { return TSDB_CODE_SUCCESS; } - SCatalog* pCatalog = NULL; - SAppInstInfo* pAppInfo = getAppInfo(pRequest); + SCatalog* pCatalog = NULL; + SAppInstInfo* pAppInfo = getAppInfo(pRequest); int32_t code = catalogGetHandle(pAppInfo->clusterId, &pCatalog); if (code) { return code; } - SEpSet epset = getEpSet_s(&pAppInfo->mgmtEp); + SEpSet epset = getEpSet_s(&pAppInfo->mgmtEp); SQueryExecRes* pRes = &pRequest->body.resInfo.execRes; switch (pRes->msgType) { @@ -566,8 +565,8 @@ int32_t handleQueryExecRsp(SRequestObj* pRequest) { break; } default: - tscError("0x%"PRIx64", invalid exec result for request type %d, reqId:0x%"PRIx64, pRequest->self, - pRequest->type, pRequest->requestId); + tscError("0x%" PRIx64 ", invalid exec result for request type %d, reqId:0x%" PRIx64, pRequest->self, + pRequest->type, pRequest->requestId); code = TSDB_CODE_APP_ERROR; } @@ -575,13 +574,13 @@ int32_t handleQueryExecRsp(SRequestObj* pRequest) { } void schedulerExecCb(SQueryResult* pResult, void* param, int32_t code) { - SRequestObj* pRequest = (SRequestObj*) param; + SRequestObj* pRequest = (SRequestObj*)param; pRequest->code = code; STscObj* pTscObj = pRequest->pTscObj; if (code != TSDB_CODE_SUCCESS && NEED_CLIENT_HANDLE_ERROR(code)) { - tscDebug("0x%"PRIx64" client retry to handle the error, code:%d - %s, tryCount:%d, reqId:0x%"PRIx64, pRequest->self, code, tstrerror(code), - pRequest->retry, pRequest->requestId); + tscDebug("0x%" PRIx64 " client retry to handle the error, code:%d - %s, tryCount:%d, reqId:0x%" PRIx64, + pRequest->self, code, tstrerror(code), pRequest->retry, pRequest->requestId); pRequest->prevCode = code; doAsyncQuery(pRequest, true); return; @@ -589,7 +588,7 @@ void schedulerExecCb(SQueryResult* pResult, void* param, int32_t code) { if (code == TSDB_CODE_SUCCESS) { code = handleQueryExecRsp(pRequest); - ASSERT(pRequest->code == TSDB_CODE_SUCCESS); + ASSERT(pRequest->code == TSDB_CODE_SUCCESS); pRequest->code = code; } @@ -697,16 +696,17 @@ void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery) { schedulerAsyncExecJob(pAppInfo->pTransporter, pNodeList, pRequest->body.pDag, &pRequest->body.queryJob, pRequest->sqlstr, pRequest->metric.start, schedulerExecCb, pRequest); } else { - tscError("0x%"PRIx64" failed to create query plan, code:%s 0x%"PRIx64, pRequest->self, tstrerror(code), pRequest->requestId); + tscError("0x%" PRIx64 " failed to create query plan, code:%s 0x%" PRIx64, pRequest->self, tstrerror(code), + pRequest->requestId); pRequest->body.queryFp(pRequest->body.param, pRequest, code); } - //todo not to be released here + // todo not to be released here taosArrayDestroy(pNodeList); break; } case QUERY_EXEC_MODE_EMPTY_RESULT: - pRequest->type = TSDB_SQL_RETRIEVE_EMPTY_RESULT; + pRequest->type = TSDB_SQL_RETRIEVE_EMPTY_RESULT; pRequest->body.queryFp(pRequest->body.param, pRequest, 0); break; default: @@ -1349,14 +1349,14 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32 p += sizeof(uint64_t); // check fields - for(int32_t i = 0; i < numOfCols; ++i) { - int16_t type = *(int16_t*) p; + for (int32_t i = 0; i < numOfCols; ++i) { + int16_t type = *(int16_t*)p; p += sizeof(int16_t); - int32_t bytes = *(int32_t*) p; + int32_t bytes = *(int32_t*)p; p += sizeof(int32_t); -// ASSERT(type == pFields[i].type && bytes == pFields[i].bytes); + ASSERT(type == pFields[i].type && bytes == pFields[i].bytes); } int32_t* colLength = (int32_t*)p; diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index 7d49c4206f..48c85cf265 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -132,6 +132,7 @@ typedef struct { // statistics int64_t pollCnt; // offset + int64_t committedOffset; int64_t currentOffset; // connection info int32_t vgId; @@ -193,6 +194,26 @@ typedef struct { void* userParam; } SMqCommitCbParam; +typedef struct { + tmq_t* tmq; + int8_t automatic; + int8_t async; + int8_t freeOffsets; + int8_t waitingRspNum; + int8_t totalRspNum; + tmq_resp_err_t rspErr; + tmq_commit_cb* userCb; + SArray* successfulOffsets; + SArray* failedOffsets; + void* userParam; + tsem_t rspSem; +} SMqCommitCbParamSet; + +typedef struct { + SMqCommitCbParamSet* params; + STqOffset* pOffset; +} SMqCommitCbParam2; + tmq_conf_t* tmq_conf_new() { tmq_conf_t* conf = taosMemoryCalloc(1, sizeof(tmq_conf_t)); conf->withTbName = false; @@ -343,6 +364,135 @@ int32_t tmqCommitCb(void* param, const SDataBuf* pMsg, int32_t code) { return 0; } +int32_t tmqCommitCb2(void* param, const SDataBuf* pBuf, int32_t code) { + SMqCommitCbParam2* pParam = (SMqCommitCbParam2*)param; + SMqCommitCbParamSet* pParamSet = (SMqCommitCbParamSet*)pParam->params; + // push into array + if (code == 0) { + taosArrayPush(pParamSet->failedOffsets, &pParam->pOffset); + } else { + taosArrayPush(pParamSet->successfulOffsets, &pParam->pOffset); + } + // count down waiting rsp + int8_t waitingRspNum = atomic_sub_fetch_8(&pParam->params->waitingRspNum, 1); + ASSERT(waitingRspNum >= 0); + + if (waitingRspNum == 0) { + // if no more waiting rsp + if (pParamSet->async) { + // call async cb func + if (pParamSet->automatic && pParamSet->tmq->commitCb) { + pParamSet->tmq->commitCb(pParamSet->tmq, pParamSet->rspErr, NULL, pParamSet->tmq->commitCbUserParam); + } else if (!pParamSet->automatic && pParamSet->userCb) { + // sem post + pParamSet->userCb(pParamSet->tmq, pParamSet->rspErr, NULL, pParamSet->userParam); + } + } + + taosArrayDestroyP(pParamSet->successfulOffsets, taosMemoryFree); + taosArrayDestroyP(pParamSet->failedOffsets, taosMemoryFree); + } + return 0; +} + +int32_t tmqComitInner2(tmq_t* tmq, int8_t automatic, int8_t async, tmq_commit_cb* userCb, void* userParam) { + int32_t code = -1; + + SMqCommitCbParamSet* pParamSet = taosMemoryCalloc(1, sizeof(SMqCommitCbParamSet)); + if (pParamSet == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + pParamSet->tmq = tmq; + pParamSet->automatic = automatic; + pParamSet->async = async; + pParamSet->freeOffsets = 1; + pParamSet->userCb = userCb; + pParamSet->userParam = userParam; + tsem_init(&pParamSet->rspSem, 0, 0); + + for (int32_t i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) { + SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); + for (int32_t j = 0; j < taosArrayGetSize(pTopic->vgs); j++) { + SMqClientVg* pVg = taosArrayGet(pTopic->vgs, i); + STqOffset* pOffset = taosMemoryCalloc(1, sizeof(STqOffset)); + if (pOffset == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + int32_t tlen = strlen(tmq->groupId); + memcpy(pOffset->subKey, tmq->groupId, tlen); + pOffset->subKey[tlen] = TMQ_SEPARATOR; + strcpy(pOffset->subKey + tlen + 1, pTopic->topicName); + int32_t len; + int32_t code; + tEncodeSize(tEncodeSTqOffset, pOffset, len, code); + if (code < 0) { + ASSERT(0); + } + void* buf = taosMemoryCalloc(1, sizeof(SMsgHead) + len); + ((SMsgHead*)buf)->vgId = htonl(pVg->vgId); + + void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); + + SEncoder encoder; + tEncoderInit(&encoder, abuf, len); + tEncodeSTqOffset(&encoder, pOffset); + + // build param + SMqCommitCbParam2* pParam = taosMemoryCalloc(1, sizeof(SMqCommitCbParam2)); + pParam->params = pParamSet; + pParam->pOffset = pOffset; + + // build send info + SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); + if (pMsgSendInfo == NULL) { + // TODO + continue; + } + pMsgSendInfo->msgInfo = (SDataBuf){ + .pData = buf, + .len = len, + .handle = NULL, + }; + + pMsgSendInfo->requestId = generateRequestId(); + pMsgSendInfo->requestObjRefId = 0; + pMsgSendInfo->param = pParam; + pMsgSendInfo->fp = tmqCommitCb2; + pMsgSendInfo->msgType = TDMT_MND_MQ_COMMIT_OFFSET; + // send msg + + SEpSet epSet = getEpSet_s(&tmq->pTscObj->pAppInfo->mgmtEp); + int64_t transporterId = 0; + asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, pMsgSendInfo); + } + } + + if (!async) { + tsem_wait(&pParamSet->rspSem); + code = pParamSet->rspErr; + tsem_destroy(&pParamSet->rspSem); + } else { + code = 0; + } + + if (code != 0 && async) { + if (automatic) { + tmq->commitCb(tmq, code, NULL, tmq->commitCbUserParam); + } else { + userCb(tmq, code, NULL, userParam); + } + } + + if (!async) { + taosArrayDestroyP(pParamSet->successfulOffsets, taosMemoryFree); + taosArrayDestroyP(pParamSet->failedOffsets, taosMemoryFree); + } + + return 0; +} + int32_t tmqCommitInner(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, int8_t automatic, int8_t async, tmq_commit_cb* userCb, void* userParam) { SMqCMCommitOffsetReq req; @@ -890,12 +1040,13 @@ bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqAskEpRsp* pRsp) { sprintf(vgKey, "%s:%d", topic.topicName, pVgEp->vgId); int64_t* pOffset = taosHashGet(pHash, vgKey, strlen(vgKey)); int64_t offset = pVgEp->offset; - tscDebug("consumer %ld epoch %d vg %d offset og to %ld", tmq->consumerId, epoch, pVgEp->vgId, offset); + tscDebug("consumer %ld(epoch %d) original offset of vg %d is %ld", tmq->consumerId, epoch, pVgEp->vgId, offset); if (pOffset != NULL) { offset = *pOffset; - tscDebug("consumer %ld epoch %d vg %d found %s", tmq->consumerId, epoch, pVgEp->vgId, vgKey); + tscDebug("consumer %ld(epoch %d) receive offset of vg %d, full key is %s", tmq->consumerId, epoch, pVgEp->vgId, + vgKey); } - tscDebug("consumer %ld epoch %d vg %d offset set to %ld", tmq->consumerId, epoch, pVgEp->vgId, offset); + tscDebug("consumer %ld(epoch %d) offset of vg %d updated to %ld", tmq->consumerId, epoch, pVgEp->vgId, offset); SMqClientVg clientVg = { .pollCnt = 0, .currentOffset = offset, @@ -1226,9 +1377,8 @@ SMqRspObj* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_RSP) { SMqPollRspWrapper* pollRspWrapper = (SMqPollRspWrapper*)rspWrapper; /*atomic_sub_fetch_32(&tmq->readyRequest, 1);*/ - /*printf("handle poll rsp %d\n", rspMsg->head.mqMsgType);*/ - if (pollRspWrapper->msg.head.epoch == atomic_load_32(&tmq->epoch)) { - /*printf("epoch match\n");*/ + int32_t consumerEpoch = atomic_load_32(&tmq->epoch); + if (pollRspWrapper->msg.head.epoch == consumerEpoch) { SMqClientVg* pVg = pollRspWrapper->vgHandle; /*printf("vg %d offset %ld up to %ld\n", pVg->vgId, pVg->currentOffset, rspMsg->msg.rspOffset);*/ pVg->currentOffset = pollRspWrapper->msg.rspOffset; @@ -1243,7 +1393,8 @@ SMqRspObj* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { taosFreeQitem(pollRspWrapper); return pRsp; } else { - /*printf("epoch mismatch\n");*/ + tscDebug("msg discard since epoch mismatch: msg epoch %d, consumer epoch %d\n", pollRspWrapper->msg.head.epoch, + consumerEpoch); taosFreeQitem(pollRspWrapper); } } else { @@ -1263,10 +1414,14 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) { SMqRspObj* rspObj; int64_t startTime = taosGetTimestampMs(); +#if 0 + tmqHandleAllDelayedTask(tmq); + tmqPollImpl(tmq, timeout); rspObj = tmqHandleAllRsp(tmq, timeout, false); if (rspObj) { return (TAOS_RES*)rspObj; } +#endif // in no topic status also need process delayed task if (atomic_load_8(&tmq->status) == TMQ_CONSUMER_STATUS__INIT) { @@ -1359,8 +1514,7 @@ const char* tmq_get_table_name(TAOS_RES* res) { pRspObj->resIter >= pRspObj->rsp.blockNum) { return NULL; } - const char* name = taosArrayGetP(pRspObj->rsp.blockTbName, pRspObj->resIter); - return name; + return (const char*)taosArrayGetP(pRspObj->rsp.blockTbName, pRspObj->resIter); } return NULL; } diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index b7a9ef88b6..4d1f5a34b1 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1756,7 +1756,7 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, boo createTbReq.type = TSDB_CHILD_TABLE; createTbReq.ctb.suid = suid; - STagVal tagVal = {.cid = 1, + STagVal tagVal = {.cid = pDataBlock->info.numOfCols + 1, .type = TSDB_DATA_TYPE_UBIGINT, .pData = (uint8_t*)&pDataBlock->info.groupId, .nData = sizeof(uint64_t)}; @@ -1821,7 +1821,7 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, boo createTbReq.type = TSDB_CHILD_TABLE; createTbReq.ctb.suid = suid; - STagVal tagVal = {.cid = 1, + STagVal tagVal = {.cid = pDataBlock->info.numOfCols + 1, .type = TSDB_DATA_TYPE_UBIGINT, .pData = (uint8_t*)&pDataBlock->info.groupId, .nData = sizeof(uint64_t)}; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index d16ab57ea9..4ebd91b0da 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -4746,3 +4746,32 @@ void tFreeSMAlterStbRsp(SMAlterStbRsp *pRsp) { taosMemoryFree(pRsp->pMeta); } } + +int32_t tEncodeSTqOffset(SEncoder *pEncoder, const STqOffset *pOffset) { + if (tEncodeI8(pEncoder, pOffset->type) < 0) return -1; + if (pOffset->type == TMQ_OFFSET__SNAPSHOT) { + if (tEncodeI64(pEncoder, pOffset->uid) < 0) return -1; + if (tEncodeI64(pEncoder, pOffset->ts) < 0) return -1; + } else if (pOffset->type == TMQ_OFFSET__LOG) { + if (tEncodeI64(pEncoder, pOffset->version) < 0) return -1; + } else { + ASSERT(0); + } + if (tEncodeCStr(pEncoder, pOffset->subKey) < 0) return -1; + return 0; +} + +int32_t tDecodeSTqOffset(SDecoder *pDecoder, STqOffset *pOffset) { + if (tDecodeI8(pDecoder, &pOffset->type) < 0) return -1; + if (pOffset->type == TMQ_OFFSET__SNAPSHOT) { + if (tDecodeI64(pDecoder, &pOffset->uid) < 0) return -1; + if (tDecodeI64(pDecoder, &pOffset->ts) < 0) return -1; + } else if (pOffset->type == TMQ_OFFSET__LOG) { + if (tDecodeI64(pDecoder, &pOffset->version) < 0) return -1; + } else { + ASSERT(0); + } + if (tDecodeCStrTo(pDecoder, pOffset->subKey) < 0) return -1; + return 0; +} + diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index ee120576c3..750006d05f 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -344,6 +344,7 @@ SArray *vmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_VND_SUBMIT_RSMA, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_MQ_VG_CHANGE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_MQ_VG_DELETE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_MQ_COMMIT_OFFSET, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_CONSUME, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_DELETE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_QUERY_HEARTBEAT, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER; diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index 5a8564bfd1..2ee0673ce5 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -41,7 +41,6 @@ extern "C" { #define tqTrace(...) do { if (tqDebugFlag & DEBUG_TRACE) { taosPrintLog("TQ ", DEBUG_TRACE, tqDebugFlag, __VA_ARGS__); }} while(0) // clang-format on -typedef struct STqOffsetCfg STqOffsetCfg; typedef struct STqOffsetStore STqOffsetStore; // tqRead @@ -127,14 +126,15 @@ typedef struct { } STqHandle; struct STQ { - char* path; - SHashObj* pushMgr; // consumerId -> STqHandle* - SHashObj* handles; // subKey -> STqHandle - SHashObj* pStreamTasks; // taksId -> SStreamTask - SVnode* pVnode; - SWal* pWal; - TDB* pMetaStore; - TTB* pExecStore; + char* path; + SHashObj* pushMgr; // consumerId -> STqHandle* + SHashObj* handles; // subKey -> STqHandle + SHashObj* pStreamTasks; // taksId -> SStreamTask + STqOffsetStore* pOffsetStore; + SVnode* pVnode; + SWal* pWal; + TDB* pMetaStore; + TTB* pExecStore; }; typedef struct { @@ -157,17 +157,19 @@ int32_t tqMetaClose(STQ* pTq); int32_t tqMetaSaveHandle(STQ* pTq, const char* key, const STqHandle* pHandle); int32_t tqMetaDeleteHandle(STQ* pTq, const char* key); +typedef struct { + int32_t size; +} STqOffsetHead; + +STqOffsetStore* tqOffsetOpen(); +void tqOffsetClose(STqOffsetStore*); +STqOffset* tqOffsetRead(STqOffsetStore* pStore, const char* subscribeKey); +int32_t tqOffsetWrite(STqOffsetStore* pStore, const STqOffset* pOffset); +int32_t tqOffsetSnapshot(STqOffsetStore* pStore); + // tqSink void tqTableSink(SStreamTask* pTask, void* vnode, int64_t ver, void* data); -// tqOffset -STqOffsetStore* tqOffsetOpen(STqOffsetCfg*); -void tqOffsetClose(STqOffsetStore*); -int64_t tqOffsetFetch(STqOffsetStore* pStore, const char* subscribeKey); -int32_t tqOffsetCommit(STqOffsetStore* pStore, const char* subscribeKey, int64_t offset); -int32_t tqOffsetPersist(STqOffsetStore* pStore, const char* subscribeKey); -int32_t tqOffsetPersistAll(STqOffsetStore* pStore); - #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 0c2b09a493..300a5f890e 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -137,6 +137,7 @@ int tqCommit(STQ*); int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd); int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen); int32_t tqProcessVgDeleteReq(STQ* pTq, char* msg, int32_t msgLen); +int32_t tqProcessOffsetCommitReq(STQ* pTq, char* msg, int32_t msgLen); int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId); int32_t tqProcessTaskDeploy(STQ* pTq, char* msg, int32_t msgLen); int32_t tqProcessStreamTrigger(STQ* pTq, SSubmitReq* data); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 4a5ea49d79..22685c1e19 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -66,19 +66,23 @@ STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal) { ASSERT(0); } + if (tqOffsetOpen(pTq) < 0) { + ASSERT(0); + } + return pTq; } void tqClose(STQ* pTq) { if (pTq) { - taosMemoryFreeClear(pTq->path); + tqOffsetClose(pTq->pOffsetStore); taosHashCleanup(pTq->handles); taosHashCleanup(pTq->pStreamTasks); taosHashCleanup(pTq->pushMgr); + taosMemoryFree(pTq->path); tqMetaClose(pTq); taosMemoryFree(pTq); } - // TODO } int32_t tqSendPollRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqDataBlkRsp* pRsp) { @@ -109,6 +113,33 @@ int32_t tqSendPollRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, con return 0; } +int32_t tqProcessOffsetCommitReq(STQ* pTq, char* msg, int32_t msgLen) { + STqOffset offset = {0}; + SDecoder decoder; + tDecoderInit(&decoder, msg, msgLen); + if (tDecodeSTqOffset(&decoder, &offset) < 0) { + ASSERT(0); + return -1; + } + tDecoderClear(&decoder); + + if (offset.type == TMQ_OFFSET__SNAPSHOT) { + tqDebug("receive offset commit msg to %s, offset(type:snapshot) uid: %ld, ts: %ld", offset.subKey, offset.uid, + offset.ts); + } else if (offset.type == TMQ_OFFSET__LOG) { + tqDebug("receive offset commit msg to %s, offset(type:log) version: %ld", offset.subKey, offset.version); + } else { + ASSERT(0); + } + + if (tqOffsetWrite(pTq->pOffsetStore, &offset) < 0) { + ASSERT(0); + return -1; + } + + return 0; +} + int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { SMqPollReq* pReq = pMsg->pCont; int64_t consumerId = pReq->consumerId; diff --git a/source/dnode/vnode/src/tq/tqOffset.c b/source/dnode/vnode/src/tq/tqOffset.c index 4d83a67579..2db49d7cf5 100644 --- a/source/dnode/vnode/src/tq/tqOffset.c +++ b/source/dnode/vnode/src/tq/tqOffset.c @@ -16,26 +16,113 @@ #include "tq.h" -enum ETqOffsetPersist { - TQ_OFFSET_PERSIST__LAZY = 1, - TQ_OFFSET_PERSIST__EAGER, -}; - -struct STqOffsetCfg { - int8_t persistPolicy; -}; - struct STqOffsetStore { - STqOffsetCfg cfg; - SHashObj* pHash; // SHashObj + STQ* pTq; + SHashObj* pHash; // SHashObj }; -STqOffsetStore* tqOffsetOpen(STqOffsetCfg* pCfg) { - STqOffsetStore* pStore = taosMemoryMalloc(sizeof(STqOffsetStore)); +STqOffsetStore* tqOffsetOpen(STQ* pTq) { + STqOffsetStore* pStore = taosMemoryCalloc(1, sizeof(STqOffsetStore)); if (pStore == NULL) { return NULL; } - memcpy(&pStore->cfg, pCfg, sizeof(STqOffsetCfg)); pStore->pHash = taosHashInit(64, MurmurHash3_32, true, HASH_NO_LOCK); + if (pStore->pHash == NULL) { + if (pStore->pHash) taosHashCleanup(pStore->pHash); + return NULL; + } + TdFilePtr pFile = taosOpenFile(pStore->pTq->path, TD_FILE_READ); + if (pFile != NULL) { + STqOffsetHead head = {0}; + int64_t code; + + while (1) { + if ((code = taosReadFile(pFile, &head, sizeof(STqOffsetHead))) != sizeof(STqOffsetHead)) { + if (code < 0) { + break; + } else { + ASSERT(0); + // TODO handle error + } + } + int32_t size = htonl(head.size); + void* memBuf = taosMemoryCalloc(1, size); + if ((code = taosReadFile(pFile, memBuf, size)) != size) { + ASSERT(0); + // TODO handle error + } + STqOffset offset; + SDecoder decoder; + tDecoderInit(&decoder, memBuf, size); + if (tDecodeSTqOffset(&decoder, &offset) < 0) { + ASSERT(0); + } + tDecoderClear(&decoder); + if (taosHashPut(pStore->pHash, offset.subKey, strlen(offset.subKey), &offset, sizeof(STqOffset)) < 0) { + ASSERT(0); + // TODO + } + } + + taosCloseFile(&pFile); + } return pStore; } + +void tqOffsetClose(STqOffsetStore* pStore) { + tqOffsetSnapshot(pStore); + taosHashCleanup(pStore->pHash); +} + +STqOffset* tqOffsetRead(STqOffsetStore* pStore, const char* subscribeKey) { + return (STqOffset*)taosHashGet(pStore->pHash, subscribeKey, strlen(subscribeKey)); +} + +int32_t tqOffsetWrite(STqOffsetStore* pStore, const STqOffset* pOffset) { + return taosHashPut(pStore->pHash, pOffset->subKey, strlen(pOffset->subKey), pOffset, sizeof(STqOffset)); +} + +int32_t tqOffsetSnapshot(STqOffsetStore* pStore) { + // open file + // TODO file name should be with a version + TdFilePtr pFile = taosOpenFile(pStore->pTq->path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND); + if (pFile == NULL) { + ASSERT(0); + return -1; + } + void* pIter = NULL; + while (1) { + pIter = taosHashIterate(pStore->pHash, pIter); + if (pIter == NULL) break; + STqOffset* pOffset = (STqOffset*)pIter; + int32_t bodyLen; + int32_t code; + tEncodeSize(tEncodeSTqOffset, pOffset, bodyLen, code); + ASSERT(code == 0); + if (code < 0) { + ASSERT(0); + taosHashCancelIterate(pStore->pHash, pIter); + return -1; + } + + int32_t totLen = sizeof(STqOffsetHead) + bodyLen; + void* buf = taosMemoryCalloc(1, totLen); + void* abuf = POINTER_SHIFT(buf, sizeof(STqOffsetHead)); + + ((STqOffsetHead*)buf)->size = htonl(bodyLen); + SEncoder encoder; + tEncoderInit(&encoder, abuf, bodyLen); + tEncodeSTqOffset(&encoder, pOffset); + // write file + int64_t writeLen; + if ((writeLen = taosWriteFile(pFile, buf, totLen)) != bodyLen) { + ASSERT(0); + tqError("write offset incomplete, len %d, write len %ld", bodyLen, writeLen); + taosHashCancelIterate(pStore->pHash, pIter); + return -1; + } + } + // close and rename file + taosCloseFile(&pFile); + return 0; +} diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index ab2efa4791..186ab4b528 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -148,17 +148,24 @@ int32_t vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp case TDMT_VND_MQ_VG_CHANGE: if (tqProcessVgChangeReq(pVnode->pTq, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), pMsg->contLen - sizeof(SMsgHead)) < 0) { - // TODO: handle error + goto _err; } break; case TDMT_VND_MQ_VG_DELETE: if (tqProcessVgDeleteReq(pVnode->pTq, pMsg->pCont, pMsg->contLen) < 0) { - // TODO: handle error + goto _err; + } + break; + case TDMT_VND_MQ_COMMIT_OFFSET: + if (tqProcessOffsetCommitReq(pVnode->pTq, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), + pMsg->contLen - sizeof(SMsgHead)) < 0) { + goto _err; } break; case TDMT_STREAM_TASK_DEPLOY: { if (tqProcessTaskDeploy(pVnode->pTq, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), pMsg->contLen - sizeof(SMsgHead)) < 0) { + goto _err; } } break; case TDMT_VND_ALTER_CONFIRM: @@ -901,8 +908,8 @@ static int32_t vnodeProcessAlterConfirmReq(SVnode *pVnode, int64_t version, void static int32_t vnodeProcessAlterHasnRangeReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { vInfo("vgId:%d, alter hashrange msg will be processed", TD_VID(pVnode)); - // todo - // 1. stop work + // todo + // 1. stop work // 2. adjust hash range / compact / remove wals / rename vgroups // 3. reload sync return 0; From b298d45bb7e24cca6ca014d7e9de5c59efac6e2e Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Tue, 14 Jun 2022 13:46:28 +0800 Subject: [PATCH 54/78] feat(stream): create sub table --- source/common/src/tdatablock.c | 13 ++--- tests/script/jenkins/basic.txt | 1 + tests/script/tsim/stream/partitionby.sim | 68 ++++++++++++++++++++++++ 3 files changed, 76 insertions(+), 6 deletions(-) create mode 100644 tests/script/tsim/stream/partitionby.sim diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index b7a9ef88b6..5affcf5100 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1708,6 +1708,7 @@ char* buildCtbNameByGroupId(const char* stbName, uint64_t groupId) { pTag->keyLen = strlen(pTag->key); pTag->type = TSDB_DATA_TYPE_UBIGINT; pTag->u = groupId; + pTag->length = sizeof(uint64_t); taosArrayPush(tags, &pTag); void* cname = taosMemoryCalloc(1, TSDB_TABLE_NAME_LEN + 1); @@ -1756,10 +1757,10 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, boo createTbReq.type = TSDB_CHILD_TABLE; createTbReq.ctb.suid = suid; - STagVal tagVal = {.cid = 1, + STagVal tagVal = {.cid = pDataBlock->info.numOfCols + 1, .type = TSDB_DATA_TYPE_UBIGINT, - .pData = (uint8_t*)&pDataBlock->info.groupId, - .nData = sizeof(uint64_t)}; + .i64 = (int64_t)pDataBlock->info.groupId, + }; STag* pTag = NULL; taosArrayClear(tagArray); taosArrayPush(tagArray, &tagVal); @@ -1821,10 +1822,10 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, boo createTbReq.type = TSDB_CHILD_TABLE; createTbReq.ctb.suid = suid; - STagVal tagVal = {.cid = 1, + STagVal tagVal = {.cid = pDataBlock->info.numOfCols + 1, .type = TSDB_DATA_TYPE_UBIGINT, - .pData = (uint8_t*)&pDataBlock->info.groupId, - .nData = sizeof(uint64_t)}; + .i64 = (int64_t)pDataBlock->info.groupId, + }; taosArrayClear(tagArray); taosArrayPush(tagArray, &tagVal); STag* pTag = NULL; diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index db8a055362..0a03ff58fc 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -78,6 +78,7 @@ # ./test.sh -f tsim/stream/state0.sim # ./test.sh -f tsim/stream/triggerInterval0.sim # ./test.sh -f tsim/stream/triggerSession0.sim +# ./test.sh -f tsim/stream/partitionby.sim # ---- transaction diff --git a/tests/script/tsim/stream/partitionby.sim b/tests/script/tsim/stream/partitionby.sim new file mode 100644 index 0000000000..df1e096551 --- /dev/null +++ b/tests/script/tsim/stream/partitionby.sim @@ -0,0 +1,68 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sleep 50 +sql connect + +sql create database test vgroups 4; +sql use test; +sql create stable st(ts timestamp, a int, b int , c int, d double) tags(ta int,tb int,tc int); +sql create table ts1 using st tags(1,1,1); +sql create table ts2 using st tags(2,2,2); +sql create table ts3 using st tags(3,2,2); +sql create table ts4 using st tags(4,2,2); +sql create stream stream_t1 trigger at_once into streamtST1 as select _wstartts, count(*) c1, count(d) c2 , sum(a) c3 , max(b) c4, min(c) c5 from st partition by ta,tb,tc interval(10s); + +sql insert into ts1 values(1648791213001,1,12,3,1.0); +sql insert into ts2 values(1648791213001,1,12,3,1.0); + +sql insert into ts3 values(1648791213001,1,12,3,1.0); +sql insert into ts4 values(1648791213001,1,12,3,1.0); +$loop_count = 0 + +loop0: +sleep 300 +sql select * from streamtST1; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 4 then +print =====rows=$rows +goto loop0 +endi + + +sql create database test1 vgroups 1; +sql use test1; +sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int); +sql create table ts1 using st tags(1,2,3); +sql create table ts2 using st tags(1,3,4); +sql create table ts3 using st tags(1,4,5); + +sql create stream streams1 trigger at_once into streamt as select _wstartts, count(*) c1, count(a) c2 from st partition by ta,tb,tc interval(10s); + + +sql insert into ts1 values(1648791211000,1,2,3); + +sql insert into ts2 values(1648791211000,1,2,3); + +$loop_count = 0 + +loop0: +sleep 300 +sql select * from streamt; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 2 then +print =====rows=$rows +goto loop0 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file From 92cf4c4cb613cd046099c5d54390f22adf064632 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Tue, 14 Jun 2022 14:38:35 +0800 Subject: [PATCH 55/78] os: fix lz4 error --- source/os/src/osLz4.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/os/src/osLz4.c b/source/os/src/osLz4.c index f4e4951857..7f3c42fb6c 100644 --- a/source/os/src/osLz4.c +++ b/source/os/src/osLz4.c @@ -45,13 +45,13 @@ int32_t BUILDIN_CLZL(uint64_t val) { #else _MyBitScanReverse64(&r, val); #endif - return (int)(r >> 3); + return (int)(63 - r); } int32_t BUILDIN_CLZ(uint32_t val) { unsigned long r = 0; _BitScanReverse(&r, val); - return (int)(r >> 3); + return (int)(31 - r); } int32_t BUILDIN_CTZL(uint64_t val) { @@ -61,13 +61,13 @@ int32_t BUILDIN_CTZL(uint64_t val) { #else _MyBitScanForward64(&r, val); #endif - return (int)(r >> 3); + return (int)(r); } int32_t BUILDIN_CTZ(uint32_t val) { unsigned long r = 0; _BitScanForward(&r, val); - return (int)(r >> 3); + return (int)(r); } #endif From 7435221e3280de9b723bc8108b289e18ad6b2298 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 14 Jun 2022 14:40:01 +0800 Subject: [PATCH 56/78] opt: optimize generating groupid in partition/group by tag --- include/util/taoserror.h | 2 ++ source/libs/executor/src/executorimpl.c | 16 +++++++++++----- source/util/src/terror.c | 1 + 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 03308e395f..8d9951f9e3 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -695,6 +695,8 @@ int32_t* taosGetErrno(); #define TSDB_CODE_RSMA_INVALID_ENV TAOS_DEF_ERROR_CODE(0, 0x3150) #define TSDB_CODE_RSMA_INVALID_STAT TAOS_DEF_ERROR_CODE(0, 0x3151) +//index +#define TSDB_CODE_INDEX_REBUILDING TAOS_DEF_ERROR_CODE(0, 0x3200) #ifdef __cplusplus } diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index e4c072a396..f878e9e027 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -4690,8 +4690,9 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo return NULL; } - SArray* groupKyes = extractPartitionColInfo(pTableScanNode->pPartitionKeys); - code = generateGroupIdMap(pTableListInfo, pHandle, groupKyes); //todo for json + SArray* groupKeys = extractPartitionColInfo(pTableScanNode->pPartitionKeys); + code = generateGroupIdMap(pTableListInfo, pHandle, groupKeys); //todo for json + taosArrayDestroy(groupKeys); if (code){ tsdbCleanupReadHandle(pDataReader); return NULL; @@ -4725,8 +4726,9 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo qDebug("%s pDataReader is not NULL", GET_TASKID(pTaskInfo)); } - SArray* groupKyes = extractPartitionColInfo(pTableScanNode->pPartitionKeys); - int32_t code = generateGroupIdMap(pTableListInfo, pHandle, groupKyes); //todo for json + SArray* groupKeys = extractPartitionColInfo(pTableScanNode->pPartitionKeys); + int32_t code = generateGroupIdMap(pTableListInfo, pHandle, groupKeys); //todo for json + taosArrayDestroy(groupKeys); if (code){ tsdbCleanupReadHandle(pDataReader); return NULL; @@ -5059,6 +5061,7 @@ SArray* extractColumnInfo(SNodeList* pNodeList) { } SArray* extractPartitionColInfo(SNodeList* pNodeList) { + if(!pNodeList) return NULL; size_t numOfCols = LIST_LENGTH(pNodeList); SArray* pList = taosArrayInit(numOfCols, sizeof(SColumn)); if (pList == NULL) { @@ -5163,7 +5166,9 @@ int32_t getTableList(void* metaHandle, int32_t tableType, uint64_t tableUid, STa SArray* res = taosArrayInit(8, sizeof(uint64_t)); code = doFilterTag(pTagCond, &metaArg, res); - if (code != TSDB_CODE_SUCCESS) { + if (code == TSDB_CODE_INDEX_REBUILDING){ // todo + // doFilter(); + } else if (code != TSDB_CODE_SUCCESS) { qError("failed to get tableIds, reason: %s, suid: %" PRIu64 "", tstrerror(code), tableUid); taosArrayDestroy(res); terrno = code; @@ -5171,6 +5176,7 @@ int32_t getTableList(void* metaHandle, int32_t tableType, uint64_t tableUid, STa } else { qDebug("sucess to get tableIds, size: %d, suid: %" PRIu64 "", (int)taosArrayGetSize(res), tableUid); } + for (int i = 0; i < taosArrayGetSize(res); i++) { STableKeyInfo info = {.lastKey = TSKEY_INITIAL_VAL, .uid = *(uint64_t*)taosArrayGet(res, i)}; taosArrayPush(pListInfo->pTableList, &info); diff --git a/source/util/src/terror.c b/source/util/src/terror.c index e122ad0ab6..079d5ef590 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -566,6 +566,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TSMA_RM_SKEY_IN_HASH, "Rm tsma skey in cac TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_INVALID_ENV, "Invalid rsma env") TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_INVALID_STAT, "Invalid rsma state") +TAOS_DEFINE_ERROR(TSDB_CODE_INDEX_REBUILDING, "Index is rebuilding") #ifdef TAOS_ERROR_C }; From 5268b5233aa772c4304c11b970d0752b283ab9ba Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Tue, 14 Jun 2022 14:40:06 +0800 Subject: [PATCH 57/78] refactor(sync): update replica index in snapshot sender --- source/libs/sync/src/syncMain.c | 50 ++++++++++++++++++----------- source/libs/sync/src/syncSnapshot.c | 8 +++-- 2 files changed, 37 insertions(+), 21 deletions(-) diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 8be8c14b8f..feced0afdf 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -561,7 +561,7 @@ int32_t syncNodePropose(SSyncNode* pSyncNode, const SRpcMsg* pMsg, bool isWeak) stub.createTime = taosGetTimestampMs(); stub.rpcMsg = *pMsg; uint64_t seqNum = syncRespMgrAdd(pSyncNode->pSyncRespMgr, &stub); - sDebug("vgId:%d, sync event propose, type:%s seq:%" PRIu64 " handle:%p", pSyncNode->vgId, TMSG_INFO(pMsg->msgType), + sDebug("vgId:%d sync event propose, type:%s seq:%" PRIu64 " handle:%p", pSyncNode->vgId, TMSG_INFO(pMsg->msgType), seqNum, pMsg->info.handle); SyncClientRequest* pSyncMsg = syncClientRequestBuild2(pMsg, seqNum, isWeak, pSyncNode->vgId); @@ -1244,7 +1244,13 @@ void syncNodeUpdateConfig(SSyncNode* pSyncNode, SSyncCfg* newConfig, SyncIndex l SRaftId oldReplicasId[TSDB_MAX_REPLICA]; memcpy(oldReplicasId, pSyncNode->replicasId, sizeof(oldReplicasId)); SSyncSnapshotSender* oldSenders[TSDB_MAX_REPLICA]; - memcpy(oldSenders, pSyncNode->senders, sizeof(oldSenders)); + for (int i = 0; i < TSDB_MAX_REPLICA; ++i) { + oldSenders[i] = (pSyncNode->senders)[i]; + sDebug("vgId:%d sync event save senders %d, %p", pSyncNode->vgId, i, oldSenders[i]); + if (gRaftDetailLog) { + ; + } + } // init internal pSyncNode->myNodeInfo = pSyncNode->pRaftCfg->cfg.nodeInfo[pSyncNode->pRaftCfg->cfg.myIndex]; @@ -1286,22 +1292,24 @@ void syncNodeUpdateConfig(SSyncNode* pSyncNode, SSyncCfg* newConfig, SyncIndex l // reset new for (int i = 0; i < pSyncNode->replicaNum; ++i) { // reset sender + bool reset = false; for (int j = 0; j < TSDB_MAX_REPLICA; ++j) { if (syncUtilSameId(&(pSyncNode->replicasId)[i], &oldReplicasId[j])) { char host[128]; uint16_t port; syncUtilU642Addr((pSyncNode->replicasId)[i].addr, host, sizeof(host), &port); - sDebug("vgId:%d sync event reset sender for %lu, %s:%d", pSyncNode->vgId, (pSyncNode->replicasId)[i].addr, host, - port); + sDebug("vgId:%d sync event reset sender for %lu, newIndex:%d, %s:%d, %p", pSyncNode->vgId, + (pSyncNode->replicasId)[i].addr, i, host, port, oldSenders[j]); (pSyncNode->senders)[i] = oldSenders[j]; oldSenders[j] = NULL; + reset = true; + + // reset replicaIndex + int32_t oldreplicaIndex = (pSyncNode->senders)[i]->replicaIndex; + (pSyncNode->senders)[i]->replicaIndex = i; + sDebug("vgId:%d sync event udpate replicaIndex from %d to %d, %s:%d, %p, reset:%d", pSyncNode->vgId, + oldreplicaIndex, i, host, port, (pSyncNode->senders)[i], reset); } - - // reset replicaIndex - int32_t oldreplicaIndex = (pSyncNode->senders)[i]->replicaIndex; - (pSyncNode->senders)[i]->replicaIndex = i; - - sDebug("vgId:%d sync event udpate replicaIndex from %d to %d", pSyncNode->vgId, oldreplicaIndex, i); } } @@ -1309,7 +1317,7 @@ void syncNodeUpdateConfig(SSyncNode* pSyncNode, SSyncCfg* newConfig, SyncIndex l for (int i = 0; i < TSDB_MAX_REPLICA; ++i) { if ((pSyncNode->senders)[i] == NULL) { (pSyncNode->senders)[i] = snapshotSenderCreate(pSyncNode, i); - sDebug("vgId:%d sync event create new sender replicaIndex:%d", pSyncNode->vgId, i); + sDebug("vgId:%d sync event create new sender %p replicaIndex:%d", pSyncNode->vgId, (pSyncNode->senders)[i], i); } } @@ -1317,8 +1325,8 @@ void syncNodeUpdateConfig(SSyncNode* pSyncNode, SSyncCfg* newConfig, SyncIndex l for (int i = 0; i < TSDB_MAX_REPLICA; ++i) { if (oldSenders[i] != NULL) { snapshotSenderDestroy(oldSenders[i]); + sDebug("vgId:%d sync event delete old sender %p replicaIndex:%d", pSyncNode->vgId, oldSenders[i], i); oldSenders[i] = NULL; - sDebug("vgId:%d sync event delete old sender replicaIndex:%d", pSyncNode->vgId, i); } } @@ -1378,8 +1386,8 @@ void syncNodeUpdateTerm(SSyncNode* pSyncNode, SyncTerm term) { } void syncNodeBecomeFollower(SSyncNode* pSyncNode, const char* debugStr) { - sDebug("vgId:%d sync event become follower, isStandBy:%d, %s", pSyncNode->vgId, pSyncNode->pRaftCfg->isStandBy, - debugStr); + sDebug("vgId:%d sync event become follower, isStandBy:%d, replicaNum:%d, %s", pSyncNode->vgId, + pSyncNode->pRaftCfg->isStandBy, pSyncNode->replicaNum, debugStr); // maybe clear leader cache if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) { @@ -1413,8 +1421,8 @@ void syncNodeBecomeFollower(SSyncNode* pSyncNode, const char* debugStr) { // /\ UNCHANGED <> // void syncNodeBecomeLeader(SSyncNode* pSyncNode, const char* debugStr) { - sDebug("vgId:%d sync event become leader, isStandBy:%d, %s", pSyncNode->vgId, pSyncNode->pRaftCfg->isStandBy, - debugStr); + sDebug("vgId:%d sync event become leader, isStandBy:%d, replicaNum:%d %s", pSyncNode->vgId, + pSyncNode->pRaftCfg->isStandBy, pSyncNode->replicaNum, debugStr); // state change pSyncNode->state = TAOS_SYNC_STATE_LEADER; @@ -2028,14 +2036,18 @@ static int32_t syncNodeConfigChange(SSyncNode* ths, SRpcMsg* pRpcMsg, SSyncRaftE // change isStandBy to normal if (!isDrop) { + char tmpbuf[128]; + snprintf(tmpbuf, sizeof(tmpbuf), "config change from %d to %d", oldSyncCfg.replicaNum, newSyncCfg.replicaNum); if (ths->state == TAOS_SYNC_STATE_LEADER) { - syncNodeBecomeLeader(ths, "config change"); + syncNodeBecomeLeader(ths, tmpbuf); } else { - syncNodeBecomeFollower(ths, "config change"); + syncNodeBecomeFollower(ths, tmpbuf); } } } else { - syncNodeBecomeFollower(ths, "config change2"); + char tmpbuf[128]; + snprintf(tmpbuf, sizeof(tmpbuf), "config change2 from %d to %d", oldSyncCfg.replicaNum, newSyncCfg.replicaNum); + syncNodeBecomeFollower(ths, tmpbuf); } if (gRaftDetailLog) { diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index d21a1e96e9..afebfa798e 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -588,6 +588,8 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { // maybe update lastconfig if (pMsg->lastConfigIndex >= SYNC_INDEX_BEGIN) { + int32_t oldReplicaNum = pSyncNode->replicaNum; + // update new config myIndex bool IamInNew = false; SSyncCfg newSyncCfg = pMsg->lastConfig; @@ -614,10 +616,12 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { // change isStandBy to normal if (!isDrop) { + char tmpbuf[128]; + snprintf(tmpbuf, sizeof(tmpbuf), "config change3 from %d to %d", oldReplicaNum, newSyncCfg.replicaNum); if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) { - syncNodeBecomeLeader(pSyncNode, "config change3"); + syncNodeBecomeLeader(pSyncNode, tmpbuf); } else { - syncNodeBecomeFollower(pSyncNode, "config change3"); + syncNodeBecomeFollower(pSyncNode, tmpbuf); } } } From fa25fa32e0c56c2f3788493452e9ae77cf8cc755 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Tue, 14 Jun 2022 14:54:11 +0800 Subject: [PATCH 58/78] refactor(sync): add test syncRaftIdChec --- source/libs/sync/test/syncRaftIdCheck.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/libs/sync/test/syncRaftIdCheck.cpp b/source/libs/sync/test/syncRaftIdCheck.cpp index f23c0db82b..90560e91e7 100644 --- a/source/libs/sync/test/syncRaftIdCheck.cpp +++ b/source/libs/sync/test/syncRaftIdCheck.cpp @@ -15,14 +15,14 @@ int main(int argc, char** argv) { char host[128]; uint16_t port; syncUtilU642Addr(u64, host, sizeof(host), &port); - printf("%s:%d \n", host, port); + printf("%lu -> %s:%d \n", u64, host, port); } else if (argc == 3) { uint64_t u64; char* host = argv[1]; uint16_t port = atoi(argv[2]); - syncUtilAddr2U64(host, port); - printf("%lu \n", u64); + u64 = syncUtilAddr2U64(host, port); + printf("%s:%d -> %lu \n", host, port, u64); } else { usage(argv[0]); exit(-1); From 4113472e89c69c1bd55036b03016346305409351 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 14 Jun 2022 15:00:28 +0800 Subject: [PATCH 59/78] add avg function translate partial/merge --- include/libs/function/functionMgt.h | 2 + source/libs/function/inc/builtinsimpl.h | 1 + source/libs/function/src/builtins.c | 53 +++++++++++++++++++++++++ source/libs/function/src/builtinsimpl.c | 2 + 4 files changed, 58 insertions(+) diff --git a/include/libs/function/functionMgt.h b/include/libs/function/functionMgt.h index b5641f2d07..f324d8fed1 100644 --- a/include/libs/function/functionMgt.h +++ b/include/libs/function/functionMgt.h @@ -142,6 +142,8 @@ typedef enum EFunctionType { FUNCTION_TYPE_FIRST_MERGE, FUNCTION_TYPE_LAST_PARTIAL, FUNCTION_TYPE_LAST_MERGE, + FUNCTION_TYPE_AVG_PARTIAL, + FUNCTION_TYPE_AVG_MERGE, // user defined funcion FUNCTION_TYPE_UDF = 10000 diff --git a/source/libs/function/inc/builtinsimpl.h b/source/libs/function/inc/builtinsimpl.h index 7db1396603..2f31d57907 100644 --- a/source/libs/function/inc/builtinsimpl.h +++ b/source/libs/function/inc/builtinsimpl.h @@ -58,6 +58,7 @@ int32_t avgFunction(SqlFunctionCtx* pCtx); int32_t avgFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); int32_t avgInvertFunction(SqlFunctionCtx* pCtx); int32_t avgCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx); +int32_t getAvgInfoSize(); bool getStddevFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool stddevFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo); diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 1c3f274ee9..4fd0db4819 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -155,6 +155,35 @@ static int32_t translateSum(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { return TSDB_CODE_SUCCESS; } +static int32_t translateAvgPartial(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { + if (1 != LIST_LENGTH(pFunc->pParameterList)) { + return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); + } + + uint8_t paraType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type; + if (!IS_NUMERIC_TYPE(paraType) && !IS_NULL_TYPE(paraType)) { + return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); + } + + pFunc->node.resType = (SDataType){.bytes = getAvgInfoSize() + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}; + return TSDB_CODE_SUCCESS; +} + +static int32_t translateAvgMerge(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { + if (1 != LIST_LENGTH(pFunc->pParameterList)) { + return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); + } + + uint8_t paraType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type; + if (TSDB_DATA_TYPE_BINARY != paraType) { + return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); + } + + pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes, .type = TSDB_DATA_TYPE_DOUBLE}; + + return TSDB_CODE_SUCCESS; +} + static int32_t translateWduration(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { // pseudo column do not need to check parameters pFunc->node.resType = (SDataType){.bytes = sizeof(int64_t), .type = TSDB_DATA_TYPE_BIGINT}; @@ -1518,6 +1547,30 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .invertFunc = avgInvertFunction, .combineFunc = avgCombine, }, + { + .name = "_avg_partial", + .type = FUNCTION_TYPE_AVG_PARTIAL, + .classification = FUNC_MGT_AGG_FUNC, + .translateFunc = translateInNumOutDou, + .getEnvFunc = getAvgFuncEnv, + .initFunc = avgFunctionSetup, + .processFunc = avgFunction, + .finalizeFunc = avgFinalize, + .invertFunc = avgInvertFunction, + .combineFunc = avgCombine, + }, + { + .name = "_avg_merge", + .type = FUNCTION_TYPE_AVG_MERGE, + .classification = FUNC_MGT_AGG_FUNC, + .translateFunc = translateInNumOutDou, + .getEnvFunc = getAvgFuncEnv, + .initFunc = avgFunctionSetup, + .processFunc = avgFunction, + .finalizeFunc = avgFinalize, + .invertFunc = avgInvertFunction, + .combineFunc = avgCombine, + }, { .name = "percentile", .type = FUNCTION_TYPE_PERCENTILE, diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 3510a59254..ee47b1aa7f 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -624,6 +624,8 @@ bool getSumFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) { return true; } +int32_t getAvgInfoSize() { return (int32_t)sizeof(SAvgRes); } + bool getAvgFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) { pEnv->calcMemSize = sizeof(SAvgRes); return true; From 5881ef26905b46e36cd2f054a3d837f47948a487 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Tue, 14 Jun 2022 14:28:41 +0800 Subject: [PATCH 60/78] refactor: convert datablock to submit block --- include/common/tdatablock.h | 3 - source/common/src/tdatablock.c | 167 -------------------------- source/dnode/vnode/src/tq/tq.c | 2 +- source/dnode/vnode/src/tq/tqOffset.c | 19 ++- source/dnode/vnode/src/tq/tqSink.c | 169 +++++++++++++++++++++++++++ 5 files changed, 187 insertions(+), 173 deletions(-) diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index af46535c94..709462a744 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -234,9 +234,6 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks char* buildCtbNameByGroupId(const char* stbName, uint64_t groupId); -SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pSchema, bool createTb, int64_t suid, - const char* stbFullName, int32_t vgId); - static FORCE_INLINE int32_t blockGetEncodeSize(const SSDataBlock* pBlock) { return blockDataGetSerialMetaSize(pBlock->info.numOfCols) + blockDataGetSize(pBlock); } diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 4d1f5a34b1..b4ece426b3 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1728,173 +1728,6 @@ char* buildCtbNameByGroupId(const char* stbName, uint64_t groupId) { return rname.childTableName; } -SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, bool createTb, int64_t suid, - const char* stbFullName, int32_t vgId) { - SSubmitReq* ret = NULL; - SArray* tagArray = taosArrayInit(1, sizeof(STagVal)); - if (!tagArray) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; - } - - // cal size - int32_t cap = sizeof(SSubmitReq); - int32_t sz = taosArrayGetSize(pBlocks); - for (int32_t i = 0; i < sz; i++) { - SSDataBlock* pDataBlock = taosArrayGet(pBlocks, i); - int32_t rows = pDataBlock->info.rows; - // TODO min - int32_t rowSize = pDataBlock->info.rowSize; - int32_t maxLen = TD_ROW_MAX_BYTES_FROM_SCHEMA(pTSchema); - int32_t schemaLen = 0; - - if (createTb) { - SVCreateTbReq createTbReq = {0}; - char* cname = buildCtbNameByGroupId(stbFullName, pDataBlock->info.groupId); - createTbReq.name = cname; - createTbReq.flags = 0; - createTbReq.type = TSDB_CHILD_TABLE; - createTbReq.ctb.suid = suid; - - STagVal tagVal = {.cid = pDataBlock->info.numOfCols + 1, - .type = TSDB_DATA_TYPE_UBIGINT, - .pData = (uint8_t*)&pDataBlock->info.groupId, - .nData = sizeof(uint64_t)}; - STag* pTag = NULL; - taosArrayClear(tagArray); - taosArrayPush(tagArray, &tagVal); - tTagNew(tagArray, 1, false, &pTag); - if (pTag == NULL) { - tdDestroySVCreateTbReq(&createTbReq); - taosArrayDestroy(tagArray); - return NULL; - } - createTbReq.ctb.pTag = (uint8_t*)pTag; - - int32_t code; - tEncodeSize(tEncodeSVCreateTbReq, &createTbReq, schemaLen, code); - - tdDestroySVCreateTbReq(&createTbReq); - if (code < 0) { - taosArrayDestroy(tagArray); - return NULL; - } - } - - cap += sizeof(SSubmitBlk) + schemaLen + rows * maxLen; - } - - // assign data - // TODO - ret = taosMemoryCalloc(1, cap + 46); - ret = POINTER_SHIFT(ret, 46); - ret->header.vgId = vgId; - ret->version = htonl(1); - ret->length = sizeof(SSubmitReq); - ret->numOfBlocks = htonl(sz); - - void* submitBlk = POINTER_SHIFT(ret, sizeof(SSubmitReq)); - for (int32_t i = 0; i < sz; i++) { - SSDataBlock* pDataBlock = taosArrayGet(pBlocks, i); - - SSubmitBlk* blkHead = submitBlk; - blkHead->numOfRows = htons(pDataBlock->info.rows); - blkHead->sversion = htonl(pTSchema->version); - // TODO - blkHead->suid = htobe64(suid); - // uid is assigned by vnode - blkHead->uid = 0; - - int32_t rows = pDataBlock->info.rows; - /*int32_t maxLen = TD_ROW_MAX_BYTES_FROM_SCHEMA(pTSchema);*/ - /*blkHead->dataLen = htonl(rows * maxLen);*/ - blkHead->dataLen = 0; - - void* blockData = POINTER_SHIFT(submitBlk, sizeof(SSubmitBlk)); - - int32_t schemaLen = 0; - if (createTb) { - SVCreateTbReq createTbReq = {0}; - char* cname = buildCtbNameByGroupId(stbFullName, pDataBlock->info.groupId); - createTbReq.name = cname; - createTbReq.flags = 0; - createTbReq.type = TSDB_CHILD_TABLE; - createTbReq.ctb.suid = suid; - - STagVal tagVal = {.cid = pDataBlock->info.numOfCols + 1, - .type = TSDB_DATA_TYPE_UBIGINT, - .pData = (uint8_t*)&pDataBlock->info.groupId, - .nData = sizeof(uint64_t)}; - taosArrayClear(tagArray); - taosArrayPush(tagArray, &tagVal); - STag* pTag = NULL; - tTagNew(tagArray, 1, false, &pTag); - if (pTag == NULL) { - tdDestroySVCreateTbReq(&createTbReq); - taosArrayDestroy(tagArray); - taosMemoryFreeClear(ret); - return NULL; - } - createTbReq.ctb.pTag = (uint8_t*)pTag; - - int32_t code; - tEncodeSize(tEncodeSVCreateTbReq, &createTbReq, schemaLen, code); - if (code < 0) { - tdDestroySVCreateTbReq(&createTbReq); - taosArrayDestroy(tagArray); - taosMemoryFreeClear(ret); - return NULL; - } - - SEncoder encoder = {0}; - tEncoderInit(&encoder, blockData, schemaLen); - code = tEncodeSVCreateTbReq(&encoder, &createTbReq); - tEncoderClear(&encoder); - tdDestroySVCreateTbReq(&createTbReq); - - if (code < 0) { - taosArrayDestroy(tagArray); - taosMemoryFreeClear(ret); - return NULL; - } - } - blkHead->schemaLen = htonl(schemaLen); - - STSRow* rowData = POINTER_SHIFT(blockData, schemaLen); - - for (int32_t j = 0; j < rows; j++) { - SRowBuilder rb = {0}; - tdSRowInit(&rb, pTSchema->version); - tdSRowSetTpInfo(&rb, pTSchema->numOfCols, pTSchema->flen); - tdSRowResetBuf(&rb, rowData); - - for (int32_t k = 0; k < pTSchema->numOfCols; k++) { - const STColumn* pColumn = &pTSchema->columns[k]; - SColumnInfoData* pColData = taosArrayGet(pDataBlock->pDataBlock, k); - if (colDataIsNull_s(pColData, j)) { - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NONE, NULL, false, pColumn->offset, k); - } else { - void* data = colDataGetData(pColData, j); - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, data, true, pColumn->offset, k); - } - } - int32_t rowLen = TD_ROW_LEN(rowData); - rowData = POINTER_SHIFT(rowData, rowLen); - blkHead->dataLen += rowLen; - } - int32_t dataLen = blkHead->dataLen; - blkHead->dataLen = htonl(dataLen); - - ret->length += sizeof(SSubmitBlk) + schemaLen + dataLen; - blkHead = POINTER_SHIFT(blkHead, schemaLen + dataLen); - /*submitBlk = blkHead;*/ - } - - ret->length = htonl(ret->length); - taosArrayDestroy(tagArray); - return ret; -} - void blockCompressEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_t numOfCols, int8_t needCompress) { // todo extract method diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 22685c1e19..9f34ae39c0 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -47,7 +47,7 @@ void tqCleanUp() { } STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal) { - STQ* pTq = taosMemoryMalloc(sizeof(STQ)); + STQ* pTq = taosMemoryCalloc(1, sizeof(STQ)); if (pTq == NULL) { terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; return NULL; diff --git a/source/dnode/vnode/src/tq/tqOffset.c b/source/dnode/vnode/src/tq/tqOffset.c index 2db49d7cf5..41444b7288 100644 --- a/source/dnode/vnode/src/tq/tqOffset.c +++ b/source/dnode/vnode/src/tq/tqOffset.c @@ -17,21 +17,33 @@ #include "tq.h" struct STqOffsetStore { + char* fname; STQ* pTq; SHashObj* pHash; // SHashObj }; +static char* buildFileName(const char* path) { + int32_t len = strlen(path); + char* fname = taosMemoryCalloc(1, len + 20); + snprintf(fname, len + 20, "%s/offset", path); + return fname; +} + STqOffsetStore* tqOffsetOpen(STQ* pTq) { STqOffsetStore* pStore = taosMemoryCalloc(1, sizeof(STqOffsetStore)); if (pStore == NULL) { return NULL; } + pStore->pTq = pTq; + pTq->pOffsetStore = pStore; + pStore->pHash = taosHashInit(64, MurmurHash3_32, true, HASH_NO_LOCK); if (pStore->pHash == NULL) { if (pStore->pHash) taosHashCleanup(pStore->pHash); return NULL; } - TdFilePtr pFile = taosOpenFile(pStore->pTq->path, TD_FILE_READ); + char* fname = buildFileName(pStore->pTq->path); + TdFilePtr pFile = taosOpenFile(fname, TD_FILE_READ); if (pFile != NULL) { STqOffsetHead head = {0}; int64_t code; @@ -65,6 +77,7 @@ STqOffsetStore* tqOffsetOpen(STQ* pTq) { } taosCloseFile(&pFile); + taosMemoryFree(fname); } return pStore; } @@ -85,7 +98,8 @@ int32_t tqOffsetWrite(STqOffsetStore* pStore, const STqOffset* pOffset) { int32_t tqOffsetSnapshot(STqOffsetStore* pStore) { // open file // TODO file name should be with a version - TdFilePtr pFile = taosOpenFile(pStore->pTq->path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND); + char* fname = buildFileName(pStore->pTq->path); + TdFilePtr pFile = taosOpenFile(fname, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND); if (pFile == NULL) { ASSERT(0); return -1; @@ -124,5 +138,6 @@ int32_t tqOffsetSnapshot(STqOffsetStore* pStore) { } // close and rename file taosCloseFile(&pFile); + taosMemoryFree(fname); return 0; } diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index 5c0bf971fb..391a008440 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -15,6 +15,175 @@ #include "tq.h" +static SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pSchema, bool createTb, int64_t suid, + const char* stbFullName, int32_t vgId); + +static SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, bool createTb, int64_t suid, + const char* stbFullName, int32_t vgId) { + SSubmitReq* ret = NULL; + SArray* tagArray = taosArrayInit(1, sizeof(STagVal)); + if (!tagArray) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + // cal size + int32_t cap = sizeof(SSubmitReq); + int32_t sz = taosArrayGetSize(pBlocks); + for (int32_t i = 0; i < sz; i++) { + SSDataBlock* pDataBlock = taosArrayGet(pBlocks, i); + int32_t rows = pDataBlock->info.rows; + // TODO min + int32_t rowSize = pDataBlock->info.rowSize; + int32_t maxLen = TD_ROW_MAX_BYTES_FROM_SCHEMA(pTSchema); + int32_t schemaLen = 0; + + if (createTb) { + SVCreateTbReq createTbReq = {0}; + char* cname = buildCtbNameByGroupId(stbFullName, pDataBlock->info.groupId); + createTbReq.name = cname; + createTbReq.flags = 0; + createTbReq.type = TSDB_CHILD_TABLE; + createTbReq.ctb.suid = suid; + + STagVal tagVal = {.cid = pDataBlock->info.numOfCols + 1, + .type = TSDB_DATA_TYPE_UBIGINT, + .pData = (uint8_t*)&pDataBlock->info.groupId, + .nData = sizeof(uint64_t)}; + STag* pTag = NULL; + taosArrayClear(tagArray); + taosArrayPush(tagArray, &tagVal); + tTagNew(tagArray, 1, false, &pTag); + if (pTag == NULL) { + tdDestroySVCreateTbReq(&createTbReq); + taosArrayDestroy(tagArray); + return NULL; + } + createTbReq.ctb.pTag = (uint8_t*)pTag; + + int32_t code; + tEncodeSize(tEncodeSVCreateTbReq, &createTbReq, schemaLen, code); + + tdDestroySVCreateTbReq(&createTbReq); + if (code < 0) { + taosArrayDestroy(tagArray); + return NULL; + } + } + + cap += sizeof(SSubmitBlk) + schemaLen + rows * maxLen; + } + + // assign data + // TODO + ret = rpcMallocCont(cap); + ret->header.vgId = vgId; + ret->version = htonl(1); + ret->length = sizeof(SSubmitReq); + ret->numOfBlocks = htonl(sz); + + void* submitBlk = POINTER_SHIFT(ret, sizeof(SSubmitReq)); + for (int32_t i = 0; i < sz; i++) { + SSDataBlock* pDataBlock = taosArrayGet(pBlocks, i); + + SSubmitBlk* blkHead = submitBlk; + blkHead->numOfRows = htons(pDataBlock->info.rows); + blkHead->sversion = htonl(pTSchema->version); + // TODO + blkHead->suid = htobe64(suid); + // uid is assigned by vnode + blkHead->uid = 0; + + int32_t rows = pDataBlock->info.rows; + /*int32_t maxLen = TD_ROW_MAX_BYTES_FROM_SCHEMA(pTSchema);*/ + /*blkHead->dataLen = htonl(rows * maxLen);*/ + blkHead->dataLen = 0; + + void* blockData = POINTER_SHIFT(submitBlk, sizeof(SSubmitBlk)); + + int32_t schemaLen = 0; + if (createTb) { + SVCreateTbReq createTbReq = {0}; + char* cname = buildCtbNameByGroupId(stbFullName, pDataBlock->info.groupId); + createTbReq.name = cname; + createTbReq.flags = 0; + createTbReq.type = TSDB_CHILD_TABLE; + createTbReq.ctb.suid = suid; + + STagVal tagVal = {.cid = pDataBlock->info.numOfCols + 1, + .type = TSDB_DATA_TYPE_UBIGINT, + .pData = (uint8_t*)&pDataBlock->info.groupId, + .nData = sizeof(uint64_t)}; + taosArrayClear(tagArray); + taosArrayPush(tagArray, &tagVal); + STag* pTag = NULL; + tTagNew(tagArray, 1, false, &pTag); + if (pTag == NULL) { + tdDestroySVCreateTbReq(&createTbReq); + taosArrayDestroy(tagArray); + taosMemoryFreeClear(ret); + return NULL; + } + createTbReq.ctb.pTag = (uint8_t*)pTag; + + int32_t code; + tEncodeSize(tEncodeSVCreateTbReq, &createTbReq, schemaLen, code); + if (code < 0) { + tdDestroySVCreateTbReq(&createTbReq); + taosArrayDestroy(tagArray); + taosMemoryFreeClear(ret); + return NULL; + } + + SEncoder encoder = {0}; + tEncoderInit(&encoder, blockData, schemaLen); + code = tEncodeSVCreateTbReq(&encoder, &createTbReq); + tEncoderClear(&encoder); + tdDestroySVCreateTbReq(&createTbReq); + + if (code < 0) { + taosArrayDestroy(tagArray); + taosMemoryFreeClear(ret); + return NULL; + } + } + blkHead->schemaLen = htonl(schemaLen); + + STSRow* rowData = POINTER_SHIFT(blockData, schemaLen); + + for (int32_t j = 0; j < rows; j++) { + SRowBuilder rb = {0}; + tdSRowInit(&rb, pTSchema->version); + tdSRowSetTpInfo(&rb, pTSchema->numOfCols, pTSchema->flen); + tdSRowResetBuf(&rb, rowData); + + for (int32_t k = 0; k < pTSchema->numOfCols; k++) { + const STColumn* pColumn = &pTSchema->columns[k]; + SColumnInfoData* pColData = taosArrayGet(pDataBlock->pDataBlock, k); + if (colDataIsNull_s(pColData, j)) { + tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NONE, NULL, false, pColumn->offset, k); + } else { + void* data = colDataGetData(pColData, j); + tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, data, true, pColumn->offset, k); + } + } + int32_t rowLen = TD_ROW_LEN(rowData); + rowData = POINTER_SHIFT(rowData, rowLen); + blkHead->dataLen += rowLen; + } + int32_t dataLen = blkHead->dataLen; + blkHead->dataLen = htonl(dataLen); + + ret->length += sizeof(SSubmitBlk) + schemaLen + dataLen; + blkHead = POINTER_SHIFT(blkHead, schemaLen + dataLen); + /*submitBlk = blkHead;*/ + } + + ret->length = htonl(ret->length); + taosArrayDestroy(tagArray); + return ret; +} + void tqTableSink(SStreamTask* pTask, void* vnode, int64_t ver, void* data) { const SArray* pRes = (const SArray*)data; SVnode* pVnode = (SVnode*)vnode; From e82289542df8f1cb0c546b3e8db229f0efedaf89 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 14 Jun 2022 15:15:37 +0800 Subject: [PATCH 61/78] avg function add partial/merge --- source/libs/function/inc/builtinsimpl.h | 2 + source/libs/function/src/builtinsimpl.c | 52 +++++++++++++++++++++++++ 2 files changed, 54 insertions(+) diff --git a/source/libs/function/inc/builtinsimpl.h b/source/libs/function/inc/builtinsimpl.h index 2f31d57907..1bb30934c6 100644 --- a/source/libs/function/inc/builtinsimpl.h +++ b/source/libs/function/inc/builtinsimpl.h @@ -55,7 +55,9 @@ int32_t maxCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx); bool getAvgFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool avgFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo); int32_t avgFunction(SqlFunctionCtx* pCtx); +int32_t avgFunctionMerge(SqlFunctionCtx* pCtx); int32_t avgFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); +int32_t avgPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); int32_t avgInvertFunction(SqlFunctionCtx* pCtx); int32_t avgCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx); int32_t getAvgInfoSize(); diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index ee47b1aa7f..fe9b4e1c3a 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -51,6 +51,7 @@ typedef struct SAvgRes { double result; SSumRes sum; int64_t count; + int16_t type; // store the original input type, used in merge function } SAvgRes; typedef struct STuplePos { @@ -649,6 +650,7 @@ int32_t avgFunction(SqlFunctionCtx* pCtx) { int32_t type = pInput->pData[0]->info.type; SAvgRes* pAvgRes = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); + pAvgRes->type = type; // computing based on the true data block SColumnInfoData* pCol = pInput->pData[0]; @@ -771,6 +773,38 @@ _avg_over: return TSDB_CODE_SUCCESS; } +static void avgTransferInfo(SAvgRes* pInput, SAvgRes* pOutput) { + int16_t type = pInput->type; + if (IS_INTEGER_TYPE(type)) { + pOutput->sum.isum += pInput->sum.isum; + } else { + pOutput->sum.dsum += pInput->sum.dsum; + } + + pOutput->count += pInput->count; + + return; +} + +int32_t avgFunctionMerge(SqlFunctionCtx* pCtx) { + SInputColumnInfoData* pInput = &pCtx->input; + SColumnInfoData* pCol = pInput->pData[0]; + ASSERT(pCol->info.type == TSDB_DATA_TYPE_BINARY); + + SAvgRes* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); + SAvgRes* pInputInfo; + + int32_t start = pInput->startRowIndex; + char* data = colDataGetData(pCol, start); + pInputInfo = (SAvgRes*)varDataVal(data); + + avgTransferInfo(pInputInfo, pInfo); + + SET_VAL(GET_RES_INFO(pCtx), 1, 1); + + return TSDB_CODE_SUCCESS; +} + #define LIST_AVG_N(sumT, T) \ do { \ T* plist = (T*)pCol->pData; \ @@ -872,6 +906,24 @@ int32_t avgFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { return functionFinalize(pCtx, pBlock); } +int32_t avgPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { + SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); + SAvgRes* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); + int32_t resultBytes = getAvgInfoSize(); + char* res = taosMemoryCalloc(resultBytes + VARSTR_HEADER_SIZE, sizeof(char)); + + memcpy(varDataVal(res), pInfo, resultBytes); + varDataSetLen(res, resultBytes); + + int32_t slotId = pCtx->pExpr->base.resSchema.slotId; + SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); + + colDataAppend(pCol, pBlock->info.rows, res, false); + + taosMemoryFree(res); + return pResInfo->numOfRes; +} + EFuncDataRequired statisDataRequired(SFunctionNode* pFunc, STimeWindow* pTimeWindow) { return FUNC_DATA_REQUIRED_STATIS_LOAD; } From 79d9c04a4c6118069c3065434d7bc71815f73685 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Tue, 14 Jun 2022 15:22:39 +0800 Subject: [PATCH 62/78] fix: check error code --- source/dnode/vnode/src/tq/tqOffset.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tq/tqOffset.c b/source/dnode/vnode/src/tq/tqOffset.c index 41444b7288..8d6cb28065 100644 --- a/source/dnode/vnode/src/tq/tqOffset.c +++ b/source/dnode/vnode/src/tq/tqOffset.c @@ -50,7 +50,7 @@ STqOffsetStore* tqOffsetOpen(STQ* pTq) { while (1) { if ((code = taosReadFile(pFile, &head, sizeof(STqOffsetHead))) != sizeof(STqOffsetHead)) { - if (code < 0) { + if (code == 0) { break; } else { ASSERT(0); From 18c0e0ac7411e223741f60bfda569bb029884b0e Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Tue, 14 Jun 2022 15:31:47 +0800 Subject: [PATCH 63/78] fix(query): cast --- source/client/src/clientImpl.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index c6cdd42584..6c3132a3ca 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1415,7 +1415,7 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32 int32_t bytes = *(int32_t*)p; p += sizeof(int32_t); - ASSERT(type == pFields[i].type && bytes == pFields[i].bytes); + /*ASSERT(type == pFields[i].type && bytes == pFields[i].bytes);*/ } int32_t* colLength = (int32_t*)p; From 4ccbc4d85fe1caddfe2c171223afbb7fb06f69b0 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 14 Jun 2022 15:34:55 +0800 Subject: [PATCH 64/78] enable avg function partial/merge --- source/libs/function/src/builtins.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 4fd0db4819..4dae296f05 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -1546,16 +1546,18 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .finalizeFunc = avgFinalize, .invertFunc = avgInvertFunction, .combineFunc = avgCombine, + .pPartialFunc = "_avg_partial", + .pMergeFunc = "_avg_merge" }, { .name = "_avg_partial", .type = FUNCTION_TYPE_AVG_PARTIAL, .classification = FUNC_MGT_AGG_FUNC, - .translateFunc = translateInNumOutDou, + .translateFunc = translateAvgPartial, .getEnvFunc = getAvgFuncEnv, .initFunc = avgFunctionSetup, .processFunc = avgFunction, - .finalizeFunc = avgFinalize, + .finalizeFunc = avgPartialFinalize, .invertFunc = avgInvertFunction, .combineFunc = avgCombine, }, @@ -1563,10 +1565,10 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .name = "_avg_merge", .type = FUNCTION_TYPE_AVG_MERGE, .classification = FUNC_MGT_AGG_FUNC, - .translateFunc = translateInNumOutDou, + .translateFunc = translateAvgMerge, .getEnvFunc = getAvgFuncEnv, .initFunc = avgFunctionSetup, - .processFunc = avgFunction, + .processFunc = avgFunctionMerge, .finalizeFunc = avgFinalize, .invertFunc = avgInvertFunction, .combineFunc = avgCombine, From 1676bfe5cf94678a83646b73b019700fa4a0bf47 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Tue, 14 Jun 2022 15:43:56 +0800 Subject: [PATCH 65/78] fix: some problems of parser --- include/libs/parser/parser.h | 1 + include/util/taoserror.h | 1 + source/client/src/clientMain.c | 4 + source/libs/parser/inc/sql.y | 2 +- source/libs/parser/src/parAstCreater.c | 12 +- source/libs/parser/src/parTranslater.c | 159 ++- source/libs/parser/src/parUtil.c | 2 + source/libs/parser/src/parser.c | 2 + source/libs/parser/src/sql.c | 1209 +++++++++--------- source/libs/parser/test/mockCatalog.cpp | 30 +- source/libs/parser/test/parInitialATest.cpp | 18 +- source/libs/parser/test/parInitialCTest.cpp | 4 +- source/libs/parser/test/parInsertTest.cpp | 8 +- source/libs/parser/test/parSelectTest.cpp | 14 +- source/libs/parser/test/parTestUtil.cpp | 6 +- source/libs/planner/src/planSpliter.c | 3 + source/libs/planner/test/planGroupByTest.cpp | 2 + tests/script/jenkins/basic.txt | 24 +- 18 files changed, 804 insertions(+), 697 deletions(-) diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index 76a290364e..e45e5bd160 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -65,6 +65,7 @@ void qDestroyQuery(SQuery* pQueryNode); int32_t qExtractResultSchema(const SNode* pRoot, int32_t* numOfCols, SSchema** pSchema); int32_t qSetSTableIdForRsma(SNode* pStmt, int64_t uid); +void qCleanupKeywordsTable(); int32_t qBuildStmtOutput(SQuery* pQuery, SHashObj* pVgHash, SHashObj* pBlockHash); int32_t qResetStmtDataBlock(void* block, bool keepBuf); diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 03308e395f..1e515b3cbd 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -653,6 +653,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_PAR_INVALID_DELETE_WHERE TAOS_DEF_ERROR_CODE(0, 0x2655) #define TSDB_CODE_PAR_INVALID_REDISTRIBUTE_VG TAOS_DEF_ERROR_CODE(0, 0x2656) #define TSDB_CODE_PAR_FILL_NOT_ALLOWED_FUNC TAOS_DEF_ERROR_CODE(0, 0x2657) +#define TSDB_CODE_PAR_INVALID_WINDOW_PC TAOS_DEF_ERROR_CODE(0, 0x2658) //planner #define TSDB_CODE_PLAN_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x2700) diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 83b481658e..40a58b4dc5 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -25,6 +25,7 @@ #include "tref.h" #include "trpc.h" #include "version.h" +#include "functionMgt.h" #define TSC_VAR_NOT_RELEASE 1 #define TSC_VAR_RELEASED 0 @@ -61,6 +62,9 @@ void taos_cleanup(void) { cleanupTaskQueue(); + fmFuncMgtDestroy(); + qCleanupKeywordsTable(); + id = clientConnRefPool; clientConnRefPool = -1; taosCloseRef(id); diff --git a/source/libs/parser/inc/sql.y b/source/libs/parser/inc/sql.y index 83ad41aac0..8097695823 100644 --- a/source/libs/parser/inc/sql.y +++ b/source/libs/parser/inc/sql.y @@ -255,7 +255,7 @@ multi_create_clause(A) ::= multi_create_clause(B) create_subtable_clause(C). create_subtable_clause(A) ::= not_exists_opt(B) full_table_name(C) USING full_table_name(D) - specific_tags_opt(E) TAGS NK_LP literal_list(F) NK_RP table_options(G). { A = createCreateSubTableClause(pCxt, B, C, D, E, F, G); } + specific_tags_opt(E) TAGS NK_LP expression_list(F) NK_RP table_options(G). { A = createCreateSubTableClause(pCxt, B, C, D, E, F, G); } %type multi_drop_clause { SNodeList* } %destructor multi_drop_clause { nodesDestroyList($$); } diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index ca066c7056..daeea1ed68 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -230,9 +230,13 @@ SNode* releaseRawExprNode(SAstCreateContext* pCxt, SNode* pNode) { SRawExprNode* pRawExpr = (SRawExprNode*)pNode; SNode* pExpr = pRawExpr->pNode; if (nodesIsExprNode(pExpr)) { - int32_t len = TMIN(sizeof(((SExprNode*)pExpr)->aliasName) - 1, pRawExpr->n); - strncpy(((SExprNode*)pExpr)->aliasName, pRawExpr->p, len); - ((SExprNode*)pExpr)->aliasName[len] = '\0'; + if (QUERY_NODE_COLUMN == nodeType(pExpr)) { + strcpy(((SExprNode*)pExpr)->aliasName, ((SColumnNode*)pExpr)->colName); + } else { + int32_t len = TMIN(sizeof(((SExprNode*)pExpr)->aliasName) - 1, pRawExpr->n); + strncpy(((SExprNode*)pExpr)->aliasName, pRawExpr->p, len); + ((SExprNode*)pExpr)->aliasName[len] = '\0'; + } } taosMemoryFreeClear(pNode); return pExpr; @@ -801,7 +805,7 @@ SNode* setDatabaseOption(SAstCreateContext* pCxt, SNode* pOptions, EDatabaseOpti ((SDatabaseOptions*)pOptions)->pRetentions = pVal; break; case DB_OPTION_SCHEMALESS: -// ((SDatabaseOptions*)pOptions)->schemaless = taosStr2Int8(((SToken*)pVal)->z, NULL, 10); + // ((SDatabaseOptions*)pOptions)->schemaless = taosStr2Int8(((SToken*)pVal)->z, NULL, 10); ((SDatabaseOptions*)pOptions)->schemaless = 1; break; default: diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index fcf2f57dec..365704f988 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -537,13 +537,15 @@ static int32_t findAndSetColumn(STranslateContext* pCxt, SColumnNode** pColRef, SNode* pNode; FOREACH(pNode, pProjectList) { SExprNode* pExpr = (SExprNode*)pNode; - if (0 == strcmp(pCol->colName, pExpr->aliasName) || - (isPrimaryKey((STempTableNode*)pTable, pNode) && isInternalPrimaryKey(pCol))) { + if (0 == strcmp(pCol->colName, pExpr->aliasName)) { if (*pFound) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_AMBIGUOUS_COLUMN, pCol->colName); } setColumnInfoByExpr(pTable, pExpr, pColRef); *pFound = true; + } else if (isPrimaryKey((STempTableNode*)pTable, pNode) && isInternalPrimaryKey(pCol)) { + setColumnInfoByExpr(pTable, pExpr, pColRef); + *pFound = true; } } } @@ -662,9 +664,24 @@ static int32_t parseTimeFromValueNode(STranslateContext* pCxt, SValueNode* pVal) } char* pEnd = NULL; pVal->datum.i = taosStr2Int64(pVal->literal, &pEnd, 10); - return (NULL != pEnd && '\0' == *pEnd) ? TSDB_CODE_SUCCESS : TSDB_CODE_FAILED; + return (NULL != pEnd && '\0' == *pEnd) ? TSDB_CODE_SUCCESS : TSDB_CODE_PAR_WRONG_VALUE_TYPE; } else { - return TSDB_CODE_FAILED; + return TSDB_CODE_PAR_WRONG_VALUE_TYPE; + } +} + +static int32_t parseBoolFromValueNode(STranslateContext* pCxt, SValueNode* pVal) { + if (IS_VAR_DATA_TYPE(pVal->node.resType.type) || TSDB_DATA_TYPE_BOOL == pVal->node.resType.type) { + pVal->datum.b = (0 == strcasecmp(pVal->literal, "true")); + return TSDB_CODE_SUCCESS; + } else if (IS_INTEGER_TYPE(pVal->node.resType.type)) { + pVal->datum.b = (0 != taosStr2Int64(pVal->literal, NULL, 10)); + return TSDB_CODE_SUCCESS; + } else if (IS_FLOAT_TYPE(pVal->node.resType.type)) { + pVal->datum.b = (0 != taosStr2Double(pVal->literal, NULL)); + return TSDB_CODE_SUCCESS; + } else { + return TSDB_CODE_PAR_WRONG_VALUE_TYPE; } } @@ -685,7 +702,9 @@ static EDealRes translateValueImpl(STranslateContext* pCxt, SValueNode* pVal, SD case TSDB_DATA_TYPE_NULL: break; case TSDB_DATA_TYPE_BOOL: - pVal->datum.b = (0 == strcasecmp(pVal->literal, "true")); + if (TSDB_CODE_SUCCESS != parseBoolFromValueNode(pCxt, pVal)) { + return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, pVal->literal); + } *(bool*)&pVal->typeData = pVal->datum.b; break; case TSDB_DATA_TYPE_TINYINT: { @@ -1068,6 +1087,16 @@ static int32_t translateForbidFillFunc(STranslateContext* pCxt, SFunctionNode* p return TSDB_CODE_SUCCESS; } +static int32_t translateWindowPseudoColumnFunc(STranslateContext* pCxt, SFunctionNode* pFunc) { + if (!fmIsWindowPseudoColumnFunc(pFunc->funcId)) { + return TSDB_CODE_SUCCESS; + } + if (NULL == pCxt->pCurrSelectStmt->pWindow) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_WINDOW_PC); + } + return TSDB_CODE_SUCCESS; +} + static void setFuncClassification(SSelectStmt* pSelect, SFunctionNode* pFunc) { if (NULL != pSelect) { pSelect->hasAggFuncs = pSelect->hasAggFuncs ? true : fmIsAggFunc(pFunc->funcId); @@ -1097,6 +1126,9 @@ static EDealRes translateFunction(STranslateContext* pCxt, SFunctionNode* pFunc) if (TSDB_CODE_SUCCESS == pCxt->errCode) { pCxt->errCode = translateForbidFillFunc(pCxt, pFunc); } + if (TSDB_CODE_SUCCESS == pCxt->errCode) { + pCxt->errCode = translateWindowPseudoColumnFunc(pCxt, pFunc); + } if (TSDB_CODE_SUCCESS == pCxt->errCode) { setFuncClassification(pCxt->pCurrSelectStmt, pFunc); } @@ -3106,7 +3138,11 @@ static int32_t translateDropSuperTable(STranslateContext* pCxt, SDropSuperTableS pStmt->ignoreNotExists); } -static int32_t setAlterTableField(SAlterTableStmt* pStmt, SMAlterStbReq* pAlterReq) { +static int32_t buildAlterSuperTableReq(STranslateContext* pCxt, SAlterTableStmt* pStmt, SMAlterStbReq* pAlterReq) { + SName tableName; + tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &tableName), pAlterReq->name); + pAlterReq->alterType = pStmt->alterType; + if (TSDB_ALTER_TABLE_UPDATE_OPTIONS == pStmt->alterType) { pAlterReq->ttl = pStmt->pOptions->ttl; if ('\0' != pStmt->pOptions->comment[0]) { @@ -3154,15 +3190,45 @@ static int32_t setAlterTableField(SAlterTableStmt* pStmt, SMAlterStbReq* pAlterR return TSDB_CODE_SUCCESS; } -static int32_t translateAlterTable(STranslateContext* pCxt, SAlterTableStmt* pStmt) { - SMAlterStbReq alterReq = {0}; - SName tableName; - tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &tableName), alterReq.name); - alterReq.alterType = pStmt->alterType; +static SSchema* getColSchema(STableMeta* pTableMeta, const char* pTagName) { + int32_t numOfFields = getNumOfTags(pTableMeta) + getNumOfColumns(pTableMeta); + for (int32_t i = 0; i < numOfFields; ++i) { + SSchema* pTagSchema = pTableMeta->schema + i; + if (0 == strcmp(pTagName, pTagSchema->name)) { + return pTagSchema; + } + } + return NULL; +} + +static int32_t checkAlterSuperTable(STranslateContext* pCxt, SAlterTableStmt* pStmt) { if (TSDB_ALTER_TABLE_UPDATE_TAG_VAL == pStmt->alterType || TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME == pStmt->alterType) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_ALTER_TABLE); } - int32_t code = setAlterTableField(pStmt, &alterReq); + if (TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES == pStmt->alterType || + TSDB_ALTER_TABLE_UPDATE_TAG_BYTES == pStmt->alterType) { + STableMeta* pTableMeta = NULL; + int32_t code = getTableMeta(pCxt, pStmt->dbName, pStmt->tableName, &pTableMeta); + if (TSDB_CODE_SUCCESS == code) { + SSchema* pSchema = getColSchema(pTableMeta, pStmt->colName); + if (NULL == pSchema) { + code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLUMN, pStmt->colName); + } else if (!IS_VAR_DATA_TYPE(pSchema->type) || pSchema->type != pStmt->dataType.type || + pSchema->bytes >= pStmt->dataType.bytes) { + code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_MODIFY_COL); + } + } + return code; + } + return TSDB_CODE_SUCCESS; +} + +static int32_t translateAlterSuperTable(STranslateContext* pCxt, SAlterTableStmt* pStmt) { + SMAlterStbReq alterReq = {0}; + int32_t code = checkAlterSuperTable(pCxt, pStmt); + if (TSDB_CODE_SUCCESS == code) { + code = buildAlterSuperTableReq(pCxt, pStmt, &alterReq); + } if (TSDB_CODE_SUCCESS == code) { code = buildCmdMsg(pCxt, TDMT_MND_ALTER_STB, (FSerializeFunc)tSerializeSMAlterStbReq, &alterReq); } @@ -3835,7 +3901,7 @@ static int32_t translateQuery(STranslateContext* pCxt, SNode* pNode) { code = translateDropSuperTable(pCxt, (SDropSuperTableStmt*)pNode); break; case QUERY_NODE_ALTER_TABLE_STMT: - code = translateAlterTable(pCxt, (SAlterTableStmt*)pNode); + code = translateAlterSuperTable(pCxt, (SAlterTableStmt*)pNode); break; case QUERY_NODE_CREATE_USER_STMT: code = translateCreateUser(pCxt, (SCreateUserStmt*)pNode); @@ -4445,10 +4511,34 @@ static int32_t translateTagVal(STranslateContext* pCxt, uint8_t precision, SSche ? pCxt->errCode : TSDB_CODE_SUCCESS); } else { - return TSDB_CODE_FAILED; + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)pNode)->aliasName); } } +static int32_t buildJsonTagVal(STranslateContext* pCxt, SSchema* pTagSchema, SValueNode* pVal, SArray* pTagArray, + STag** ppTag) { + if (pVal->literal && strlen(pVal->literal) > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { + return buildSyntaxErrMsg(&pCxt->msgBuf, "json string too long than 4095", pVal->literal); + } + + return parseJsontoTagData(pVal->literal, pTagArray, ppTag, &pCxt->msgBuf); +} + +static int32_t buildNormalTagVal(STranslateContext* pCxt, SSchema* pTagSchema, SValueNode* pVal, SArray* pTagArray) { + if (pVal->node.resType.type != TSDB_DATA_TYPE_NULL) { + void* nodeVal = nodesGetValueFromNode(pVal); + STagVal val = {.cid = pTagSchema->colId, .type = pTagSchema->type}; + if (IS_VAR_DATA_TYPE(pTagSchema->type)) { + val.pData = varDataVal(nodeVal); + val.nData = varDataLen(nodeVal); + } else { + memcpy(&val.i64, nodeVal, pTagSchema->bytes); + } + taosArrayPush(pTagArray, &val); + } + return TSDB_CODE_SUCCESS; +} + static int32_t buildKVRowForBindTags(STranslateContext* pCxt, SCreateSubTableClause* pStmt, STableMeta* pSuperTableMeta, STag** ppTag) { int32_t numOfTags = getNumOfTags(pSuperTableMeta); @@ -4458,11 +4548,10 @@ static int32_t buildKVRowForBindTags(STranslateContext* pCxt, SCreateSubTableCla } SArray* pTagArray = taosArrayInit(LIST_LENGTH(pStmt->pValsOfTags), sizeof(STagVal)); - if (!pTagArray) { - return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_TSC_OUT_OF_MEMORY); + if (NULL == pTagArray) { + return TSDB_CODE_OUT_OF_MEMORY; } int32_t code = TSDB_CODE_SUCCESS; - int16_t nTags = 0, nBufPos = 0; SSchema* pTagSchema = getTableTagSchema(pSuperTableMeta); SNode * pTag = NULL, *pNode = NULL; bool isJson = false; @@ -4490,27 +4579,12 @@ static int32_t buildKVRowForBindTags(STranslateContext* pCxt, SCreateSubTableCla } else { REPLACE_LIST2_NODE(pVal); } - if (pTagSchema->type == TSDB_DATA_TYPE_JSON) { - if (pVal->literal && strlen(pVal->literal) > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { - code = buildSyntaxErrMsg(&pCxt->msgBuf, "json string too long than 4095", pVal->literal); - goto end; - } + if (pSchema->type == TSDB_DATA_TYPE_JSON) { isJson = true; - code = parseJsontoTagData(pVal->literal, pTagArray, ppTag, &pCxt->msgBuf); - if (code != TSDB_CODE_SUCCESS) { - goto end; - } + code = buildJsonTagVal(pCxt, pSchema, pVal, pTagArray, ppTag); } else if (pVal->node.resType.type != TSDB_DATA_TYPE_NULL) { - void* nodeVal = nodesGetValueFromNode(pVal); - STagVal val = {.cid = pTagSchema->colId, .type = pTagSchema->type}; - if (IS_VAR_DATA_TYPE(pTagSchema->type)) { - val.pData = varDataVal(nodeVal); - val.nData = varDataLen(nodeVal); - } else { - memcpy(&val.i64, nodeVal, pTagSchema->bytes); - } - taosArrayPush(pTagArray, &val); + code = buildNormalTagVal(pCxt, pSchema, pVal, pTagArray); } } @@ -4825,17 +4899,6 @@ static int32_t rewriteDropTable(STranslateContext* pCxt, SQuery* pQuery) { return rewriteToVnodeModifyOpStmt(pQuery, pBufArray); } -static SSchema* getColSchema(STableMeta* pTableMeta, const char* pTagName) { - int32_t numOfFields = getNumOfTags(pTableMeta) + getNumOfColumns(pTableMeta); - for (int32_t i = 0; i < numOfFields; ++i) { - SSchema* pTagSchema = pTableMeta->schema + i; - if (0 == strcmp(pTagName, pTagSchema->name)) { - return pTagSchema; - } - } - return NULL; -} - static int32_t buildUpdateTagValReq(STranslateContext* pCxt, SAlterTableStmt* pStmt, STableMeta* pTableMeta, SVAlterTbReq* pReq) { SSchema* pSchema = getColSchema(pTableMeta, pStmt->colName); @@ -4853,6 +4916,10 @@ static int32_t buildUpdateTagValReq(STranslateContext* pCxt, SAlterTableStmt* pS return pCxt->errCode; } + if (IS_VAR_DATA_TYPE(pSchema->type) && strlen(pStmt->pVal->literal) > pSchema->bytes) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_WRONG_VALUE_TYPE, pStmt->pVal->literal); + } + pReq->isNull = (TSDB_DATA_TYPE_NULL == pStmt->pVal->node.resType.type); if (pStmt->pVal->node.resType.type == TSDB_DATA_TYPE_JSON) { if (pStmt->pVal->literal && diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index c3ff4c63a4..42e887bb9d 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -186,6 +186,8 @@ static char* getSyntaxErrFormat(int32_t errCode) { return "The REDISTRIBUTE VGROUP statement only support 1 to 3 dnodes"; case TSDB_CODE_PAR_FILL_NOT_ALLOWED_FUNC: return "%s function not allowed in fill query"; + case TSDB_CODE_PAR_INVALID_WINDOW_PC: + return "_WSTARTTS, _WENDTS and _WDURATION can only be used in window queries"; case TSDB_CODE_OUT_OF_MEMORY: return "Out of memory"; default: diff --git a/source/libs/parser/src/parser.c b/source/libs/parser/src/parser.c index 7c330158fa..538404798d 100644 --- a/source/libs/parser/src/parser.c +++ b/source/libs/parser/src/parser.c @@ -207,6 +207,8 @@ int32_t qSetSTableIdForRsma(SNode* pStmt, int64_t uid) { return TSDB_CODE_FAILED; } +void qCleanupKeywordsTable() { taosCleanupKeywordsTable(); } + int32_t qStmtBindParams(SQuery* pQuery, TAOS_MULTI_BIND* pParams, int32_t colIdx) { int32_t code = TSDB_CODE_SUCCESS; diff --git a/source/libs/parser/src/sql.c b/source/libs/parser/src/sql.c index b2de892f30..bde9a2bd30 100644 --- a/source/libs/parser/src/sql.c +++ b/source/libs/parser/src/sql.c @@ -215,543 +215,544 @@ typedef union { ** yy_default[] Default action for each state. ** *********** Begin parsing tables **********************************************/ -#define YY_ACTTAB_COUNT (2152) +#define YY_ACTTAB_COUNT (2178) static const YYACTIONTYPE yy_action[] = { - /* 0 */ 28, 231, 1663, 1650, 611, 610, 297, 1650, 358, 1485, - /* 10 */ 314, 1483, 35, 33, 352, 36, 34, 32, 31, 30, - /* 20 */ 306, 24, 1173, 1647, 533, 442, 441, 1647, 79, 1647, - /* 30 */ 1679, 36, 34, 32, 31, 30, 152, 493, 517, 1643, - /* 40 */ 1649, 115, 278, 1643, 1649, 1643, 1649, 1171, 516, 1486, - /* 50 */ 536, 533, 1633, 1494, 536, 1794, 536, 497, 14, 1746, - /* 60 */ 35, 33, 1300, 356, 1179, 1663, 114, 147, 306, 1692, - /* 70 */ 1173, 1791, 82, 1664, 519, 1666, 1667, 515, 532, 536, - /* 80 */ 1494, 1, 1732, 1743, 1794, 1314, 280, 1728, 36, 34, - /* 90 */ 32, 31, 30, 1679, 410, 1171, 1793, 1651, 1794, 520, - /* 100 */ 1791, 517, 309, 615, 112, 1583, 14, 1746, 35, 33, - /* 110 */ 149, 516, 1179, 1172, 1791, 1633, 306, 1647, 1173, 145, - /* 120 */ 1739, 1740, 532, 1744, 36, 34, 32, 31, 30, 2, - /* 130 */ 63, 1742, 1692, 1643, 1649, 84, 1664, 519, 1666, 1667, - /* 140 */ 515, 1364, 536, 1171, 536, 1732, 953, 1794, 952, 1731, - /* 150 */ 1728, 615, 1490, 72, 14, 392, 1174, 393, 1395, 148, - /* 160 */ 1179, 1172, 96, 1791, 532, 95, 94, 93, 92, 91, - /* 170 */ 90, 89, 88, 87, 1487, 954, 56, 2, 132, 201, - /* 180 */ 1375, 1177, 1178, 39, 1224, 1225, 1227, 1228, 1229, 1230, - /* 190 */ 1231, 512, 534, 1239, 1240, 1241, 1242, 1243, 1244, 615, - /* 200 */ 400, 952, 393, 1395, 1174, 55, 1197, 66, 133, 1172, - /* 210 */ 96, 150, 1451, 95, 94, 93, 92, 91, 90, 89, - /* 220 */ 88, 87, 65, 279, 1356, 38, 429, 194, 567, 1177, - /* 230 */ 1178, 1363, 1224, 1225, 1227, 1228, 1229, 1230, 1231, 512, - /* 240 */ 534, 1239, 1240, 1241, 1242, 1243, 1244, 566, 487, 565, - /* 250 */ 564, 563, 1174, 55, 63, 105, 104, 103, 102, 101, - /* 260 */ 100, 99, 98, 97, 936, 35, 33, 110, 36, 34, - /* 270 */ 32, 31, 30, 306, 1746, 1173, 1489, 1177, 1178, 1198, - /* 280 */ 1224, 1225, 1227, 1228, 1229, 1230, 1231, 512, 534, 1239, - /* 290 */ 1240, 1241, 1242, 1243, 1244, 55, 445, 444, 1741, 1538, - /* 300 */ 1171, 443, 940, 941, 111, 440, 296, 1355, 439, 438, - /* 310 */ 437, 1536, 569, 35, 33, 1245, 1679, 1179, 26, 310, - /* 320 */ 1386, 306, 391, 1173, 486, 395, 290, 130, 36, 34, - /* 330 */ 32, 31, 30, 1196, 8, 150, 1496, 1043, 559, 558, - /* 340 */ 557, 1047, 556, 1049, 1050, 555, 1052, 552, 1171, 1058, - /* 350 */ 549, 1060, 1061, 546, 543, 150, 615, 450, 482, 618, - /* 360 */ 485, 35, 33, 585, 583, 1179, 1172, 142, 397, 306, - /* 370 */ 1633, 1173, 458, 248, 1195, 291, 1324, 289, 288, 1532, - /* 380 */ 433, 318, 9, 150, 435, 107, 193, 399, 1574, 1576, - /* 390 */ 395, 607, 603, 599, 595, 247, 1171, 1472, 453, 157, - /* 400 */ 533, 129, 1361, 447, 615, 1425, 434, 572, 192, 1174, - /* 410 */ 1794, 1385, 357, 1179, 1172, 479, 1322, 1323, 1325, 1326, - /* 420 */ 80, 1571, 1792, 242, 61, 150, 1791, 60, 159, 1494, - /* 430 */ 9, 488, 483, 51, 1177, 1178, 50, 1224, 1225, 1227, - /* 440 */ 1228, 1229, 1230, 1231, 512, 534, 1239, 1240, 1241, 1242, - /* 450 */ 1243, 1244, 615, 55, 410, 316, 529, 1174, 54, 319, - /* 460 */ 322, 1633, 1172, 130, 150, 445, 444, 130, 1538, 562, - /* 470 */ 443, 382, 1496, 111, 440, 311, 1496, 439, 438, 437, - /* 480 */ 1536, 474, 1177, 1178, 203, 1224, 1225, 1227, 1228, 1229, - /* 490 */ 1230, 1231, 512, 534, 1239, 1240, 1241, 1242, 1243, 1244, - /* 500 */ 1384, 1794, 1147, 1199, 197, 1174, 1575, 1576, 36, 34, - /* 510 */ 32, 31, 30, 147, 1307, 161, 160, 1791, 35, 33, - /* 520 */ 1197, 1663, 1155, 1156, 195, 1470, 306, 351, 1173, 350, - /* 530 */ 1177, 1178, 493, 1224, 1225, 1227, 1228, 1229, 1230, 1231, - /* 540 */ 512, 534, 1239, 1240, 1241, 1242, 1243, 1244, 253, 1679, - /* 550 */ 1633, 1524, 533, 1171, 1383, 1621, 533, 517, 1382, 1250, - /* 560 */ 277, 114, 1195, 1005, 367, 1197, 1479, 516, 106, 375, - /* 570 */ 1179, 1633, 387, 466, 533, 431, 497, 1422, 1211, 469, - /* 580 */ 1007, 1494, 569, 150, 1381, 1494, 368, 2, 1692, 343, - /* 590 */ 388, 82, 1664, 519, 1666, 1667, 515, 1481, 536, 112, - /* 600 */ 331, 1732, 1477, 1494, 1633, 280, 1728, 198, 1633, 615, - /* 610 */ 345, 341, 533, 495, 144, 1739, 1740, 1794, 1744, 1172, - /* 620 */ 1794, 940, 941, 1538, 106, 1376, 11, 10, 222, 147, - /* 630 */ 317, 436, 147, 1791, 1633, 1536, 1791, 591, 590, 589, - /* 640 */ 321, 1494, 588, 587, 586, 116, 581, 580, 579, 578, - /* 650 */ 577, 576, 575, 574, 123, 570, 32, 31, 30, 1380, - /* 660 */ 386, 1299, 1174, 381, 380, 379, 378, 377, 374, 373, + /* 0 */ 475, 1650, 392, 1663, 393, 1395, 1584, 1794, 1361, 352, + /* 10 */ 11, 10, 36, 34, 37, 35, 33, 32, 31, 1793, + /* 20 */ 313, 1647, 1173, 1791, 71, 510, 37, 35, 33, 32, + /* 30 */ 31, 1679, 1471, 33, 32, 31, 1643, 1649, 302, 535, + /* 40 */ 37, 35, 33, 32, 31, 1487, 1679, 1171, 528, 534, + /* 50 */ 507, 1794, 1483, 1633, 500, 936, 469, 512, 14, 1538, + /* 60 */ 36, 34, 1300, 147, 1179, 322, 296, 1791, 313, 1692, + /* 70 */ 1173, 1536, 281, 82, 1664, 537, 1666, 1667, 533, 115, + /* 80 */ 528, 1, 1571, 1732, 1314, 1794, 40, 280, 1728, 159, + /* 90 */ 499, 1276, 1663, 940, 941, 1171, 1211, 148, 1794, 1794, + /* 100 */ 132, 1791, 1375, 615, 1262, 510, 14, 1794, 36, 34, + /* 110 */ 147, 149, 1179, 1172, 1791, 1791, 313, 113, 1173, 147, + /* 120 */ 1679, 1746, 300, 1791, 400, 567, 393, 1395, 532, 2, + /* 130 */ 130, 509, 144, 1739, 1740, 410, 1744, 63, 534, 1496, + /* 140 */ 54, 358, 1633, 1171, 566, 1743, 565, 564, 563, 1307, + /* 150 */ 110, 615, 201, 1263, 14, 1197, 1174, 382, 1692, 1489, + /* 160 */ 1179, 1172, 273, 1664, 537, 1666, 1667, 533, 531, 528, + /* 170 */ 525, 1704, 133, 129, 1268, 278, 1451, 2, 55, 1197, + /* 180 */ 66, 1177, 1178, 63, 1224, 1225, 1227, 1228, 1229, 1230, + /* 190 */ 1231, 530, 526, 1239, 1240, 1241, 1242, 1243, 1244, 615, + /* 200 */ 475, 161, 160, 299, 1174, 1490, 1583, 65, 279, 1172, + /* 210 */ 79, 150, 194, 28, 311, 1257, 1258, 1259, 1260, 1261, + /* 220 */ 1265, 1266, 1267, 112, 37, 35, 33, 32, 31, 1177, + /* 230 */ 1178, 1486, 1224, 1225, 1227, 1228, 1229, 1230, 1231, 530, + /* 240 */ 526, 1239, 1240, 1241, 1242, 1243, 1244, 953, 56, 952, + /* 250 */ 55, 96, 1174, 1386, 95, 94, 93, 92, 91, 90, + /* 260 */ 89, 88, 87, 510, 569, 36, 34, 37, 35, 33, + /* 270 */ 32, 31, 1385, 313, 343, 1173, 954, 1177, 1178, 435, + /* 280 */ 1224, 1225, 1227, 1228, 1229, 1230, 1231, 530, 526, 1239, + /* 290 */ 1240, 1241, 1242, 1243, 1244, 345, 341, 445, 444, 1384, + /* 300 */ 1171, 434, 443, 1633, 1663, 111, 440, 485, 150, 439, + /* 310 */ 438, 437, 397, 36, 34, 1245, 25, 1179, 1195, 152, + /* 320 */ 1226, 313, 1633, 1173, 39, 55, 37, 35, 33, 32, + /* 330 */ 31, 27, 1679, 1425, 8, 1794, 1494, 1211, 316, 1198, + /* 340 */ 535, 37, 35, 33, 32, 31, 130, 1792, 1171, 1633, + /* 350 */ 534, 1791, 611, 610, 1633, 1496, 615, 501, 512, 130, + /* 360 */ 1538, 36, 34, 514, 567, 1179, 1172, 301, 1497, 313, + /* 370 */ 1692, 1173, 1536, 1005, 82, 1664, 537, 1666, 1667, 533, + /* 380 */ 150, 528, 9, 566, 1732, 565, 564, 563, 280, 1728, + /* 390 */ 1007, 55, 391, 445, 444, 395, 1171, 1364, 443, 1653, + /* 400 */ 1794, 111, 440, 485, 615, 439, 438, 437, 1469, 1174, + /* 410 */ 29, 243, 147, 1179, 1172, 356, 1791, 562, 96, 142, + /* 420 */ 290, 95, 94, 93, 92, 91, 90, 89, 88, 87, + /* 430 */ 9, 1532, 1494, 496, 1177, 1178, 1655, 1224, 1225, 1227, + /* 440 */ 1228, 1229, 1230, 1231, 530, 526, 1239, 1240, 1241, 1242, + /* 450 */ 1243, 1244, 615, 318, 150, 150, 1538, 1174, 319, 1182, + /* 460 */ 1574, 1576, 1172, 317, 150, 351, 130, 350, 1536, 291, + /* 470 */ 399, 289, 288, 395, 433, 1496, 442, 441, 435, 1155, + /* 480 */ 1156, 195, 1177, 1178, 1363, 1224, 1225, 1227, 1228, 1229, + /* 490 */ 1230, 1231, 530, 526, 1239, 1240, 1241, 1242, 1243, 1244, + /* 500 */ 434, 567, 1199, 1621, 1383, 1174, 502, 497, 105, 104, + /* 510 */ 103, 102, 101, 100, 99, 98, 97, 459, 36, 34, + /* 520 */ 566, 150, 565, 564, 563, 1185, 313, 1663, 1173, 1485, + /* 530 */ 1177, 1178, 1651, 1224, 1225, 1227, 1228, 1229, 1230, 1231, + /* 540 */ 530, 526, 1239, 1240, 1241, 1242, 1243, 1244, 331, 1647, + /* 550 */ 1250, 584, 1647, 1171, 1633, 1679, 1197, 1746, 1196, 1794, + /* 560 */ 277, 1472, 1195, 511, 1643, 1649, 1382, 1643, 1649, 375, + /* 570 */ 1179, 147, 387, 534, 120, 1791, 528, 1633, 1663, 528, + /* 580 */ 157, 1742, 37, 35, 33, 32, 31, 2, 585, 583, + /* 590 */ 388, 485, 346, 1692, 1324, 1381, 952, 83, 1664, 537, + /* 600 */ 1666, 1667, 533, 106, 528, 61, 1679, 1732, 60, 615, + /* 610 */ 431, 306, 1728, 143, 532, 471, 1633, 485, 410, 1172, + /* 620 */ 1494, 429, 1264, 253, 534, 235, 1524, 234, 1633, 357, + /* 630 */ 486, 1759, 1746, 493, 1322, 1323, 1325, 1326, 37, 35, + /* 640 */ 33, 32, 31, 1269, 1692, 1633, 1494, 1299, 273, 1664, + /* 650 */ 537, 1666, 1667, 533, 572, 528, 1741, 1705, 1470, 1380, + /* 660 */ 386, 1479, 1174, 381, 380, 379, 378, 377, 374, 373, /* 670 */ 372, 371, 370, 366, 365, 364, 363, 362, 361, 360, - /* 680 */ 359, 499, 1663, 520, 1379, 1226, 1197, 1177, 1178, 1584, - /* 690 */ 1224, 1225, 1227, 1228, 1229, 1230, 1231, 512, 534, 1239, - /* 700 */ 1240, 1241, 1242, 1243, 1244, 1264, 131, 1276, 1378, 1633, - /* 710 */ 1679, 259, 573, 533, 1466, 7, 511, 533, 496, 1471, - /* 720 */ 1200, 533, 459, 257, 53, 409, 1269, 52, 516, 1491, - /* 730 */ 1226, 533, 1633, 1610, 1633, 533, 36, 34, 32, 31, - /* 740 */ 30, 1377, 1494, 244, 162, 1374, 1494, 467, 1663, 1692, - /* 750 */ 1494, 501, 83, 1664, 519, 1666, 1667, 515, 1633, 536, - /* 760 */ 1494, 1373, 1732, 1794, 1494, 25, 299, 1728, 143, 55, - /* 770 */ 36, 34, 32, 31, 30, 147, 1679, 1469, 533, 1791, - /* 780 */ 223, 533, 281, 130, 517, 475, 1759, 435, 533, 1372, - /* 790 */ 530, 1633, 1497, 531, 516, 1633, 185, 1371, 1633, 183, - /* 800 */ 320, 476, 1370, 1369, 1663, 81, 1211, 1494, 1368, 434, - /* 810 */ 1494, 1633, 567, 504, 1262, 1692, 281, 1494, 274, 1664, - /* 820 */ 519, 1666, 1667, 515, 509, 536, 1751, 1295, 493, 561, - /* 830 */ 457, 566, 1679, 565, 564, 563, 59, 58, 355, 1633, - /* 840 */ 514, 156, 1452, 455, 1538, 207, 349, 1633, 1262, 584, - /* 850 */ 516, 1226, 1633, 1633, 1633, 1367, 1537, 114, 1633, 276, - /* 860 */ 1173, 226, 339, 1263, 337, 333, 329, 153, 324, 1298, - /* 870 */ 567, 1692, 1412, 1182, 273, 1664, 519, 1666, 1667, 515, - /* 880 */ 513, 536, 510, 1704, 1268, 1171, 1366, 1407, 1295, 566, - /* 890 */ 346, 565, 564, 563, 446, 112, 120, 1263, 187, 150, - /* 900 */ 189, 186, 1179, 188, 191, 1633, 1663, 190, 46, 448, - /* 910 */ 146, 1739, 1740, 1405, 1744, 11, 10, 1653, 1268, 1358, - /* 920 */ 1359, 480, 1181, 27, 304, 1257, 1258, 1259, 1260, 1261, - /* 930 */ 1265, 1266, 1267, 210, 1679, 451, 1633, 471, 502, 1185, - /* 940 */ 78, 615, 496, 37, 37, 460, 37, 1254, 233, 1321, - /* 950 */ 74, 1172, 516, 217, 1655, 1680, 1633, 27, 304, 1257, - /* 960 */ 1258, 1259, 1260, 1261, 1265, 1266, 1267, 118, 1663, 1396, - /* 970 */ 119, 1533, 120, 1692, 212, 46, 83, 1664, 519, 1666, - /* 980 */ 1667, 515, 977, 536, 1270, 1232, 1732, 1128, 1184, 235, - /* 990 */ 299, 1728, 143, 541, 1174, 428, 1679, 1762, 494, 978, - /* 1000 */ 1663, 225, 505, 119, 517, 228, 120, 230, 525, 3, - /* 1010 */ 1760, 241, 5, 1036, 516, 121, 252, 119, 1633, 1177, - /* 1020 */ 1178, 323, 1195, 326, 330, 286, 287, 1005, 1679, 249, - /* 1030 */ 1139, 369, 1573, 376, 1064, 1692, 517, 158, 83, 1664, - /* 1040 */ 519, 1666, 1667, 515, 1068, 536, 516, 1074, 1732, 384, - /* 1050 */ 1633, 389, 299, 1728, 1807, 383, 1072, 1201, 122, 385, - /* 1060 */ 1663, 390, 1204, 1766, 401, 398, 165, 1692, 402, 167, - /* 1070 */ 83, 1664, 519, 1666, 1667, 515, 1203, 536, 1205, 404, - /* 1080 */ 1732, 403, 170, 406, 299, 1728, 1807, 172, 1679, 407, - /* 1090 */ 1202, 175, 1663, 408, 62, 1789, 517, 411, 178, 430, - /* 1100 */ 432, 1484, 182, 86, 1480, 184, 516, 295, 124, 1179, - /* 1110 */ 1633, 125, 1482, 1478, 1615, 126, 127, 1614, 196, 461, - /* 1120 */ 1679, 199, 250, 202, 465, 462, 1200, 1692, 517, 468, - /* 1130 */ 83, 1664, 519, 1666, 1667, 515, 472, 536, 516, 205, - /* 1140 */ 1732, 481, 1633, 1663, 299, 1728, 1807, 497, 470, 478, - /* 1150 */ 473, 523, 1773, 1772, 298, 1750, 490, 208, 1763, 1692, - /* 1160 */ 211, 6, 264, 1664, 519, 1666, 1667, 515, 1753, 536, - /* 1170 */ 484, 1679, 216, 1663, 477, 113, 1295, 218, 1199, 517, - /* 1180 */ 40, 300, 1747, 506, 503, 18, 527, 1582, 1794, 516, - /* 1190 */ 137, 521, 522, 1633, 219, 1581, 526, 308, 497, 237, - /* 1200 */ 149, 1679, 528, 224, 1791, 251, 1663, 1495, 1713, 517, - /* 1210 */ 1692, 1810, 1790, 264, 1664, 519, 1666, 1667, 515, 516, - /* 1220 */ 536, 239, 71, 1633, 73, 539, 246, 254, 500, 614, - /* 1230 */ 136, 227, 1467, 229, 1679, 507, 1663, 265, 47, 1794, - /* 1240 */ 1692, 256, 517, 84, 1664, 519, 1666, 1667, 515, 258, - /* 1250 */ 536, 147, 516, 1732, 275, 1791, 1633, 508, 1728, 266, - /* 1260 */ 1627, 1626, 57, 1625, 1679, 325, 1622, 327, 328, 332, - /* 1270 */ 1166, 1167, 517, 1692, 154, 1620, 134, 1664, 519, 1666, - /* 1280 */ 1667, 515, 516, 536, 334, 335, 1633, 1663, 336, 1619, - /* 1290 */ 338, 1618, 340, 1617, 342, 1616, 1663, 344, 1600, 155, - /* 1300 */ 1142, 347, 1141, 1692, 348, 1594, 84, 1664, 519, 1666, - /* 1310 */ 1667, 515, 1593, 536, 353, 1679, 1732, 354, 1592, 498, - /* 1320 */ 1808, 1729, 1591, 517, 1679, 1111, 1566, 1565, 1564, 1563, - /* 1330 */ 1562, 1561, 517, 516, 1560, 1559, 1558, 1633, 1557, 1556, - /* 1340 */ 1555, 1554, 516, 1553, 1552, 1551, 1633, 1550, 1549, 117, - /* 1350 */ 1663, 1548, 1547, 1546, 1692, 1545, 1544, 269, 1664, 519, - /* 1360 */ 1666, 1667, 515, 1692, 536, 1543, 134, 1664, 519, 1666, - /* 1370 */ 1667, 515, 1113, 536, 1542, 1541, 1540, 1539, 1679, 1424, - /* 1380 */ 1392, 943, 163, 108, 942, 1391, 517, 1608, 1602, 1590, - /* 1390 */ 1663, 171, 1589, 394, 489, 140, 516, 164, 109, 169, - /* 1400 */ 1633, 396, 1579, 303, 45, 174, 1663, 1473, 1423, 1421, - /* 1410 */ 1809, 1419, 1417, 412, 413, 414, 417, 1692, 1679, 971, - /* 1420 */ 274, 1664, 519, 1666, 1667, 515, 514, 536, 416, 420, - /* 1430 */ 418, 421, 422, 1415, 1679, 426, 516, 424, 1404, 425, - /* 1440 */ 1633, 1403, 517, 1390, 1475, 1077, 181, 1078, 1474, 1413, - /* 1450 */ 1004, 582, 516, 1003, 1663, 584, 1633, 1692, 292, 305, - /* 1460 */ 273, 1664, 519, 1666, 1667, 515, 1408, 536, 1002, 1705, - /* 1470 */ 449, 1001, 998, 1692, 997, 996, 274, 1664, 519, 1666, - /* 1480 */ 1667, 515, 1679, 536, 293, 1406, 1663, 294, 1389, 452, - /* 1490 */ 517, 180, 1388, 454, 456, 85, 1607, 1149, 1601, 463, - /* 1500 */ 516, 1588, 1587, 141, 1633, 1586, 1578, 307, 49, 427, - /* 1510 */ 423, 419, 415, 179, 1679, 67, 1663, 204, 464, 4, - /* 1520 */ 37, 1692, 517, 206, 274, 1664, 519, 1666, 1667, 515, - /* 1530 */ 15, 536, 516, 128, 209, 43, 1633, 1320, 64, 200, - /* 1540 */ 135, 177, 213, 215, 1679, 22, 48, 1653, 214, 1313, - /* 1550 */ 68, 23, 517, 1692, 42, 1292, 260, 1664, 519, 1666, - /* 1560 */ 1667, 515, 516, 536, 221, 1291, 1633, 1663, 138, 1349, - /* 1570 */ 17, 1338, 1344, 1343, 301, 1348, 10, 1347, 302, 19, - /* 1580 */ 1255, 139, 1234, 1692, 151, 29, 268, 1664, 519, 1666, - /* 1590 */ 1667, 515, 12, 536, 1233, 1679, 20, 1219, 176, 16, - /* 1600 */ 168, 518, 173, 517, 405, 41, 13, 1663, 1577, 238, - /* 1610 */ 74, 524, 1652, 516, 232, 21, 234, 1633, 1318, 1189, - /* 1620 */ 236, 240, 166, 69, 70, 243, 1695, 540, 1236, 535, - /* 1630 */ 44, 1071, 1065, 538, 1692, 1679, 315, 270, 1664, 519, - /* 1640 */ 1666, 1667, 515, 517, 536, 542, 544, 1663, 1062, 1059, - /* 1650 */ 545, 547, 548, 516, 550, 1053, 553, 1633, 551, 1057, - /* 1660 */ 1051, 1056, 1663, 554, 1055, 1054, 75, 1042, 76, 560, - /* 1670 */ 1073, 77, 1070, 568, 1692, 1679, 969, 261, 1664, 519, - /* 1680 */ 1666, 1667, 515, 517, 536, 993, 1011, 245, 991, 571, - /* 1690 */ 1679, 986, 1008, 516, 990, 989, 988, 1633, 517, 987, - /* 1700 */ 985, 984, 1006, 981, 980, 979, 976, 975, 516, 974, - /* 1710 */ 1420, 592, 1633, 1663, 1692, 593, 594, 271, 1664, 519, - /* 1720 */ 1666, 1667, 515, 1418, 536, 1663, 597, 596, 598, 1692, - /* 1730 */ 1416, 600, 262, 1664, 519, 1666, 1667, 515, 601, 536, - /* 1740 */ 602, 1679, 604, 605, 606, 1402, 608, 609, 1401, 517, - /* 1750 */ 1414, 1387, 613, 1679, 612, 1175, 255, 616, 617, 516, - /* 1760 */ 1362, 517, 1362, 1633, 1362, 1663, 1362, 1362, 1362, 1362, - /* 1770 */ 1362, 516, 1362, 1362, 1362, 1633, 1663, 1362, 1362, 1362, - /* 1780 */ 1692, 1362, 1362, 272, 1664, 519, 1666, 1667, 515, 1362, - /* 1790 */ 536, 1362, 1692, 1679, 1362, 263, 1664, 519, 1666, 1667, - /* 1800 */ 515, 517, 536, 1362, 1679, 1362, 1362, 1362, 1362, 1362, - /* 1810 */ 1362, 516, 517, 1362, 1362, 1633, 1362, 1362, 1362, 1362, - /* 1820 */ 1362, 1362, 516, 1362, 1362, 1362, 1633, 1663, 1362, 1362, - /* 1830 */ 1362, 1362, 1692, 1362, 1362, 1675, 1664, 519, 1666, 1667, - /* 1840 */ 515, 1362, 536, 1692, 1362, 1362, 1674, 1664, 519, 1666, - /* 1850 */ 1667, 515, 1362, 536, 1362, 1679, 1362, 1663, 1362, 1362, - /* 1860 */ 1362, 1362, 1362, 517, 1362, 1362, 1362, 1362, 1362, 1362, - /* 1870 */ 1362, 1362, 1362, 516, 1362, 1362, 1362, 1633, 1362, 1362, - /* 1880 */ 1362, 1362, 1362, 1362, 1362, 1679, 1362, 1663, 1362, 1362, - /* 1890 */ 1362, 1362, 1362, 517, 1692, 1362, 1362, 1673, 1664, 519, - /* 1900 */ 1666, 1667, 515, 516, 536, 1362, 1362, 1633, 1362, 1362, - /* 1910 */ 1362, 1362, 1362, 1362, 1362, 1679, 1362, 1362, 1362, 1663, - /* 1920 */ 1362, 1362, 1362, 517, 1692, 1362, 1362, 284, 1664, 519, - /* 1930 */ 1666, 1667, 515, 516, 536, 1362, 1362, 1633, 1362, 1362, - /* 1940 */ 1362, 1362, 1362, 313, 312, 1362, 1362, 1679, 1362, 1362, - /* 1950 */ 1362, 1362, 1663, 1187, 1692, 517, 1362, 283, 1664, 519, - /* 1960 */ 1666, 1667, 515, 1362, 536, 516, 1362, 1362, 1362, 1633, - /* 1970 */ 1362, 1362, 1362, 1362, 1362, 1362, 1362, 1362, 1180, 1362, - /* 1980 */ 1679, 1362, 1362, 1362, 1362, 1663, 1692, 1362, 517, 285, - /* 1990 */ 1664, 519, 1666, 1667, 515, 1179, 536, 1362, 516, 1362, - /* 2000 */ 1362, 1362, 1633, 1362, 1362, 1362, 1362, 1362, 1362, 1362, - /* 2010 */ 1362, 1362, 1362, 1679, 1362, 1362, 493, 1362, 1362, 1692, - /* 2020 */ 1362, 517, 282, 1664, 519, 1666, 1667, 515, 1362, 536, - /* 2030 */ 1362, 516, 1362, 1362, 537, 1633, 1362, 1362, 1362, 1362, - /* 2040 */ 1362, 1362, 1362, 1362, 1183, 114, 493, 1362, 1362, 1362, - /* 2050 */ 1362, 1362, 1692, 1362, 1362, 267, 1664, 519, 1666, 1667, - /* 2060 */ 515, 1362, 536, 1362, 497, 1362, 1362, 1362, 1362, 1362, - /* 2070 */ 1362, 1362, 1362, 1362, 1362, 114, 1362, 1362, 1362, 1362, - /* 2080 */ 1362, 1362, 1362, 112, 1362, 1362, 1362, 1188, 1362, 1362, - /* 2090 */ 1362, 1362, 1362, 1362, 497, 1362, 1362, 1362, 220, 1739, - /* 2100 */ 492, 1362, 491, 1362, 1362, 1794, 1362, 1362, 1362, 1362, - /* 2110 */ 1362, 1362, 1191, 112, 1362, 1362, 1362, 149, 1362, 1362, - /* 2120 */ 1362, 1791, 1362, 534, 1239, 1240, 1362, 1362, 220, 1739, - /* 2130 */ 492, 1362, 491, 1362, 1362, 1794, 1362, 1362, 1362, 1362, - /* 2140 */ 1362, 1362, 1362, 1362, 1362, 1362, 1362, 147, 1362, 1362, - /* 2150 */ 1362, 1791, + /* 680 */ 359, 1379, 26, 1378, 281, 1422, 1481, 1177, 1178, 1200, + /* 690 */ 1224, 1225, 1227, 1228, 1229, 1230, 1231, 530, 526, 1239, + /* 700 */ 1240, 1241, 1242, 1243, 1244, 485, 131, 485, 507, 1633, + /* 710 */ 573, 259, 1466, 507, 1538, 569, 1262, 367, 1356, 320, + /* 720 */ 485, 1226, 1377, 257, 53, 1197, 1537, 52, 1374, 1373, + /* 730 */ 457, 1633, 106, 1633, 1494, 7, 1494, 115, 1663, 436, + /* 740 */ 1575, 1576, 115, 455, 162, 591, 590, 589, 321, 1494, + /* 750 */ 588, 587, 586, 116, 581, 580, 579, 578, 577, 576, + /* 760 */ 575, 574, 123, 570, 485, 1263, 1679, 1372, 1477, 55, + /* 770 */ 198, 485, 1633, 485, 511, 113, 368, 485, 1633, 1633, + /* 780 */ 113, 940, 941, 409, 534, 1491, 1268, 516, 1633, 1610, + /* 790 */ 145, 1739, 1740, 1494, 1744, 146, 1739, 1740, 1371, 1744, + /* 800 */ 1494, 1355, 1494, 529, 1692, 81, 1494, 1663, 83, 1664, + /* 810 */ 537, 1666, 1667, 533, 218, 528, 519, 1633, 1732, 1370, + /* 820 */ 1369, 524, 306, 1728, 143, 28, 311, 1257, 1258, 1259, + /* 830 */ 1260, 1261, 1265, 1266, 1267, 1679, 59, 58, 355, 1751, + /* 840 */ 1295, 156, 1760, 535, 1368, 1367, 349, 185, 1633, 187, + /* 850 */ 183, 1366, 186, 534, 1412, 1298, 561, 1633, 189, 276, + /* 860 */ 485, 188, 339, 1181, 337, 333, 329, 153, 324, 1633, + /* 870 */ 1633, 485, 467, 1692, 1407, 1663, 446, 83, 1664, 537, + /* 880 */ 1666, 1667, 533, 483, 528, 485, 485, 1732, 1405, 1494, + /* 890 */ 1226, 306, 1728, 1807, 1633, 1633, 448, 484, 244, 150, + /* 900 */ 1494, 1633, 1766, 1679, 11, 10, 191, 1663, 1295, 190, + /* 910 */ 451, 535, 466, 38, 1494, 1494, 977, 208, 1358, 1359, + /* 920 */ 1376, 534, 118, 119, 238, 1633, 120, 78, 46, 1184, + /* 930 */ 221, 38, 1452, 978, 38, 1679, 507, 74, 494, 460, + /* 940 */ 1663, 1692, 229, 535, 1254, 83, 1664, 537, 1666, 1667, + /* 950 */ 533, 1680, 528, 534, 1128, 1732, 38, 1633, 209, 306, + /* 960 */ 1728, 1807, 1396, 478, 215, 115, 428, 1036, 1679, 1321, + /* 970 */ 1789, 224, 1270, 1692, 517, 1232, 535, 83, 1664, 537, + /* 980 */ 1666, 1667, 533, 512, 528, 541, 534, 1732, 1663, 119, + /* 990 */ 1633, 306, 1728, 1807, 512, 1762, 1533, 252, 120, 121, + /* 1000 */ 508, 237, 1750, 113, 240, 520, 1692, 242, 3, 119, + /* 1010 */ 265, 1664, 537, 1666, 1667, 533, 1679, 528, 232, 1739, + /* 1020 */ 506, 5, 505, 323, 535, 1794, 1064, 1195, 326, 330, + /* 1030 */ 1068, 286, 1005, 287, 534, 249, 1794, 149, 1633, 1074, + /* 1040 */ 1072, 1791, 512, 369, 1573, 1139, 158, 376, 149, 384, + /* 1050 */ 122, 383, 1791, 385, 1692, 389, 1201, 390, 265, 1664, + /* 1060 */ 537, 1666, 1667, 533, 398, 528, 1663, 1204, 401, 165, + /* 1070 */ 402, 1203, 167, 1043, 559, 558, 557, 1047, 556, 1049, + /* 1080 */ 1050, 555, 1052, 552, 1794, 1058, 549, 1060, 1061, 546, + /* 1090 */ 543, 403, 1205, 170, 1679, 404, 147, 406, 172, 407, + /* 1100 */ 1791, 1202, 535, 408, 175, 62, 1663, 411, 178, 430, + /* 1110 */ 432, 1484, 534, 182, 1179, 1480, 1633, 86, 184, 124, + /* 1120 */ 295, 1615, 125, 1482, 1614, 250, 1478, 126, 461, 127, + /* 1130 */ 196, 462, 1692, 199, 1679, 473, 84, 1664, 537, 1666, + /* 1140 */ 1667, 533, 535, 528, 468, 202, 1732, 472, 465, 205, + /* 1150 */ 1731, 1728, 534, 491, 1582, 476, 1633, 470, 479, 1581, + /* 1160 */ 213, 70, 480, 298, 251, 1663, 1200, 219, 481, 495, + /* 1170 */ 6, 1753, 1692, 1773, 211, 1772, 84, 1664, 537, 1666, + /* 1180 */ 1667, 533, 504, 528, 1663, 1495, 1732, 488, 1295, 1199, + /* 1190 */ 523, 1728, 489, 1679, 490, 1763, 223, 305, 498, 114, + /* 1200 */ 41, 535, 518, 521, 307, 230, 19, 72, 539, 254, + /* 1210 */ 228, 534, 1679, 1467, 231, 1633, 246, 136, 260, 47, + /* 1220 */ 535, 137, 1747, 256, 258, 614, 275, 266, 1627, 1626, + /* 1230 */ 534, 1692, 57, 1625, 1633, 134, 1664, 537, 1666, 1667, + /* 1240 */ 533, 1622, 528, 1713, 325, 1810, 327, 1166, 328, 1167, + /* 1250 */ 1692, 618, 154, 236, 84, 1664, 537, 1666, 1667, 533, + /* 1260 */ 1663, 528, 332, 1790, 1732, 248, 515, 239, 1620, 1729, + /* 1270 */ 304, 303, 522, 334, 241, 336, 1619, 107, 513, 1808, + /* 1280 */ 1187, 335, 338, 607, 603, 599, 595, 247, 1679, 1618, + /* 1290 */ 340, 1617, 1663, 297, 342, 1616, 535, 344, 1600, 155, + /* 1300 */ 347, 348, 1173, 1142, 1141, 1180, 534, 1594, 1593, 353, + /* 1310 */ 1633, 1592, 80, 354, 1591, 216, 1111, 1566, 1565, 1564, + /* 1320 */ 1679, 1563, 1179, 1562, 1561, 487, 1692, 1171, 535, 1560, + /* 1330 */ 274, 1664, 537, 1666, 1667, 533, 1559, 528, 534, 450, + /* 1340 */ 1558, 1557, 1633, 1556, 1179, 1555, 1554, 1553, 482, 1552, + /* 1350 */ 1551, 1550, 1549, 117, 458, 1548, 1663, 1547, 1692, 1546, + /* 1360 */ 1545, 492, 274, 1664, 537, 1666, 1667, 533, 193, 528, + /* 1370 */ 1544, 1183, 1543, 474, 108, 1113, 203, 1542, 1541, 1540, + /* 1380 */ 453, 1539, 1424, 615, 1679, 447, 1392, 943, 163, 942, + /* 1390 */ 192, 1391, 535, 1172, 1147, 140, 197, 1608, 1602, 109, + /* 1400 */ 394, 164, 534, 1590, 169, 396, 1633, 171, 1589, 1663, + /* 1410 */ 1579, 1473, 1423, 174, 1188, 51, 1421, 971, 50, 1419, + /* 1420 */ 414, 1663, 1692, 1417, 1415, 413, 269, 1664, 537, 1666, + /* 1430 */ 1667, 533, 412, 528, 416, 417, 1174, 1679, 418, 1191, + /* 1440 */ 421, 420, 422, 424, 425, 535, 426, 1404, 1403, 1679, + /* 1450 */ 526, 1239, 1240, 1390, 310, 534, 1475, 535, 45, 1633, + /* 1460 */ 1078, 1177, 1178, 503, 1077, 1474, 582, 534, 1004, 1003, + /* 1470 */ 1413, 1633, 181, 1002, 1663, 1692, 1001, 584, 998, 134, + /* 1480 */ 1664, 537, 1666, 1667, 533, 997, 528, 1692, 996, 292, + /* 1490 */ 1408, 274, 1664, 537, 1666, 1667, 533, 293, 528, 449, + /* 1500 */ 1406, 294, 1679, 1389, 452, 454, 1388, 312, 456, 180, + /* 1510 */ 535, 85, 1607, 1149, 49, 1601, 463, 1588, 1587, 1586, + /* 1520 */ 534, 141, 1663, 1809, 1633, 128, 1578, 427, 423, 419, + /* 1530 */ 415, 179, 204, 464, 200, 67, 15, 1663, 1577, 210, + /* 1540 */ 1692, 207, 206, 477, 274, 1664, 537, 1666, 1667, 533, + /* 1550 */ 1679, 528, 48, 68, 214, 314, 64, 212, 535, 177, + /* 1560 */ 74, 69, 4, 220, 38, 1679, 217, 1318, 534, 222, + /* 1570 */ 1189, 1320, 1633, 535, 44, 1313, 226, 1663, 135, 225, + /* 1580 */ 16, 227, 23, 534, 1653, 73, 17, 1633, 1692, 1292, + /* 1590 */ 24, 1291, 274, 1664, 537, 1666, 1667, 533, 233, 528, + /* 1600 */ 43, 1652, 138, 1692, 18, 1679, 1349, 261, 1664, 537, + /* 1610 */ 1666, 1667, 533, 535, 528, 42, 176, 1663, 168, 13, + /* 1620 */ 173, 1338, 405, 534, 1344, 10, 1343, 1633, 308, 1348, + /* 1630 */ 1347, 309, 20, 1255, 1695, 1219, 1663, 1236, 1234, 139, + /* 1640 */ 166, 527, 30, 1692, 1233, 1679, 12, 268, 1664, 537, + /* 1650 */ 1666, 1667, 533, 535, 528, 21, 151, 536, 538, 1042, + /* 1660 */ 560, 22, 540, 534, 1679, 315, 542, 1633, 1065, 1062, + /* 1670 */ 545, 544, 535, 547, 1059, 548, 550, 553, 1053, 551, + /* 1680 */ 1057, 1051, 534, 1692, 554, 75, 1633, 270, 1664, 537, + /* 1690 */ 1666, 1667, 533, 1663, 528, 1056, 1073, 76, 77, 1070, + /* 1700 */ 969, 1071, 1692, 568, 993, 1055, 262, 1664, 537, 1666, + /* 1710 */ 1667, 533, 1054, 528, 1011, 571, 245, 991, 990, 989, + /* 1720 */ 986, 1679, 1008, 988, 1420, 1663, 987, 985, 984, 535, + /* 1730 */ 1006, 981, 980, 979, 976, 1418, 975, 974, 592, 534, + /* 1740 */ 593, 594, 598, 1633, 596, 597, 1416, 601, 600, 602, + /* 1750 */ 1414, 605, 606, 1679, 604, 1402, 608, 609, 1401, 1692, + /* 1760 */ 1387, 535, 613, 271, 1664, 537, 1666, 1667, 533, 612, + /* 1770 */ 528, 534, 616, 1663, 1175, 1633, 255, 617, 1362, 1362, + /* 1780 */ 1362, 1362, 1362, 1362, 1362, 1362, 1362, 1362, 1663, 1362, + /* 1790 */ 1362, 1692, 1362, 1362, 1362, 263, 1664, 537, 1666, 1667, + /* 1800 */ 533, 1679, 528, 1362, 1362, 1362, 1362, 1362, 1362, 535, + /* 1810 */ 1362, 1362, 1362, 1362, 1362, 1362, 1679, 1362, 1362, 534, + /* 1820 */ 1362, 1663, 1362, 1633, 535, 1362, 1362, 1362, 1362, 1362, + /* 1830 */ 1362, 1362, 1362, 1362, 534, 1362, 1663, 1362, 1633, 1692, + /* 1840 */ 1362, 1362, 1362, 272, 1664, 537, 1666, 1667, 533, 1679, + /* 1850 */ 528, 1362, 1362, 1663, 1692, 1362, 1362, 535, 264, 1664, + /* 1860 */ 537, 1666, 1667, 533, 1679, 528, 1362, 534, 1362, 1362, + /* 1870 */ 1362, 1633, 535, 1362, 1362, 1362, 1362, 1362, 1362, 1362, + /* 1880 */ 1362, 1679, 534, 1362, 1362, 1362, 1633, 1692, 1362, 535, + /* 1890 */ 1362, 1675, 1664, 537, 1666, 1667, 533, 1362, 528, 534, + /* 1900 */ 1362, 1362, 1692, 1633, 1362, 1362, 1674, 1664, 537, 1666, + /* 1910 */ 1667, 533, 1362, 528, 1362, 1362, 1663, 1362, 1362, 1692, + /* 1920 */ 1362, 1362, 1362, 1673, 1664, 537, 1666, 1667, 533, 1362, + /* 1930 */ 528, 1663, 1362, 1362, 1362, 1362, 1362, 1362, 1362, 1362, + /* 1940 */ 1362, 1362, 1362, 1362, 1679, 1362, 1362, 1362, 1362, 1362, + /* 1950 */ 1362, 1362, 535, 1362, 1362, 1362, 1362, 1362, 1362, 1679, + /* 1960 */ 1362, 1362, 534, 1362, 1362, 1362, 1633, 535, 1362, 1362, + /* 1970 */ 1362, 1362, 1362, 1362, 1362, 1362, 1362, 534, 1362, 1362, + /* 1980 */ 1362, 1633, 1692, 1663, 1362, 1362, 284, 1664, 537, 1666, + /* 1990 */ 1667, 533, 1362, 528, 1362, 1362, 1362, 1692, 1362, 1362, + /* 2000 */ 1362, 283, 1664, 537, 1666, 1667, 533, 1362, 528, 1362, + /* 2010 */ 1362, 1679, 1362, 1362, 1362, 1663, 1362, 1362, 1362, 535, + /* 2020 */ 1362, 1362, 1362, 1362, 1362, 1362, 1362, 1362, 1362, 534, + /* 2030 */ 1362, 1362, 1362, 1633, 1362, 1362, 1362, 1362, 1362, 1362, + /* 2040 */ 1362, 1362, 1362, 1679, 1362, 1362, 1362, 1362, 1362, 1692, + /* 2050 */ 1362, 535, 1362, 285, 1664, 537, 1666, 1667, 533, 1362, + /* 2060 */ 528, 534, 1362, 1663, 1362, 1633, 1362, 1362, 1362, 1362, + /* 2070 */ 1362, 1362, 507, 1362, 1362, 1362, 1362, 1362, 1362, 1362, + /* 2080 */ 1362, 1692, 1362, 1362, 1362, 282, 1664, 537, 1666, 1667, + /* 2090 */ 533, 1679, 528, 1362, 1362, 1362, 1362, 1362, 1362, 535, + /* 2100 */ 1362, 115, 1362, 1362, 1362, 1362, 1362, 1362, 1362, 534, + /* 2110 */ 1362, 1362, 1362, 1633, 1362, 1362, 1362, 1362, 1362, 512, + /* 2120 */ 1362, 1362, 1362, 1362, 1362, 1362, 1362, 1362, 1362, 1692, + /* 2130 */ 1362, 1362, 1362, 267, 1664, 537, 1666, 1667, 533, 113, + /* 2140 */ 528, 1362, 1362, 1362, 1362, 1362, 1362, 1362, 1362, 1362, + /* 2150 */ 1362, 1362, 1362, 1362, 232, 1739, 506, 1362, 505, 1362, + /* 2160 */ 1362, 1794, 1362, 1362, 1362, 1362, 1362, 1362, 1362, 1362, + /* 2170 */ 1362, 1362, 1362, 147, 1362, 1362, 1362, 1791, }; static const YYCODETYPE yy_lookahead[] = { - /* 0 */ 324, 325, 243, 273, 251, 252, 276, 273, 250, 273, - /* 10 */ 276, 272, 12, 13, 298, 12, 13, 14, 15, 16, - /* 20 */ 20, 2, 22, 293, 250, 257, 258, 293, 253, 293, - /* 30 */ 271, 12, 13, 14, 15, 16, 262, 250, 279, 309, - /* 40 */ 310, 266, 284, 309, 310, 309, 310, 47, 289, 274, - /* 50 */ 320, 250, 293, 279, 320, 339, 320, 298, 58, 311, - /* 60 */ 12, 13, 14, 262, 64, 243, 279, 351, 20, 310, - /* 70 */ 22, 355, 313, 314, 315, 316, 317, 318, 20, 320, - /* 80 */ 279, 81, 323, 335, 339, 82, 327, 328, 12, 13, - /* 90 */ 14, 15, 16, 271, 57, 47, 351, 273, 339, 289, - /* 100 */ 355, 279, 292, 103, 317, 295, 58, 311, 12, 13, - /* 110 */ 351, 289, 64, 113, 355, 293, 20, 293, 22, 332, - /* 120 */ 333, 334, 20, 336, 12, 13, 14, 15, 16, 81, - /* 130 */ 255, 335, 310, 309, 310, 313, 314, 315, 316, 317, - /* 140 */ 318, 0, 320, 47, 320, 323, 20, 339, 22, 327, - /* 150 */ 328, 103, 277, 253, 58, 246, 156, 248, 249, 351, - /* 160 */ 64, 113, 21, 355, 20, 24, 25, 26, 27, 28, - /* 170 */ 29, 30, 31, 32, 274, 49, 4, 81, 242, 55, - /* 180 */ 244, 181, 182, 81, 184, 185, 186, 187, 188, 189, + /* 0 */ 289, 273, 246, 243, 248, 249, 295, 339, 240, 297, + /* 10 */ 1, 2, 12, 13, 12, 13, 14, 15, 16, 351, + /* 20 */ 20, 293, 22, 355, 253, 20, 12, 13, 14, 15, + /* 30 */ 16, 271, 0, 14, 15, 16, 308, 309, 310, 279, + /* 40 */ 12, 13, 14, 15, 16, 274, 271, 47, 320, 289, + /* 50 */ 250, 339, 272, 293, 279, 4, 297, 297, 58, 271, + /* 60 */ 12, 13, 14, 351, 64, 297, 278, 355, 20, 309, + /* 70 */ 22, 283, 58, 313, 314, 315, 316, 317, 318, 279, + /* 80 */ 320, 81, 279, 323, 82, 339, 81, 327, 328, 286, + /* 90 */ 315, 82, 243, 42, 43, 47, 82, 351, 339, 339, + /* 100 */ 242, 355, 244, 103, 90, 20, 58, 339, 12, 13, + /* 110 */ 351, 351, 64, 113, 355, 355, 20, 317, 22, 351, + /* 120 */ 271, 311, 263, 355, 246, 93, 248, 249, 279, 81, + /* 130 */ 271, 331, 332, 333, 334, 57, 336, 255, 289, 280, + /* 140 */ 3, 250, 293, 47, 112, 335, 114, 115, 116, 14, + /* 150 */ 268, 103, 55, 139, 58, 20, 156, 75, 309, 277, + /* 160 */ 64, 113, 313, 314, 315, 316, 317, 318, 319, 320, + /* 170 */ 321, 322, 256, 145, 160, 284, 260, 81, 81, 20, + /* 180 */ 83, 181, 182, 255, 184, 185, 186, 187, 188, 189, /* 190 */ 190, 191, 192, 193, 194, 195, 196, 197, 198, 103, - /* 200 */ 246, 22, 248, 249, 156, 81, 20, 83, 256, 113, - /* 210 */ 21, 211, 260, 24, 25, 26, 27, 28, 29, 30, - /* 220 */ 31, 32, 165, 166, 148, 81, 47, 170, 93, 181, - /* 230 */ 182, 0, 184, 185, 186, 187, 188, 189, 190, 191, - /* 240 */ 192, 193, 194, 195, 196, 197, 198, 112, 20, 114, - /* 250 */ 115, 116, 156, 81, 255, 24, 25, 26, 27, 28, - /* 260 */ 29, 30, 31, 32, 4, 12, 13, 268, 12, 13, - /* 270 */ 14, 15, 16, 20, 311, 22, 277, 181, 182, 20, + /* 200 */ 289, 119, 120, 292, 156, 277, 295, 165, 166, 113, + /* 210 */ 253, 211, 170, 199, 200, 201, 202, 203, 204, 205, + /* 220 */ 206, 207, 208, 266, 12, 13, 14, 15, 16, 181, + /* 230 */ 182, 274, 184, 185, 186, 187, 188, 189, 190, 191, + /* 240 */ 192, 193, 194, 195, 196, 197, 198, 20, 4, 22, + /* 250 */ 81, 21, 156, 243, 24, 25, 26, 27, 28, 29, + /* 260 */ 30, 31, 32, 20, 57, 12, 13, 12, 13, 14, + /* 270 */ 15, 16, 243, 20, 151, 22, 49, 181, 182, 93, /* 280 */ 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, - /* 290 */ 194, 195, 196, 197, 198, 81, 60, 61, 335, 271, - /* 300 */ 47, 65, 42, 43, 68, 69, 278, 231, 72, 73, - /* 310 */ 74, 283, 57, 12, 13, 14, 271, 64, 2, 263, - /* 320 */ 243, 20, 247, 22, 279, 250, 35, 271, 12, 13, - /* 330 */ 14, 15, 16, 20, 81, 211, 280, 94, 95, 96, - /* 340 */ 97, 98, 99, 100, 101, 102, 103, 104, 47, 106, - /* 350 */ 107, 108, 109, 110, 111, 211, 103, 4, 143, 19, - /* 360 */ 315, 12, 13, 257, 258, 64, 113, 270, 14, 20, - /* 370 */ 293, 22, 19, 33, 20, 84, 181, 86, 87, 282, - /* 380 */ 89, 281, 81, 211, 93, 45, 33, 247, 288, 289, - /* 390 */ 250, 51, 52, 53, 54, 55, 47, 0, 45, 55, - /* 400 */ 250, 145, 240, 50, 103, 0, 115, 64, 55, 156, - /* 410 */ 339, 243, 262, 64, 113, 220, 221, 222, 223, 224, - /* 420 */ 80, 279, 351, 83, 80, 211, 355, 83, 286, 279, - /* 430 */ 81, 216, 217, 80, 181, 182, 83, 184, 185, 186, + /* 290 */ 194, 195, 196, 197, 198, 172, 173, 60, 61, 243, + /* 300 */ 47, 115, 65, 293, 243, 68, 69, 250, 211, 72, + /* 310 */ 73, 74, 14, 12, 13, 14, 2, 64, 20, 262, + /* 320 */ 185, 20, 293, 22, 81, 81, 12, 13, 14, 15, + /* 330 */ 16, 2, 271, 0, 81, 339, 279, 82, 263, 20, + /* 340 */ 279, 12, 13, 14, 15, 16, 271, 351, 47, 293, + /* 350 */ 289, 355, 251, 252, 293, 280, 103, 20, 297, 271, + /* 360 */ 271, 12, 13, 226, 93, 64, 113, 278, 280, 20, + /* 370 */ 309, 22, 283, 47, 313, 314, 315, 316, 317, 318, + /* 380 */ 211, 320, 81, 112, 323, 114, 115, 116, 327, 328, + /* 390 */ 64, 81, 247, 60, 61, 250, 47, 0, 65, 44, + /* 400 */ 339, 68, 69, 250, 103, 72, 73, 74, 0, 156, + /* 410 */ 324, 325, 351, 64, 113, 262, 355, 92, 21, 270, + /* 420 */ 35, 24, 25, 26, 27, 28, 29, 30, 31, 32, + /* 430 */ 81, 282, 279, 143, 181, 182, 81, 184, 185, 186, /* 440 */ 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, - /* 450 */ 197, 198, 103, 81, 57, 263, 116, 156, 3, 263, - /* 460 */ 298, 293, 113, 271, 211, 60, 61, 271, 271, 92, - /* 470 */ 65, 75, 280, 68, 69, 278, 280, 72, 73, 74, - /* 480 */ 283, 141, 181, 182, 144, 184, 185, 186, 187, 188, + /* 450 */ 197, 198, 103, 281, 211, 211, 271, 156, 263, 47, + /* 460 */ 288, 289, 113, 278, 211, 155, 271, 157, 283, 84, + /* 470 */ 247, 86, 87, 250, 89, 280, 257, 258, 93, 167, + /* 480 */ 168, 169, 181, 182, 0, 184, 185, 186, 187, 188, /* 490 */ 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, - /* 500 */ 243, 339, 162, 20, 164, 156, 288, 289, 12, 13, - /* 510 */ 14, 15, 16, 351, 14, 119, 120, 355, 12, 13, - /* 520 */ 20, 243, 167, 168, 169, 0, 20, 155, 22, 157, - /* 530 */ 181, 182, 250, 184, 185, 186, 187, 188, 189, 190, - /* 540 */ 191, 192, 193, 194, 195, 196, 197, 198, 264, 271, - /* 550 */ 293, 267, 250, 47, 243, 0, 250, 279, 243, 14, - /* 560 */ 18, 279, 20, 47, 262, 20, 272, 289, 262, 27, - /* 570 */ 64, 293, 30, 302, 250, 269, 298, 0, 82, 298, - /* 580 */ 64, 279, 57, 211, 243, 279, 262, 81, 310, 151, - /* 590 */ 48, 313, 314, 315, 316, 317, 318, 272, 320, 317, - /* 600 */ 45, 323, 272, 279, 293, 327, 328, 272, 293, 103, - /* 610 */ 172, 173, 250, 331, 332, 333, 334, 339, 336, 113, - /* 620 */ 339, 42, 43, 271, 262, 244, 1, 2, 145, 351, - /* 630 */ 278, 269, 351, 355, 293, 283, 355, 60, 61, 62, - /* 640 */ 63, 279, 65, 66, 67, 68, 69, 70, 71, 72, - /* 650 */ 73, 74, 75, 76, 77, 78, 14, 15, 16, 243, - /* 660 */ 118, 4, 156, 121, 122, 123, 124, 125, 126, 127, + /* 500 */ 115, 93, 20, 0, 243, 156, 216, 217, 24, 25, + /* 510 */ 26, 27, 28, 29, 30, 31, 32, 297, 12, 13, + /* 520 */ 112, 211, 114, 115, 116, 113, 20, 243, 22, 273, + /* 530 */ 181, 182, 273, 184, 185, 186, 187, 188, 189, 190, + /* 540 */ 191, 192, 193, 194, 195, 196, 197, 198, 45, 293, + /* 550 */ 14, 41, 293, 47, 293, 271, 20, 311, 20, 339, + /* 560 */ 18, 0, 20, 279, 308, 309, 243, 308, 309, 27, + /* 570 */ 64, 351, 30, 289, 41, 355, 320, 293, 243, 320, + /* 580 */ 55, 335, 12, 13, 14, 15, 16, 81, 257, 258, + /* 590 */ 48, 250, 82, 309, 181, 243, 22, 313, 314, 315, + /* 600 */ 316, 317, 318, 262, 320, 80, 271, 323, 83, 103, + /* 610 */ 269, 327, 328, 329, 279, 82, 293, 250, 57, 113, + /* 620 */ 279, 47, 139, 264, 289, 341, 267, 145, 293, 262, + /* 630 */ 346, 347, 311, 220, 221, 222, 223, 224, 12, 13, + /* 640 */ 14, 15, 16, 160, 309, 293, 279, 4, 313, 314, + /* 650 */ 315, 316, 317, 318, 64, 320, 335, 322, 0, 243, + /* 660 */ 118, 272, 156, 121, 122, 123, 124, 125, 126, 127, /* 670 */ 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, - /* 680 */ 138, 226, 243, 289, 243, 185, 20, 181, 182, 295, + /* 680 */ 138, 243, 199, 243, 58, 0, 272, 181, 182, 20, /* 690 */ 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, - /* 700 */ 194, 195, 196, 197, 198, 139, 18, 82, 243, 293, - /* 710 */ 271, 23, 259, 250, 261, 37, 272, 250, 279, 0, - /* 720 */ 20, 250, 298, 35, 36, 262, 160, 39, 289, 262, - /* 730 */ 185, 250, 293, 262, 293, 250, 12, 13, 14, 15, - /* 740 */ 16, 243, 279, 262, 56, 243, 279, 262, 243, 310, - /* 750 */ 279, 41, 313, 314, 315, 316, 317, 318, 293, 320, - /* 760 */ 279, 243, 323, 339, 279, 199, 327, 328, 329, 81, - /* 770 */ 12, 13, 14, 15, 16, 351, 271, 0, 250, 355, - /* 780 */ 341, 250, 58, 271, 279, 346, 347, 93, 250, 243, - /* 790 */ 262, 293, 280, 262, 289, 293, 85, 243, 293, 88, - /* 800 */ 262, 296, 243, 243, 243, 117, 82, 279, 243, 115, - /* 810 */ 279, 293, 93, 41, 90, 310, 58, 279, 313, 314, - /* 820 */ 315, 316, 317, 318, 58, 320, 209, 210, 250, 272, - /* 830 */ 21, 112, 271, 114, 115, 116, 148, 149, 150, 293, - /* 840 */ 279, 153, 260, 34, 271, 145, 158, 293, 90, 41, - /* 850 */ 289, 185, 293, 293, 293, 243, 283, 279, 293, 171, - /* 860 */ 22, 358, 174, 139, 176, 177, 178, 179, 180, 212, - /* 870 */ 93, 310, 0, 47, 313, 314, 315, 316, 317, 318, - /* 880 */ 319, 320, 321, 322, 160, 47, 243, 0, 210, 112, - /* 890 */ 82, 114, 115, 116, 22, 317, 41, 139, 85, 211, - /* 900 */ 85, 88, 64, 88, 85, 293, 243, 88, 41, 22, - /* 910 */ 332, 333, 334, 0, 336, 1, 2, 44, 160, 196, - /* 920 */ 197, 349, 47, 199, 200, 201, 202, 203, 204, 205, - /* 930 */ 206, 207, 208, 41, 271, 22, 293, 82, 228, 113, - /* 940 */ 81, 103, 279, 41, 41, 306, 41, 181, 41, 82, - /* 950 */ 91, 113, 289, 343, 81, 271, 293, 199, 200, 201, - /* 960 */ 202, 203, 204, 205, 206, 207, 208, 41, 243, 249, - /* 970 */ 41, 282, 41, 310, 82, 41, 313, 314, 315, 316, - /* 980 */ 317, 318, 47, 320, 82, 82, 323, 82, 113, 82, - /* 990 */ 327, 328, 329, 41, 156, 251, 271, 312, 337, 64, - /* 1000 */ 243, 352, 230, 41, 279, 352, 41, 352, 82, 340, - /* 1010 */ 347, 82, 213, 82, 289, 41, 82, 41, 293, 181, - /* 1020 */ 182, 308, 20, 250, 45, 307, 257, 47, 271, 300, - /* 1030 */ 154, 250, 250, 287, 82, 310, 279, 40, 313, 314, - /* 1040 */ 315, 316, 317, 318, 82, 320, 289, 82, 323, 139, - /* 1050 */ 293, 250, 327, 328, 329, 285, 82, 20, 82, 285, - /* 1060 */ 243, 245, 20, 338, 304, 245, 255, 310, 289, 255, - /* 1070 */ 313, 314, 315, 316, 317, 318, 20, 320, 20, 299, - /* 1080 */ 323, 297, 255, 297, 327, 328, 329, 255, 271, 279, - /* 1090 */ 20, 255, 243, 290, 255, 338, 279, 250, 255, 245, - /* 1100 */ 271, 271, 271, 250, 271, 271, 289, 245, 271, 64, - /* 1110 */ 293, 271, 271, 271, 293, 271, 271, 293, 253, 163, - /* 1120 */ 271, 253, 304, 253, 289, 303, 20, 310, 279, 250, - /* 1130 */ 313, 314, 315, 316, 317, 318, 279, 320, 289, 253, - /* 1140 */ 323, 219, 293, 243, 327, 328, 329, 298, 297, 293, - /* 1150 */ 290, 218, 348, 348, 293, 338, 147, 294, 312, 310, - /* 1160 */ 294, 225, 313, 314, 315, 316, 317, 318, 345, 320, - /* 1170 */ 293, 271, 344, 243, 214, 279, 210, 308, 20, 279, - /* 1180 */ 40, 232, 311, 229, 227, 81, 291, 294, 339, 289, - /* 1190 */ 342, 293, 293, 293, 330, 294, 142, 293, 298, 279, - /* 1200 */ 351, 271, 290, 353, 355, 267, 243, 279, 326, 279, - /* 1210 */ 310, 359, 354, 313, 314, 315, 316, 317, 318, 289, - /* 1220 */ 320, 253, 253, 293, 81, 275, 253, 250, 354, 245, - /* 1230 */ 305, 353, 261, 353, 271, 354, 243, 265, 301, 339, - /* 1240 */ 310, 254, 279, 313, 314, 315, 316, 317, 318, 241, - /* 1250 */ 320, 351, 289, 323, 265, 355, 293, 327, 328, 265, - /* 1260 */ 0, 0, 40, 0, 271, 72, 0, 47, 175, 175, - /* 1270 */ 47, 47, 279, 310, 47, 0, 313, 314, 315, 316, - /* 1280 */ 317, 318, 289, 320, 47, 47, 293, 243, 175, 0, - /* 1290 */ 175, 0, 47, 0, 47, 0, 243, 47, 0, 81, - /* 1300 */ 113, 160, 156, 310, 159, 0, 313, 314, 315, 316, - /* 1310 */ 317, 318, 0, 320, 152, 271, 323, 151, 0, 356, - /* 1320 */ 357, 328, 0, 279, 271, 44, 0, 0, 0, 0, - /* 1330 */ 0, 0, 279, 289, 0, 0, 0, 293, 0, 0, - /* 1340 */ 0, 0, 289, 0, 0, 0, 293, 0, 0, 40, - /* 1350 */ 243, 0, 0, 0, 310, 0, 0, 313, 314, 315, - /* 1360 */ 316, 317, 318, 310, 320, 0, 313, 314, 315, 316, - /* 1370 */ 317, 318, 22, 320, 0, 0, 0, 0, 271, 0, - /* 1380 */ 0, 14, 40, 37, 14, 0, 279, 0, 0, 0, - /* 1390 */ 243, 147, 0, 44, 350, 41, 289, 38, 37, 37, - /* 1400 */ 293, 44, 0, 296, 90, 37, 243, 0, 0, 0, - /* 1410 */ 357, 0, 0, 47, 45, 37, 45, 310, 271, 59, - /* 1420 */ 313, 314, 315, 316, 317, 318, 279, 320, 47, 47, - /* 1430 */ 37, 45, 37, 0, 271, 37, 289, 47, 0, 45, - /* 1440 */ 293, 0, 279, 0, 0, 22, 88, 47, 0, 0, - /* 1450 */ 47, 41, 289, 47, 243, 41, 293, 310, 22, 296, - /* 1460 */ 313, 314, 315, 316, 317, 318, 0, 320, 47, 322, - /* 1470 */ 48, 47, 47, 310, 47, 47, 313, 314, 315, 316, - /* 1480 */ 317, 318, 271, 320, 22, 0, 243, 22, 0, 47, - /* 1490 */ 279, 33, 0, 22, 22, 20, 0, 47, 0, 22, - /* 1500 */ 289, 0, 0, 45, 293, 0, 0, 296, 145, 51, - /* 1510 */ 52, 53, 54, 55, 271, 81, 243, 37, 145, 41, - /* 1520 */ 41, 310, 279, 140, 313, 314, 315, 316, 317, 318, - /* 1530 */ 215, 320, 289, 161, 82, 41, 293, 82, 80, 142, - /* 1540 */ 81, 83, 81, 44, 271, 81, 145, 44, 41, 82, - /* 1550 */ 81, 41, 279, 310, 41, 82, 313, 314, 315, 316, - /* 1560 */ 317, 318, 289, 320, 44, 82, 293, 243, 44, 82, - /* 1570 */ 41, 82, 47, 47, 47, 47, 2, 47, 47, 41, - /* 1580 */ 181, 44, 82, 310, 44, 81, 313, 314, 315, 316, - /* 1590 */ 317, 318, 81, 320, 82, 271, 81, 22, 140, 215, - /* 1600 */ 142, 183, 144, 279, 146, 209, 215, 243, 0, 37, - /* 1610 */ 91, 143, 44, 289, 82, 81, 81, 293, 82, 22, - /* 1620 */ 81, 140, 164, 81, 81, 44, 81, 47, 82, 81, - /* 1630 */ 81, 113, 82, 92, 310, 271, 47, 313, 314, 315, - /* 1640 */ 316, 317, 318, 279, 320, 81, 47, 243, 82, 82, - /* 1650 */ 81, 47, 81, 289, 47, 82, 47, 293, 81, 105, - /* 1660 */ 82, 105, 243, 81, 105, 105, 81, 22, 81, 93, - /* 1670 */ 47, 81, 22, 58, 310, 271, 59, 313, 314, 315, - /* 1680 */ 316, 317, 318, 279, 320, 47, 64, 41, 47, 79, - /* 1690 */ 271, 22, 64, 289, 47, 47, 47, 293, 279, 47, - /* 1700 */ 47, 47, 47, 47, 47, 47, 47, 47, 289, 47, - /* 1710 */ 0, 47, 293, 243, 310, 45, 37, 313, 314, 315, - /* 1720 */ 316, 317, 318, 0, 320, 243, 45, 47, 37, 310, - /* 1730 */ 0, 47, 313, 314, 315, 316, 317, 318, 45, 320, - /* 1740 */ 37, 271, 47, 45, 37, 0, 47, 46, 0, 279, - /* 1750 */ 0, 0, 21, 271, 22, 22, 22, 21, 20, 289, - /* 1760 */ 360, 279, 360, 293, 360, 243, 360, 360, 360, 360, - /* 1770 */ 360, 289, 360, 360, 360, 293, 243, 360, 360, 360, - /* 1780 */ 310, 360, 360, 313, 314, 315, 316, 317, 318, 360, - /* 1790 */ 320, 360, 310, 271, 360, 313, 314, 315, 316, 317, - /* 1800 */ 318, 279, 320, 360, 271, 360, 360, 360, 360, 360, - /* 1810 */ 360, 289, 279, 360, 360, 293, 360, 360, 360, 360, - /* 1820 */ 360, 360, 289, 360, 360, 360, 293, 243, 360, 360, - /* 1830 */ 360, 360, 310, 360, 360, 313, 314, 315, 316, 317, - /* 1840 */ 318, 360, 320, 310, 360, 360, 313, 314, 315, 316, - /* 1850 */ 317, 318, 360, 320, 360, 271, 360, 243, 360, 360, - /* 1860 */ 360, 360, 360, 279, 360, 360, 360, 360, 360, 360, - /* 1870 */ 360, 360, 360, 289, 360, 360, 360, 293, 360, 360, - /* 1880 */ 360, 360, 360, 360, 360, 271, 360, 243, 360, 360, - /* 1890 */ 360, 360, 360, 279, 310, 360, 360, 313, 314, 315, - /* 1900 */ 316, 317, 318, 289, 320, 360, 360, 293, 360, 360, - /* 1910 */ 360, 360, 360, 360, 360, 271, 360, 360, 360, 243, - /* 1920 */ 360, 360, 360, 279, 310, 360, 360, 313, 314, 315, - /* 1930 */ 316, 317, 318, 289, 320, 360, 360, 293, 360, 360, - /* 1940 */ 360, 360, 360, 12, 13, 360, 360, 271, 360, 360, - /* 1950 */ 360, 360, 243, 22, 310, 279, 360, 313, 314, 315, - /* 1960 */ 316, 317, 318, 360, 320, 289, 360, 360, 360, 293, - /* 1970 */ 360, 360, 360, 360, 360, 360, 360, 360, 47, 360, - /* 1980 */ 271, 360, 360, 360, 360, 243, 310, 360, 279, 313, - /* 1990 */ 314, 315, 316, 317, 318, 64, 320, 360, 289, 360, - /* 2000 */ 360, 360, 293, 360, 360, 360, 360, 360, 360, 360, - /* 2010 */ 360, 360, 360, 271, 360, 360, 250, 360, 360, 310, - /* 2020 */ 360, 279, 313, 314, 315, 316, 317, 318, 360, 320, - /* 2030 */ 360, 289, 360, 360, 103, 293, 360, 360, 360, 360, - /* 2040 */ 360, 360, 360, 360, 113, 279, 250, 360, 360, 360, - /* 2050 */ 360, 360, 310, 360, 360, 313, 314, 315, 316, 317, - /* 2060 */ 318, 360, 320, 360, 298, 360, 360, 360, 360, 360, - /* 2070 */ 360, 360, 360, 360, 360, 279, 360, 360, 360, 360, - /* 2080 */ 360, 360, 360, 317, 360, 360, 360, 156, 360, 360, - /* 2090 */ 360, 360, 360, 360, 298, 360, 360, 360, 332, 333, - /* 2100 */ 334, 360, 336, 360, 360, 339, 360, 360, 360, 360, - /* 2110 */ 360, 360, 181, 317, 360, 360, 360, 351, 360, 360, - /* 2120 */ 360, 355, 360, 192, 193, 194, 360, 360, 332, 333, - /* 2130 */ 334, 360, 336, 360, 360, 339, 360, 360, 360, 360, - /* 2140 */ 360, 360, 360, 360, 360, 360, 360, 351, 360, 360, - /* 2150 */ 360, 355, 360, 360, 360, 360, 360, 360, 360, 360, - /* 2160 */ 360, 360, 360, 360, 360, 360, 360, 360, 360, 360, - /* 2170 */ 360, 360, 360, 360, 360, 360, 360, 360, 360, 360, - /* 2180 */ 360, 360, 360, 360, + /* 700 */ 194, 195, 196, 197, 198, 250, 18, 250, 250, 293, + /* 710 */ 259, 23, 261, 250, 271, 57, 90, 262, 148, 262, + /* 720 */ 250, 185, 243, 35, 36, 20, 283, 39, 243, 243, + /* 730 */ 21, 293, 262, 293, 279, 37, 279, 279, 243, 269, + /* 740 */ 288, 289, 279, 34, 56, 60, 61, 62, 63, 279, + /* 750 */ 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, + /* 760 */ 75, 76, 77, 78, 250, 139, 271, 243, 272, 81, + /* 770 */ 272, 250, 293, 250, 279, 317, 262, 250, 293, 293, + /* 780 */ 317, 42, 43, 262, 289, 262, 160, 41, 293, 262, + /* 790 */ 332, 333, 334, 279, 336, 332, 333, 334, 243, 336, + /* 800 */ 279, 231, 279, 272, 309, 117, 279, 243, 313, 314, + /* 810 */ 315, 316, 317, 318, 145, 320, 41, 293, 323, 243, + /* 820 */ 243, 58, 327, 328, 329, 199, 200, 201, 202, 203, + /* 830 */ 204, 205, 206, 207, 208, 271, 148, 149, 150, 209, + /* 840 */ 210, 153, 347, 279, 243, 243, 158, 85, 293, 85, + /* 850 */ 88, 243, 88, 289, 0, 212, 272, 293, 85, 171, + /* 860 */ 250, 88, 174, 47, 176, 177, 178, 179, 180, 293, + /* 870 */ 293, 250, 262, 309, 0, 243, 22, 313, 314, 315, + /* 880 */ 316, 317, 318, 262, 320, 250, 250, 323, 0, 279, + /* 890 */ 185, 327, 328, 329, 293, 293, 22, 262, 262, 211, + /* 900 */ 279, 293, 338, 271, 1, 2, 85, 243, 210, 88, + /* 910 */ 22, 279, 301, 41, 279, 279, 47, 41, 196, 197, + /* 920 */ 244, 289, 41, 41, 358, 293, 41, 81, 41, 113, + /* 930 */ 41, 41, 260, 64, 41, 271, 250, 91, 349, 305, + /* 940 */ 243, 309, 343, 279, 181, 313, 314, 315, 316, 317, + /* 950 */ 318, 271, 320, 289, 82, 323, 41, 293, 82, 327, + /* 960 */ 328, 329, 249, 82, 82, 279, 251, 82, 271, 82, + /* 970 */ 338, 82, 82, 309, 228, 82, 279, 313, 314, 315, + /* 980 */ 316, 317, 318, 297, 320, 41, 289, 323, 243, 41, + /* 990 */ 293, 327, 328, 329, 297, 312, 282, 82, 41, 41, + /* 1000 */ 337, 352, 338, 317, 352, 230, 309, 352, 340, 41, + /* 1010 */ 313, 314, 315, 316, 317, 318, 271, 320, 332, 333, + /* 1020 */ 334, 213, 336, 307, 279, 339, 82, 20, 250, 45, + /* 1030 */ 82, 306, 47, 257, 289, 299, 339, 351, 293, 82, + /* 1040 */ 82, 355, 297, 250, 250, 154, 40, 287, 351, 139, + /* 1050 */ 82, 285, 355, 285, 309, 250, 20, 245, 313, 314, + /* 1060 */ 315, 316, 317, 318, 245, 320, 243, 20, 303, 255, + /* 1070 */ 289, 20, 255, 94, 95, 96, 97, 98, 99, 100, + /* 1080 */ 101, 102, 103, 104, 339, 106, 107, 108, 109, 110, + /* 1090 */ 111, 296, 20, 255, 271, 298, 351, 296, 255, 279, + /* 1100 */ 355, 20, 279, 290, 255, 255, 243, 250, 255, 245, + /* 1110 */ 271, 271, 289, 271, 64, 271, 293, 250, 271, 271, + /* 1120 */ 245, 293, 271, 271, 293, 303, 271, 271, 163, 271, + /* 1130 */ 253, 302, 309, 253, 271, 290, 313, 314, 315, 316, + /* 1140 */ 317, 318, 279, 320, 250, 253, 323, 279, 289, 253, + /* 1150 */ 327, 328, 289, 218, 294, 293, 293, 296, 142, 294, + /* 1160 */ 253, 253, 291, 293, 267, 243, 20, 294, 290, 219, + /* 1170 */ 225, 345, 309, 348, 279, 348, 313, 314, 315, 316, + /* 1180 */ 317, 318, 147, 320, 243, 279, 323, 214, 210, 20, + /* 1190 */ 327, 328, 293, 271, 293, 312, 294, 293, 293, 279, + /* 1200 */ 40, 279, 227, 229, 232, 307, 81, 81, 275, 250, + /* 1210 */ 344, 289, 271, 261, 330, 293, 253, 304, 265, 300, + /* 1220 */ 279, 342, 311, 254, 241, 245, 265, 265, 0, 0, + /* 1230 */ 289, 309, 40, 0, 293, 313, 314, 315, 316, 317, + /* 1240 */ 318, 0, 320, 326, 72, 359, 47, 47, 175, 47, + /* 1250 */ 309, 19, 47, 353, 313, 314, 315, 316, 317, 318, + /* 1260 */ 243, 320, 175, 354, 323, 33, 354, 353, 0, 328, + /* 1270 */ 12, 13, 354, 47, 353, 175, 0, 45, 356, 357, + /* 1280 */ 22, 47, 175, 51, 52, 53, 54, 55, 271, 0, + /* 1290 */ 47, 0, 243, 276, 47, 0, 279, 47, 0, 81, + /* 1300 */ 160, 159, 22, 113, 156, 47, 289, 0, 0, 152, + /* 1310 */ 293, 0, 80, 151, 0, 83, 44, 0, 0, 0, + /* 1320 */ 271, 0, 64, 0, 0, 276, 309, 47, 279, 0, + /* 1330 */ 313, 314, 315, 316, 317, 318, 0, 320, 289, 4, + /* 1340 */ 0, 0, 293, 0, 64, 0, 0, 0, 116, 0, + /* 1350 */ 0, 0, 0, 40, 19, 0, 243, 0, 309, 0, + /* 1360 */ 0, 103, 313, 314, 315, 316, 317, 318, 33, 320, + /* 1370 */ 0, 113, 0, 141, 37, 22, 144, 0, 0, 0, + /* 1380 */ 45, 0, 0, 103, 271, 50, 0, 14, 40, 14, + /* 1390 */ 55, 0, 279, 113, 162, 41, 164, 0, 0, 37, + /* 1400 */ 44, 38, 289, 0, 37, 44, 293, 147, 0, 243, + /* 1410 */ 0, 0, 0, 37, 156, 80, 0, 59, 83, 0, + /* 1420 */ 37, 243, 309, 0, 0, 45, 313, 314, 315, 316, + /* 1430 */ 317, 318, 47, 320, 47, 45, 156, 271, 37, 181, + /* 1440 */ 45, 47, 37, 47, 45, 279, 37, 0, 0, 271, + /* 1450 */ 192, 193, 194, 0, 276, 289, 0, 279, 90, 293, + /* 1460 */ 47, 181, 182, 350, 22, 0, 41, 289, 47, 47, + /* 1470 */ 0, 293, 88, 47, 243, 309, 47, 41, 47, 313, + /* 1480 */ 314, 315, 316, 317, 318, 47, 320, 309, 47, 22, + /* 1490 */ 0, 313, 314, 315, 316, 317, 318, 22, 320, 48, + /* 1500 */ 0, 22, 271, 0, 47, 22, 0, 276, 22, 33, + /* 1510 */ 279, 20, 0, 47, 145, 0, 22, 0, 0, 0, + /* 1520 */ 289, 45, 243, 357, 293, 161, 0, 51, 52, 53, + /* 1530 */ 54, 55, 37, 145, 142, 81, 81, 243, 0, 81, + /* 1540 */ 309, 82, 140, 143, 313, 314, 315, 316, 317, 318, + /* 1550 */ 271, 320, 145, 81, 140, 276, 80, 37, 279, 83, + /* 1560 */ 91, 81, 41, 82, 41, 271, 44, 82, 289, 81, + /* 1570 */ 22, 82, 293, 279, 41, 82, 41, 243, 81, 81, + /* 1580 */ 215, 44, 81, 289, 44, 81, 215, 293, 309, 82, + /* 1590 */ 41, 82, 313, 314, 315, 316, 317, 318, 44, 320, + /* 1600 */ 41, 44, 44, 309, 41, 271, 82, 313, 314, 315, + /* 1610 */ 316, 317, 318, 279, 320, 209, 140, 243, 142, 215, + /* 1620 */ 144, 82, 146, 289, 47, 2, 47, 293, 47, 47, + /* 1630 */ 47, 47, 41, 181, 81, 22, 243, 82, 82, 44, + /* 1640 */ 164, 81, 81, 309, 82, 271, 81, 313, 314, 315, + /* 1650 */ 316, 317, 318, 279, 320, 81, 44, 183, 92, 22, + /* 1660 */ 93, 81, 47, 289, 271, 47, 81, 293, 82, 82, + /* 1670 */ 81, 47, 279, 47, 82, 81, 47, 47, 82, 81, + /* 1680 */ 105, 82, 289, 309, 81, 81, 293, 313, 314, 315, + /* 1690 */ 316, 317, 318, 243, 320, 105, 47, 81, 81, 22, + /* 1700 */ 59, 113, 309, 58, 47, 105, 313, 314, 315, 316, + /* 1710 */ 317, 318, 105, 320, 64, 79, 41, 47, 47, 47, + /* 1720 */ 22, 271, 64, 47, 0, 243, 47, 47, 47, 279, + /* 1730 */ 47, 47, 47, 47, 47, 0, 47, 47, 47, 289, + /* 1740 */ 45, 37, 37, 293, 47, 45, 0, 45, 47, 37, + /* 1750 */ 0, 45, 37, 271, 47, 0, 47, 46, 0, 309, + /* 1760 */ 0, 279, 21, 313, 314, 315, 316, 317, 318, 22, + /* 1770 */ 320, 289, 21, 243, 22, 293, 22, 20, 360, 360, + /* 1780 */ 360, 360, 360, 360, 360, 360, 360, 360, 243, 360, + /* 1790 */ 360, 309, 360, 360, 360, 313, 314, 315, 316, 317, + /* 1800 */ 318, 271, 320, 360, 360, 360, 360, 360, 360, 279, + /* 1810 */ 360, 360, 360, 360, 360, 360, 271, 360, 360, 289, + /* 1820 */ 360, 243, 360, 293, 279, 360, 360, 360, 360, 360, + /* 1830 */ 360, 360, 360, 360, 289, 360, 243, 360, 293, 309, + /* 1840 */ 360, 360, 360, 313, 314, 315, 316, 317, 318, 271, + /* 1850 */ 320, 360, 360, 243, 309, 360, 360, 279, 313, 314, + /* 1860 */ 315, 316, 317, 318, 271, 320, 360, 289, 360, 360, + /* 1870 */ 360, 293, 279, 360, 360, 360, 360, 360, 360, 360, + /* 1880 */ 360, 271, 289, 360, 360, 360, 293, 309, 360, 279, + /* 1890 */ 360, 313, 314, 315, 316, 317, 318, 360, 320, 289, + /* 1900 */ 360, 360, 309, 293, 360, 360, 313, 314, 315, 316, + /* 1910 */ 317, 318, 360, 320, 360, 360, 243, 360, 360, 309, + /* 1920 */ 360, 360, 360, 313, 314, 315, 316, 317, 318, 360, + /* 1930 */ 320, 243, 360, 360, 360, 360, 360, 360, 360, 360, + /* 1940 */ 360, 360, 360, 360, 271, 360, 360, 360, 360, 360, + /* 1950 */ 360, 360, 279, 360, 360, 360, 360, 360, 360, 271, + /* 1960 */ 360, 360, 289, 360, 360, 360, 293, 279, 360, 360, + /* 1970 */ 360, 360, 360, 360, 360, 360, 360, 289, 360, 360, + /* 1980 */ 360, 293, 309, 243, 360, 360, 313, 314, 315, 316, + /* 1990 */ 317, 318, 360, 320, 360, 360, 360, 309, 360, 360, + /* 2000 */ 360, 313, 314, 315, 316, 317, 318, 360, 320, 360, + /* 2010 */ 360, 271, 360, 360, 360, 243, 360, 360, 360, 279, + /* 2020 */ 360, 360, 360, 360, 360, 360, 360, 360, 360, 289, + /* 2030 */ 360, 360, 360, 293, 360, 360, 360, 360, 360, 360, + /* 2040 */ 360, 360, 360, 271, 360, 360, 360, 360, 360, 309, + /* 2050 */ 360, 279, 360, 313, 314, 315, 316, 317, 318, 360, + /* 2060 */ 320, 289, 360, 243, 360, 293, 360, 360, 360, 360, + /* 2070 */ 360, 360, 250, 360, 360, 360, 360, 360, 360, 360, + /* 2080 */ 360, 309, 360, 360, 360, 313, 314, 315, 316, 317, + /* 2090 */ 318, 271, 320, 360, 360, 360, 360, 360, 360, 279, + /* 2100 */ 360, 279, 360, 360, 360, 360, 360, 360, 360, 289, + /* 2110 */ 360, 360, 360, 293, 360, 360, 360, 360, 360, 297, + /* 2120 */ 360, 360, 360, 360, 360, 360, 360, 360, 360, 309, + /* 2130 */ 360, 360, 360, 313, 314, 315, 316, 317, 318, 317, + /* 2140 */ 320, 360, 360, 360, 360, 360, 360, 360, 360, 360, + /* 2150 */ 360, 360, 360, 360, 332, 333, 334, 360, 336, 360, + /* 2160 */ 360, 339, 360, 360, 360, 360, 360, 360, 360, 360, + /* 2170 */ 360, 360, 360, 351, 360, 360, 360, 355, }; #define YY_SHIFT_COUNT (618) #define YY_SHIFT_MIN (0) -#define YY_SHIFT_MAX (1931) +#define YY_SHIFT_MAX (1760) static const unsigned short int yy_shift_ofst[] = { /* 0 */ 688, 0, 0, 48, 96, 96, 96, 96, 253, 253, /* 10 */ 96, 96, 301, 349, 506, 349, 349, 349, 349, 349, /* 20 */ 349, 349, 349, 349, 349, 349, 349, 349, 349, 349, - /* 30 */ 349, 349, 349, 349, 349, 349, 349, 349, 144, 144, - /* 40 */ 102, 102, 102, 1931, 1931, 1931, 1931, 372, 124, 214, - /* 50 */ 58, 58, 260, 260, 172, 214, 214, 58, 58, 58, - /* 60 */ 58, 58, 58, 58, 37, 58, 58, 186, 228, 259, - /* 70 */ 186, 58, 58, 186, 58, 186, 186, 259, 186, 58, - /* 80 */ 255, 542, 724, 758, 758, 189, 236, 838, 838, 838, - /* 90 */ 838, 838, 838, 838, 838, 838, 838, 838, 838, 838, - /* 100 */ 838, 838, 838, 838, 838, 838, 291, 126, 354, 354, - /* 110 */ 397, 516, 483, 483, 483, 525, 516, 313, 259, 186, - /* 120 */ 186, 259, 377, 343, 243, 243, 243, 243, 243, 243, - /* 130 */ 243, 340, 141, 405, 76, 195, 57, 215, 500, 545, - /* 140 */ 579, 179, 694, 700, 617, 678, 617, 455, 455, 455, - /* 150 */ 657, 666, 799, 1002, 979, 980, 876, 1002, 1002, 997, - /* 160 */ 910, 910, 1002, 1037, 1037, 1042, 37, 259, 37, 1056, - /* 170 */ 1058, 37, 1056, 37, 313, 1070, 37, 37, 1002, 37, - /* 180 */ 1037, 186, 186, 186, 186, 186, 186, 186, 186, 186, - /* 190 */ 186, 186, 1002, 1037, 1045, 1045, 1042, 255, 956, 259, - /* 200 */ 255, 1002, 1056, 255, 313, 1070, 255, 1106, 922, 933, - /* 210 */ 1045, 922, 933, 1045, 1045, 186, 936, 1009, 960, 799, - /* 220 */ 966, 313, 1158, 1140, 954, 957, 949, 954, 957, 954, - /* 230 */ 957, 1104, 933, 1045, 1045, 933, 1045, 1054, 313, 1070, - /* 240 */ 255, 377, 255, 313, 1143, 343, 1002, 255, 1037, 2152, - /* 250 */ 2152, 2152, 2152, 2152, 2152, 2152, 577, 1458, 231, 353, - /* 260 */ 3, 19, 316, 256, 496, 719, 777, 112, 112, 112, - /* 270 */ 112, 112, 112, 112, 112, 135, 438, 344, 396, 355, - /* 280 */ 625, 566, 642, 642, 642, 642, 555, 808, 711, 813, - /* 290 */ 815, 819, 872, 887, 913, 809, 855, 867, 892, 914, - /* 300 */ 723, 710, 772, 902, 766, 903, 873, 905, 907, 926, - /* 310 */ 929, 931, 826, 875, 934, 952, 962, 965, 974, 976, - /* 320 */ 859, 935, 1260, 1261, 1222, 1263, 1193, 1266, 1220, 1093, - /* 330 */ 1223, 1224, 1227, 1094, 1275, 1237, 1238, 1113, 1289, 1115, - /* 340 */ 1291, 1245, 1293, 1247, 1295, 1250, 1298, 1218, 1141, 1145, - /* 350 */ 1187, 1146, 1305, 1312, 1162, 1166, 1318, 1322, 1281, 1326, - /* 360 */ 1327, 1328, 1329, 1330, 1331, 1334, 1335, 1336, 1338, 1339, - /* 370 */ 1340, 1341, 1343, 1344, 1345, 1347, 1348, 1309, 1351, 1352, - /* 380 */ 1353, 1355, 1356, 1365, 1350, 1374, 1375, 1376, 1377, 1379, - /* 390 */ 1380, 1342, 1346, 1354, 1367, 1349, 1370, 1357, 1385, 1359, - /* 400 */ 1361, 1387, 1388, 1389, 1362, 1244, 1392, 1402, 1368, 1407, - /* 410 */ 1360, 1408, 1409, 1366, 1369, 1378, 1411, 1381, 1371, 1393, - /* 420 */ 1412, 1382, 1386, 1395, 1433, 1390, 1394, 1398, 1438, 1441, - /* 430 */ 1443, 1444, 1314, 1358, 1400, 1423, 1448, 1403, 1406, 1421, - /* 440 */ 1424, 1410, 1414, 1425, 1427, 1428, 1449, 1436, 1466, 1462, - /* 450 */ 1422, 1485, 1465, 1442, 1488, 1471, 1492, 1472, 1475, 1496, - /* 460 */ 1363, 1450, 1498, 1372, 1477, 1373, 1397, 1501, 1502, 1505, - /* 470 */ 1401, 1506, 1434, 1480, 1383, 1478, 1479, 1315, 1452, 1494, - /* 480 */ 1455, 1459, 1461, 1464, 1467, 1507, 1499, 1503, 1469, 1510, - /* 490 */ 1384, 1473, 1483, 1520, 1396, 1513, 1524, 1487, 1529, 1391, - /* 500 */ 1489, 1525, 1526, 1527, 1528, 1530, 1531, 1489, 1574, 1399, - /* 510 */ 1538, 1500, 1504, 1512, 1537, 1511, 1515, 1540, 1575, 1418, - /* 520 */ 1534, 1532, 1536, 1535, 1539, 1468, 1542, 1608, 1572, 1481, - /* 530 */ 1543, 1519, 1568, 1581, 1545, 1546, 1548, 1597, 1549, 1541, - /* 540 */ 1550, 1580, 1589, 1564, 1566, 1599, 1569, 1567, 1604, 1571, - /* 550 */ 1573, 1607, 1577, 1578, 1609, 1582, 1554, 1556, 1559, 1560, - /* 560 */ 1645, 1576, 1585, 1587, 1623, 1590, 1518, 1650, 1617, 1615, - /* 570 */ 1638, 1622, 1610, 1646, 1641, 1647, 1648, 1649, 1652, 1669, - /* 580 */ 1653, 1654, 1628, 1410, 1655, 1414, 1656, 1657, 1658, 1659, - /* 590 */ 1660, 1662, 1710, 1664, 1670, 1679, 1723, 1680, 1681, 1691, - /* 600 */ 1730, 1684, 1693, 1703, 1750, 1695, 1698, 1707, 1745, 1699, - /* 610 */ 1701, 1748, 1751, 1732, 1731, 1733, 1734, 1736, 1738, + /* 30 */ 349, 349, 349, 349, 349, 349, 349, 349, 349, 243, + /* 40 */ 243, 5, 5, 5, 1258, 1258, 1258, 310, 97, 169, + /* 50 */ 85, 85, 51, 51, 244, 169, 169, 85, 85, 85, + /* 60 */ 85, 85, 85, 85, 78, 85, 85, 159, 319, 159, + /* 70 */ 85, 85, 159, 337, 85, 159, 159, 319, 159, 85, + /* 80 */ 207, 542, 14, 626, 626, 230, 237, 1280, 1280, 1280, + /* 90 */ 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, + /* 100 */ 1280, 1280, 1280, 1280, 1280, 1280, 385, 227, 298, 298, + /* 110 */ 561, 326, 658, 482, 482, 482, 326, 538, 319, 159, + /* 120 */ 159, 319, 325, 590, 979, 979, 979, 979, 979, 979, + /* 130 */ 979, 1232, 397, 333, 570, 413, 42, 290, 135, 536, + /* 140 */ 739, 574, 186, 669, 630, 698, 630, 137, 137, 137, + /* 150 */ 643, 705, 808, 1007, 984, 985, 891, 1007, 1007, 1006, + /* 160 */ 910, 910, 1007, 1036, 1036, 1047, 78, 319, 78, 1051, + /* 170 */ 1072, 78, 1051, 78, 538, 1081, 78, 78, 1007, 78, + /* 180 */ 1036, 159, 159, 159, 159, 159, 159, 159, 159, 159, + /* 190 */ 159, 159, 1007, 1036, 1050, 1050, 1047, 207, 965, 319, + /* 200 */ 207, 1007, 1051, 207, 538, 1081, 207, 935, 1050, 935, + /* 210 */ 1050, 1016, 538, 1081, 207, 325, 207, 538, 1146, 950, + /* 220 */ 935, 1050, 1050, 950, 935, 1050, 1050, 159, 945, 1035, + /* 230 */ 973, 808, 978, 538, 1169, 1160, 974, 975, 972, 974, + /* 240 */ 975, 974, 975, 1125, 1126, 590, 1007, 207, 1036, 2178, + /* 250 */ 2178, 2178, 2178, 2178, 2178, 2178, 685, 1476, 484, 1335, + /* 260 */ 32, 2, 314, 329, 28, 255, 408, 212, 212, 212, + /* 270 */ 212, 212, 212, 212, 212, 271, 123, 525, 82, 312, + /* 280 */ 9, 483, 19, 19, 19, 19, 503, 510, 762, 764, + /* 290 */ 773, 821, 854, 874, 888, 709, 533, 872, 876, 881, + /* 300 */ 882, 885, 887, 412, 816, 889, 903, 722, 746, 775, + /* 310 */ 890, 763, 893, 355, 915, 944, 948, 957, 958, 968, + /* 320 */ 846, 869, 1228, 1229, 1192, 1233, 1172, 1241, 1199, 1073, + /* 330 */ 1200, 1202, 1205, 1087, 1268, 1226, 1234, 1100, 1276, 1107, + /* 340 */ 1289, 1243, 1291, 1247, 1295, 1250, 1298, 1218, 1140, 1142, + /* 350 */ 1190, 1148, 1307, 1308, 1157, 1162, 1311, 1314, 1272, 1317, + /* 360 */ 1318, 1319, 1321, 1323, 1324, 1329, 1336, 1340, 1341, 1343, + /* 370 */ 1345, 1346, 1347, 1349, 1350, 1351, 1352, 1313, 1355, 1357, + /* 380 */ 1359, 1360, 1370, 1372, 1353, 1377, 1378, 1379, 1381, 1382, + /* 390 */ 1386, 1348, 1337, 1354, 1373, 1356, 1375, 1361, 1391, 1363, + /* 400 */ 1362, 1397, 1398, 1403, 1367, 1260, 1408, 1410, 1376, 1411, + /* 410 */ 1358, 1412, 1416, 1385, 1380, 1383, 1419, 1387, 1390, 1401, + /* 420 */ 1423, 1394, 1395, 1405, 1424, 1396, 1399, 1409, 1447, 1448, + /* 430 */ 1453, 1456, 1368, 1384, 1413, 1442, 1465, 1421, 1422, 1426, + /* 440 */ 1429, 1425, 1436, 1431, 1438, 1441, 1470, 1467, 1490, 1475, + /* 450 */ 1451, 1500, 1479, 1457, 1503, 1483, 1506, 1486, 1491, 1512, + /* 460 */ 1369, 1466, 1515, 1364, 1494, 1388, 1392, 1517, 1518, 1519, + /* 470 */ 1407, 1526, 1454, 1495, 1402, 1455, 1459, 1458, 1400, 1472, + /* 480 */ 1538, 1520, 1414, 1480, 1469, 1522, 1521, 1523, 1365, 1481, + /* 490 */ 1485, 1488, 1548, 1533, 1489, 1497, 1498, 1501, 1493, 1535, + /* 500 */ 1537, 1540, 1504, 1549, 1371, 1507, 1509, 1554, 1406, 1559, + /* 510 */ 1557, 1558, 1524, 1563, 1404, 1539, 1577, 1579, 1581, 1582, + /* 520 */ 1583, 1584, 1539, 1623, 1452, 1591, 1553, 1555, 1560, 1556, + /* 530 */ 1561, 1562, 1595, 1565, 1574, 1612, 1613, 1474, 1580, 1566, + /* 540 */ 1586, 1615, 1618, 1585, 1587, 1624, 1589, 1592, 1626, 1594, + /* 550 */ 1596, 1629, 1598, 1599, 1630, 1603, 1575, 1590, 1600, 1607, + /* 560 */ 1637, 1567, 1604, 1616, 1649, 1617, 1588, 1677, 1641, 1645, + /* 570 */ 1657, 1650, 1636, 1675, 1670, 1671, 1672, 1676, 1679, 1698, + /* 580 */ 1680, 1681, 1658, 1425, 1683, 1436, 1684, 1685, 1686, 1687, + /* 590 */ 1689, 1690, 1724, 1691, 1695, 1704, 1735, 1697, 1700, 1705, + /* 600 */ 1746, 1701, 1702, 1712, 1750, 1707, 1706, 1715, 1755, 1709, + /* 610 */ 1711, 1758, 1760, 1747, 1741, 1752, 1754, 1751, 1757, }; #define YY_REDUCE_COUNT (255) -#define YY_REDUCE_MIN (-324) -#define YY_REDUCE_MAX (1796) +#define YY_REDUCE_MIN (-332) +#define YY_REDUCE_MAX (1822) static const short yy_reduce_ofst[] = { - /* 0 */ 162, -241, 278, 439, 663, 725, 757, 817, 849, 900, - /* 10 */ -178, 930, 561, 963, 993, 505, 1044, 1053, 1107, 1147, - /* 20 */ 1163, 1211, 1243, 1273, 1324, 1364, 1404, 1419, 1470, 1482, - /* 30 */ 1522, 1533, 1584, 1614, 1644, 1676, 1709, 1742, 1766, 1796, - /* 40 */ 282, -213, 578, -270, -266, -264, -176, -284, 281, 424, - /* 50 */ 306, 362, -91, -46, -255, -192, 71, -226, -199, 150, - /* 60 */ 302, 324, 463, 467, -1, 471, 485, 28, 45, -190, - /* 70 */ 56, 528, 531, 197, 481, 192, 352, 100, 196, 538, - /* 80 */ -225, -242, -324, -324, -324, -64, -48, 77, 168, 257, - /* 90 */ 311, 315, 341, 416, 441, 465, 498, 502, 518, 546, - /* 100 */ 554, 559, 560, 565, 612, 643, 97, -247, 75, 140, - /* 110 */ -125, -232, -252, -204, -37, -100, 106, 142, 394, 512, - /* 120 */ 573, 218, 284, 453, -261, 294, 325, 330, 335, 444, - /* 130 */ 557, 271, 381, 582, 503, 572, 639, 610, 684, 684, - /* 140 */ 720, 744, 689, 685, 661, 661, 661, 649, 653, 655, - /* 150 */ 669, 684, 713, 773, 718, 769, 729, 781, 782, 746, - /* 160 */ 770, 774, 801, 816, 820, 760, 811, 779, 814, 784, - /* 170 */ 780, 827, 786, 832, 810, 803, 836, 839, 847, 843, - /* 180 */ 854, 829, 830, 831, 833, 834, 837, 840, 841, 842, - /* 190 */ 844, 845, 853, 862, 821, 824, 818, 865, 822, 835, - /* 200 */ 868, 879, 851, 870, 857, 860, 886, 846, 804, 863, - /* 210 */ 856, 805, 866, 861, 877, 684, 823, 828, 848, 869, - /* 220 */ 661, 896, 871, 864, 858, 850, 852, 874, 878, 881, - /* 230 */ 880, 882, 893, 898, 899, 901, 904, 895, 920, 912, - /* 240 */ 968, 938, 969, 928, 950, 971, 977, 973, 984, 937, - /* 250 */ 925, 972, 989, 994, 987, 1008, + /* 0 */ -232, -240, 61, 284, 495, 564, 632, 664, 697, 745, + /* 10 */ 823, 863, -151, 922, 941, 1017, 1049, 1113, 1166, 1178, + /* 20 */ 335, 1231, 1279, 1294, 1334, 1374, 1393, 1450, 1482, 1530, + /* 30 */ 1545, 1578, 1593, 1610, 1673, 1688, 1740, 1772, 1820, 686, + /* 40 */ 1822, -200, 458, 463, -272, 256, 259, -288, -241, 220, + /* 50 */ 341, 470, -244, -122, -332, -254, -4, 57, 153, 367, + /* 60 */ 455, 514, 521, 523, -118, 527, 610, -212, -89, -141, + /* 70 */ 621, 635, 89, -225, 636, 75, 185, 172, 195, 457, + /* 80 */ -43, -109, 86, 86, 86, -142, -84, 10, 29, 56, + /* 90 */ 261, 323, 352, 416, 438, 440, 479, 485, 486, 524, + /* 100 */ 555, 576, 577, 601, 602, 608, 149, 101, 145, 223, + /* 110 */ -72, 219, -229, -190, 246, 321, 331, -197, -289, 88, + /* 120 */ 443, 452, 359, 451, -220, 389, 414, 496, 498, 531, + /* 130 */ 584, 611, 676, 672, 566, 589, 634, 599, 680, 680, + /* 140 */ 713, 715, 714, 683, 663, 663, 663, 649, 652, 655, + /* 150 */ 668, 680, 716, 778, 725, 776, 736, 793, 794, 760, + /* 160 */ 766, 768, 805, 812, 819, 765, 814, 781, 817, 795, + /* 170 */ 797, 838, 801, 843, 820, 813, 849, 850, 857, 853, + /* 180 */ 864, 839, 840, 842, 844, 847, 848, 851, 852, 855, + /* 190 */ 856, 858, 867, 875, 828, 831, 822, 877, 829, 859, + /* 200 */ 880, 894, 861, 892, 868, 845, 896, 860, 862, 865, + /* 210 */ 870, 871, 895, 878, 907, 897, 908, 906, 883, 825, + /* 220 */ 873, 899, 901, 827, 902, 904, 905, 680, 826, 866, + /* 230 */ 879, 898, 663, 920, 911, 884, 909, 900, 886, 912, + /* 240 */ 914, 918, 921, 917, 933, 952, 959, 963, 980, 919, + /* 250 */ 913, 953, 961, 962, 969, 983, }; static const YYACTIONTYPE yy_default[] = { /* 0 */ 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, @@ -765,7 +766,7 @@ static const YYACTIONTYPE yy_default[] = { /* 80 */ 1427, 1567, 1360, 1734, 1360, 1360, 1360, 1360, 1360, 1360, /* 90 */ 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, /* 100 */ 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, - /* 110 */ 1429, 1360, 1745, 1745, 1745, 1427, 1360, 1360, 1360, 1360, + /* 110 */ 1429, 1360, 1427, 1745, 1745, 1745, 1360, 1360, 1360, 1360, /* 120 */ 1360, 1360, 1523, 1360, 1360, 1360, 1360, 1360, 1360, 1360, /* 130 */ 1360, 1603, 1360, 1360, 1811, 1360, 1609, 1769, 1360, 1360, /* 140 */ 1360, 1360, 1476, 1761, 1737, 1751, 1738, 1796, 1796, 1796, @@ -774,18 +775,18 @@ static const YYACTIONTYPE yy_default[] = { /* 170 */ 1360, 1429, 1360, 1429, 1360, 1360, 1429, 1429, 1360, 1429, /* 180 */ 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, /* 190 */ 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1427, 1605, 1360, - /* 200 */ 1427, 1360, 1360, 1427, 1360, 1360, 1427, 1360, 1776, 1774, - /* 210 */ 1360, 1776, 1774, 1360, 1360, 1360, 1788, 1784, 1767, 1765, - /* 220 */ 1751, 1360, 1360, 1360, 1802, 1798, 1814, 1802, 1798, 1802, - /* 230 */ 1798, 1360, 1774, 1360, 1360, 1774, 1360, 1580, 1360, 1360, - /* 240 */ 1427, 1360, 1427, 1360, 1492, 1360, 1360, 1427, 1360, 1597, + /* 200 */ 1427, 1360, 1360, 1427, 1360, 1360, 1427, 1774, 1360, 1774, + /* 210 */ 1360, 1580, 1360, 1360, 1427, 1360, 1427, 1360, 1360, 1776, + /* 220 */ 1774, 1360, 1360, 1776, 1774, 1360, 1360, 1360, 1788, 1784, + /* 230 */ 1767, 1765, 1751, 1360, 1360, 1360, 1802, 1798, 1814, 1802, + /* 240 */ 1798, 1802, 1798, 1360, 1492, 1360, 1360, 1427, 1360, 1597, /* 250 */ 1611, 1526, 1526, 1526, 1430, 1365, 1360, 1360, 1360, 1360, /* 260 */ 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1678, 1787, 1786, /* 270 */ 1710, 1709, 1708, 1706, 1677, 1488, 1360, 1360, 1360, 1360, /* 280 */ 1360, 1360, 1671, 1672, 1670, 1669, 1360, 1360, 1360, 1360, - /* 290 */ 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1735, - /* 300 */ 1360, 1799, 1803, 1360, 1360, 1360, 1654, 1360, 1360, 1360, - /* 310 */ 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, + /* 290 */ 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, + /* 300 */ 1360, 1360, 1360, 1360, 1360, 1360, 1735, 1360, 1799, 1803, + /* 310 */ 1360, 1360, 1360, 1654, 1360, 1360, 1360, 1360, 1360, 1360, /* 320 */ 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, /* 330 */ 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, /* 340 */ 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, @@ -801,13 +802,13 @@ static const YYACTIONTYPE yy_default[] = { /* 440 */ 1360, 1457, 1456, 1360, 1360, 1360, 1360, 1360, 1360, 1360, /* 450 */ 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, /* 460 */ 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, - /* 470 */ 1360, 1360, 1360, 1360, 1360, 1758, 1768, 1360, 1360, 1360, - /* 480 */ 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1654, 1360, 1785, - /* 490 */ 1360, 1744, 1740, 1360, 1360, 1736, 1360, 1360, 1797, 1360, - /* 500 */ 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1730, 1360, - /* 510 */ 1703, 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1665, - /* 520 */ 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, - /* 530 */ 1360, 1360, 1653, 1360, 1694, 1360, 1360, 1360, 1360, 1360, + /* 470 */ 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, + /* 480 */ 1360, 1360, 1360, 1360, 1360, 1360, 1758, 1768, 1360, 1360, + /* 490 */ 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, + /* 500 */ 1360, 1654, 1360, 1785, 1360, 1744, 1740, 1360, 1360, 1736, + /* 510 */ 1653, 1360, 1360, 1797, 1360, 1360, 1360, 1360, 1360, 1360, + /* 520 */ 1360, 1360, 1360, 1730, 1360, 1703, 1694, 1360, 1360, 1360, + /* 530 */ 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1665, 1360, 1360, /* 540 */ 1360, 1360, 1360, 1520, 1360, 1360, 1360, 1360, 1360, 1360, /* 550 */ 1360, 1360, 1360, 1360, 1360, 1360, 1505, 1503, 1502, 1501, /* 560 */ 1360, 1498, 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, @@ -1438,7 +1439,7 @@ static const char *const yyTokenName[] = { /* 273 */ "signed_literal", /* 274 */ "create_subtable_clause", /* 275 */ "specific_tags_opt", - /* 276 */ "literal_list", + /* 276 */ "expression_list", /* 277 */ "drop_table_clause", /* 278 */ "col_name_list", /* 279 */ "table_name", @@ -1458,21 +1459,21 @@ static const char *const yyTokenName[] = { /* 293 */ "duration_literal", /* 294 */ "sliding_opt", /* 295 */ "func", - /* 296 */ "expression_list", - /* 297 */ "topic_name", - /* 298 */ "query_expression", - /* 299 */ "cgroup_name", - /* 300 */ "analyze_opt", - /* 301 */ "explain_options", - /* 302 */ "agg_func_opt", - /* 303 */ "bufsize_opt", - /* 304 */ "stream_name", - /* 305 */ "stream_options", - /* 306 */ "into_opt", - /* 307 */ "dnode_list", - /* 308 */ "where_clause_opt", - /* 309 */ "signed", - /* 310 */ "literal_func", + /* 296 */ "topic_name", + /* 297 */ "query_expression", + /* 298 */ "cgroup_name", + /* 299 */ "analyze_opt", + /* 300 */ "explain_options", + /* 301 */ "agg_func_opt", + /* 302 */ "bufsize_opt", + /* 303 */ "stream_name", + /* 304 */ "stream_options", + /* 305 */ "into_opt", + /* 306 */ "dnode_list", + /* 307 */ "where_clause_opt", + /* 308 */ "signed", + /* 309 */ "literal_func", + /* 310 */ "literal_list", /* 311 */ "table_alias", /* 312 */ "column_alias", /* 313 */ "expression", @@ -1654,7 +1655,7 @@ static const char *const yyRuleName[] = { /* 122 */ "alter_table_clause ::= full_table_name SET TAG column_name NK_EQ signed_literal", /* 123 */ "multi_create_clause ::= create_subtable_clause", /* 124 */ "multi_create_clause ::= multi_create_clause create_subtable_clause", - /* 125 */ "create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_tags_opt TAGS NK_LP literal_list NK_RP table_options", + /* 125 */ "create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_tags_opt TAGS NK_LP expression_list NK_RP table_options", /* 126 */ "multi_drop_clause ::= drop_table_clause", /* 127 */ "multi_drop_clause ::= multi_drop_clause drop_table_clause", /* 128 */ "drop_table_clause ::= exists_opt full_table_name", @@ -2132,13 +2133,13 @@ static void yy_destructor( case 293: /* duration_literal */ case 294: /* sliding_opt */ case 295: /* func */ - case 298: /* query_expression */ - case 301: /* explain_options */ - case 305: /* stream_options */ - case 306: /* into_opt */ - case 308: /* where_clause_opt */ - case 309: /* signed */ - case 310: /* literal_func */ + case 297: /* query_expression */ + case 300: /* explain_options */ + case 304: /* stream_options */ + case 305: /* into_opt */ + case 307: /* where_clause_opt */ + case 308: /* signed */ + case 309: /* literal_func */ case 313: /* expression */ case 314: /* pseudo_column */ case 315: /* column_reference */ @@ -2174,7 +2175,7 @@ static void yy_destructor( case 241: /* account_options */ case 242: /* alter_account_options */ case 244: /* alter_account_option */ - case 303: /* bufsize_opt */ + case 302: /* bufsize_opt */ { } @@ -2188,9 +2189,9 @@ static void yy_destructor( case 279: /* table_name */ case 289: /* function_name */ case 290: /* index_name */ - case 297: /* topic_name */ - case 299: /* cgroup_name */ - case 304: /* stream_name */ + case 296: /* topic_name */ + case 298: /* cgroup_name */ + case 303: /* stream_name */ case 311: /* table_alias */ case 312: /* column_alias */ case 318: /* star_func */ @@ -2209,8 +2210,8 @@ static void yy_destructor( break; case 253: /* not_exists_opt */ case 255: /* exists_opt */ - case 300: /* analyze_opt */ - case 302: /* agg_func_opt */ + case 299: /* analyze_opt */ + case 301: /* agg_func_opt */ case 340: /* set_quantifier_opt */ { @@ -2225,12 +2226,12 @@ static void yy_destructor( case 267: /* tags_def */ case 268: /* multi_drop_clause */ case 275: /* specific_tags_opt */ - case 276: /* literal_list */ + case 276: /* expression_list */ case 278: /* col_name_list */ case 281: /* func_name_list */ case 292: /* func_list */ - case 296: /* expression_list */ - case 307: /* dnode_list */ + case 306: /* dnode_list */ + case 310: /* literal_list */ case 319: /* star_func_para_list */ case 321: /* other_para_list */ case 341: /* select_list */ @@ -2700,7 +2701,7 @@ static const struct { { 269, -6 }, /* (122) alter_table_clause ::= full_table_name SET TAG column_name NK_EQ signed_literal */ { 266, -1 }, /* (123) multi_create_clause ::= create_subtable_clause */ { 266, -2 }, /* (124) multi_create_clause ::= multi_create_clause create_subtable_clause */ - { 274, -10 }, /* (125) create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_tags_opt TAGS NK_LP literal_list NK_RP table_options */ + { 274, -10 }, /* (125) create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_tags_opt TAGS NK_LP expression_list NK_RP table_options */ { 268, -1 }, /* (126) multi_drop_clause ::= drop_table_clause */ { 268, -2 }, /* (127) multi_drop_clause ::= multi_drop_clause drop_table_clause */ { 277, -2 }, /* (128) drop_table_clause ::= exists_opt full_table_name */ @@ -2807,27 +2808,27 @@ static const struct { { 240, -2 }, /* (229) cmd ::= DESCRIBE full_table_name */ { 240, -3 }, /* (230) cmd ::= RESET QUERY CACHE */ { 240, -4 }, /* (231) cmd ::= EXPLAIN analyze_opt explain_options query_expression */ - { 300, 0 }, /* (232) analyze_opt ::= */ - { 300, -1 }, /* (233) analyze_opt ::= ANALYZE */ - { 301, 0 }, /* (234) explain_options ::= */ - { 301, -3 }, /* (235) explain_options ::= explain_options VERBOSE NK_BOOL */ - { 301, -3 }, /* (236) explain_options ::= explain_options RATIO NK_FLOAT */ + { 299, 0 }, /* (232) analyze_opt ::= */ + { 299, -1 }, /* (233) analyze_opt ::= ANALYZE */ + { 300, 0 }, /* (234) explain_options ::= */ + { 300, -3 }, /* (235) explain_options ::= explain_options VERBOSE NK_BOOL */ + { 300, -3 }, /* (236) explain_options ::= explain_options RATIO NK_FLOAT */ { 240, -6 }, /* (237) cmd ::= COMPACT VNODES IN NK_LP integer_list NK_RP */ { 240, -10 }, /* (238) cmd ::= CREATE agg_func_opt FUNCTION not_exists_opt function_name AS NK_STRING OUTPUTTYPE type_name bufsize_opt */ { 240, -4 }, /* (239) cmd ::= DROP FUNCTION exists_opt function_name */ - { 302, 0 }, /* (240) agg_func_opt ::= */ - { 302, -1 }, /* (241) agg_func_opt ::= AGGREGATE */ - { 303, 0 }, /* (242) bufsize_opt ::= */ - { 303, -2 }, /* (243) bufsize_opt ::= BUFSIZE NK_INTEGER */ + { 301, 0 }, /* (240) agg_func_opt ::= */ + { 301, -1 }, /* (241) agg_func_opt ::= AGGREGATE */ + { 302, 0 }, /* (242) bufsize_opt ::= */ + { 302, -2 }, /* (243) bufsize_opt ::= BUFSIZE NK_INTEGER */ { 240, -8 }, /* (244) cmd ::= CREATE STREAM not_exists_opt stream_name stream_options into_opt AS query_expression */ { 240, -4 }, /* (245) cmd ::= DROP STREAM exists_opt stream_name */ - { 306, 0 }, /* (246) into_opt ::= */ - { 306, -2 }, /* (247) into_opt ::= INTO full_table_name */ - { 305, 0 }, /* (248) stream_options ::= */ - { 305, -3 }, /* (249) stream_options ::= stream_options TRIGGER AT_ONCE */ - { 305, -3 }, /* (250) stream_options ::= stream_options TRIGGER WINDOW_CLOSE */ - { 305, -4 }, /* (251) stream_options ::= stream_options TRIGGER MAX_DELAY duration_literal */ - { 305, -3 }, /* (252) stream_options ::= stream_options WATERMARK duration_literal */ + { 305, 0 }, /* (246) into_opt ::= */ + { 305, -2 }, /* (247) into_opt ::= INTO full_table_name */ + { 304, 0 }, /* (248) stream_options ::= */ + { 304, -3 }, /* (249) stream_options ::= stream_options TRIGGER AT_ONCE */ + { 304, -3 }, /* (250) stream_options ::= stream_options TRIGGER WINDOW_CLOSE */ + { 304, -4 }, /* (251) stream_options ::= stream_options TRIGGER MAX_DELAY duration_literal */ + { 304, -3 }, /* (252) stream_options ::= stream_options WATERMARK duration_literal */ { 240, -3 }, /* (253) cmd ::= KILL CONNECTION NK_INTEGER */ { 240, -3 }, /* (254) cmd ::= KILL QUERY NK_INTEGER */ { 240, -3 }, /* (255) cmd ::= KILL TRANSACTION NK_INTEGER */ @@ -2835,8 +2836,8 @@ static const struct { { 240, -4 }, /* (257) cmd ::= MERGE VGROUP NK_INTEGER NK_INTEGER */ { 240, -4 }, /* (258) cmd ::= REDISTRIBUTE VGROUP NK_INTEGER dnode_list */ { 240, -3 }, /* (259) cmd ::= SPLIT VGROUP NK_INTEGER */ - { 307, -2 }, /* (260) dnode_list ::= DNODE NK_INTEGER */ - { 307, -3 }, /* (261) dnode_list ::= dnode_list DNODE NK_INTEGER */ + { 306, -2 }, /* (260) dnode_list ::= DNODE NK_INTEGER */ + { 306, -3 }, /* (261) dnode_list ::= dnode_list DNODE NK_INTEGER */ { 240, -3 }, /* (262) cmd ::= SYNCDB db_name REPLICA */ { 240, -4 }, /* (263) cmd ::= DELETE FROM full_table_name where_clause_opt */ { 240, -1 }, /* (264) cmd ::= query_expression */ @@ -2849,12 +2850,12 @@ static const struct { { 243, -1 }, /* (271) literal ::= NULL */ { 243, -1 }, /* (272) literal ::= NK_QUESTION */ { 293, -1 }, /* (273) duration_literal ::= NK_VARIABLE */ - { 309, -1 }, /* (274) signed ::= NK_INTEGER */ - { 309, -2 }, /* (275) signed ::= NK_PLUS NK_INTEGER */ - { 309, -2 }, /* (276) signed ::= NK_MINUS NK_INTEGER */ - { 309, -1 }, /* (277) signed ::= NK_FLOAT */ - { 309, -2 }, /* (278) signed ::= NK_PLUS NK_FLOAT */ - { 309, -2 }, /* (279) signed ::= NK_MINUS NK_FLOAT */ + { 308, -1 }, /* (274) signed ::= NK_INTEGER */ + { 308, -2 }, /* (275) signed ::= NK_PLUS NK_INTEGER */ + { 308, -2 }, /* (276) signed ::= NK_MINUS NK_INTEGER */ + { 308, -1 }, /* (277) signed ::= NK_FLOAT */ + { 308, -2 }, /* (278) signed ::= NK_PLUS NK_FLOAT */ + { 308, -2 }, /* (279) signed ::= NK_MINUS NK_FLOAT */ { 273, -1 }, /* (280) signed_literal ::= signed */ { 273, -1 }, /* (281) signed_literal ::= NK_STRING */ { 273, -1 }, /* (282) signed_literal ::= NK_BOOL */ @@ -2862,8 +2863,8 @@ static const struct { { 273, -1 }, /* (284) signed_literal ::= duration_literal */ { 273, -1 }, /* (285) signed_literal ::= NULL */ { 273, -1 }, /* (286) signed_literal ::= literal_func */ - { 276, -1 }, /* (287) literal_list ::= signed_literal */ - { 276, -3 }, /* (288) literal_list ::= literal_list NK_COMMA signed_literal */ + { 310, -1 }, /* (287) literal_list ::= signed_literal */ + { 310, -3 }, /* (288) literal_list ::= literal_list NK_COMMA signed_literal */ { 250, -1 }, /* (289) db_name ::= NK_ID */ { 279, -1 }, /* (290) table_name ::= NK_ID */ { 271, -1 }, /* (291) column_name ::= NK_ID */ @@ -2872,9 +2873,9 @@ static const struct { { 312, -1 }, /* (294) column_alias ::= NK_ID */ { 245, -1 }, /* (295) user_name ::= NK_ID */ { 290, -1 }, /* (296) index_name ::= NK_ID */ - { 297, -1 }, /* (297) topic_name ::= NK_ID */ - { 304, -1 }, /* (298) stream_name ::= NK_ID */ - { 299, -1 }, /* (299) cgroup_name ::= NK_ID */ + { 296, -1 }, /* (297) topic_name ::= NK_ID */ + { 303, -1 }, /* (298) stream_name ::= NK_ID */ + { 298, -1 }, /* (299) cgroup_name ::= NK_ID */ { 313, -1 }, /* (300) expression ::= literal */ { 313, -1 }, /* (301) expression ::= pseudo_column */ { 313, -1 }, /* (302) expression ::= column_reference */ @@ -2889,8 +2890,8 @@ static const struct { { 313, -3 }, /* (311) expression ::= expression NK_SLASH expression */ { 313, -3 }, /* (312) expression ::= expression NK_REM expression */ { 313, -3 }, /* (313) expression ::= column_reference NK_ARROW NK_STRING */ - { 296, -1 }, /* (314) expression_list ::= expression */ - { 296, -3 }, /* (315) expression_list ::= expression_list NK_COMMA expression */ + { 276, -1 }, /* (314) expression_list ::= expression */ + { 276, -3 }, /* (315) expression_list ::= expression_list NK_COMMA expression */ { 315, -1 }, /* (316) column_reference ::= column_name */ { 315, -3 }, /* (317) column_reference ::= table_name NK_DOT column_name */ { 314, -1 }, /* (318) pseudo_column ::= ROWTS */ @@ -2905,8 +2906,8 @@ static const struct { { 316, -4 }, /* (327) function_expression ::= star_func NK_LP star_func_para_list NK_RP */ { 316, -6 }, /* (328) function_expression ::= CAST NK_LP expression AS type_name NK_RP */ { 316, -1 }, /* (329) function_expression ::= literal_func */ - { 310, -3 }, /* (330) literal_func ::= noarg_func NK_LP NK_RP */ - { 310, -1 }, /* (331) literal_func ::= NOW */ + { 309, -3 }, /* (330) literal_func ::= noarg_func NK_LP NK_RP */ + { 309, -1 }, /* (331) literal_func ::= NOW */ { 320, -1 }, /* (332) noarg_func ::= NOW */ { 320, -1 }, /* (333) noarg_func ::= TODAY */ { 320, -1 }, /* (334) noarg_func ::= TIMEZONE */ @@ -2977,8 +2978,8 @@ static const struct { { 347, -2 }, /* (399) select_item ::= common_expression column_alias */ { 347, -3 }, /* (400) select_item ::= common_expression AS column_alias */ { 347, -3 }, /* (401) select_item ::= table_name NK_DOT NK_STAR */ - { 308, 0 }, /* (402) where_clause_opt ::= */ - { 308, -2 }, /* (403) where_clause_opt ::= WHERE search_condition */ + { 307, 0 }, /* (402) where_clause_opt ::= */ + { 307, -2 }, /* (403) where_clause_opt ::= WHERE search_condition */ { 342, 0 }, /* (404) partition_by_clause_opt ::= */ { 342, -3 }, /* (405) partition_by_clause_opt ::= PARTITION BY expression_list */ { 343, 0 }, /* (406) twindow_clause_opt ::= */ @@ -3002,7 +3003,7 @@ static const struct { { 350, -3 }, /* (424) group_by_list ::= group_by_list NK_COMMA expression */ { 345, 0 }, /* (425) having_clause_opt ::= */ { 345, -2 }, /* (426) having_clause_opt ::= HAVING search_condition */ - { 298, -4 }, /* (427) query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt */ + { 297, -4 }, /* (427) query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt */ { 351, -1 }, /* (428) query_expression_body ::= query_primary */ { 351, -4 }, /* (429) query_expression_body ::= query_expression_body UNION ALL query_expression_body */ { 351, -3 }, /* (430) query_expression_body ::= query_expression_body UNION query_expression_body */ @@ -3531,7 +3532,7 @@ static YYACTIONTYPE yy_reduce( { yylhsminor.yy424 = addNodeToList(pCxt, yymsp[-1].minor.yy424, yymsp[0].minor.yy632); } yymsp[-1].minor.yy424 = yylhsminor.yy424; break; - case 125: /* create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_tags_opt TAGS NK_LP literal_list NK_RP table_options */ + case 125: /* create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_tags_opt TAGS NK_LP expression_list NK_RP table_options */ { yylhsminor.yy632 = createCreateSubTableClause(pCxt, yymsp[-9].minor.yy137, yymsp[-8].minor.yy632, yymsp[-6].minor.yy632, yymsp[-5].minor.yy424, yymsp[-2].minor.yy424, yymsp[0].minor.yy632); } yymsp[-9].minor.yy632 = yylhsminor.yy632; break; diff --git a/source/libs/parser/test/mockCatalog.cpp b/source/libs/parser/test/mockCatalog.cpp index 3729a2e1dc..b0c6181264 100644 --- a/source/libs/parser/test/mockCatalog.cpp +++ b/source/libs/parser/test/mockCatalog.cpp @@ -141,16 +141,18 @@ void generateTestT1(MockCatalogService* mcs) { * c2 | column | VARCHAR | 20 | * tag1 | tag | INT | 4 | * tag2 | tag | VARCHAR | 20 | + * tag3 | tag | TIMESTAMP | 8 | * Child Table: st1s1, st1s2 */ void generateTestST1(MockCatalogService* mcs) { - ITableBuilder& builder = mcs->createTableBuilder("test", "st1", TSDB_SUPER_TABLE, 3, 2) + ITableBuilder& builder = mcs->createTableBuilder("test", "st1", TSDB_SUPER_TABLE, 3, 3) .setPrecision(TSDB_TIME_PRECISION_MILLI) .addColumn("ts", TSDB_DATA_TYPE_TIMESTAMP) .addColumn("c1", TSDB_DATA_TYPE_INT) .addColumn("c2", TSDB_DATA_TYPE_BINARY, 20) .addTag("tag1", TSDB_DATA_TYPE_INT) - .addTag("tag2", TSDB_DATA_TYPE_BINARY, 20); + .addTag("tag2", TSDB_DATA_TYPE_BINARY, 20) + .addTag("tag3", TSDB_DATA_TYPE_TIMESTAMP); builder.done(); mcs->createSubTable("test", "st1", "st1s1", 1); mcs->createSubTable("test", "st1", "st1s2", 2); @@ -189,17 +191,17 @@ void generateFunctions(MockCatalogService* mcs) { int32_t __catalogGetHandle(const char* clusterId, struct SCatalog** catalogHandle) { return 0; } -int32_t __catalogGetTableMeta(struct SCatalog* pCatalog, SRequestConnInfo *pConn, const SName* pTableName, +int32_t __catalogGetTableMeta(struct SCatalog* pCatalog, SRequestConnInfo* pConn, const SName* pTableName, STableMeta** pTableMeta) { return g_mockCatalogService->catalogGetTableMeta(pTableName, pTableMeta); } -int32_t __catalogGetTableHashVgroup(struct SCatalog* pCatalog, SRequestConnInfo *pConn, - const SName* pTableName, SVgroupInfo* vgInfo) { +int32_t __catalogGetTableHashVgroup(struct SCatalog* pCatalog, SRequestConnInfo* pConn, const SName* pTableName, + SVgroupInfo* vgInfo) { return g_mockCatalogService->catalogGetTableHashVgroup(pTableName, vgInfo); } -int32_t __catalogGetTableDistVgInfo(SCatalog* pCtg, SRequestConnInfo *pConn, const SName* pTableName, +int32_t __catalogGetTableDistVgInfo(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, SArray** pVgList) { return g_mockCatalogService->catalogGetTableDistVgInfo(pTableName, pVgList); } @@ -209,28 +211,26 @@ int32_t __catalogGetDBVgVersion(SCatalog* pCtg, const char* dbFName, int32_t* ve return 0; } -int32_t __catalogGetDBVgInfo(SCatalog* pCtg, SRequestConnInfo *pConn, const char* dbFName, - SArray** pVgList) { +int32_t __catalogGetDBVgInfo(SCatalog* pCtg, SRequestConnInfo* pConn, const char* dbFName, SArray** pVgList) { return g_mockCatalogService->catalogGetDBVgInfo(dbFName, pVgList); } -int32_t __catalogGetDBCfg(SCatalog* pCtg, SRequestConnInfo *pConn, const char* dbFName, SDbCfgInfo* pDbCfg) { +int32_t __catalogGetDBCfg(SCatalog* pCtg, SRequestConnInfo* pConn, const char* dbFName, SDbCfgInfo* pDbCfg) { return 0; } -int32_t __catalogChkAuth(SCatalog* pCtg, SRequestConnInfo *pConn, const char* user, const char* dbFName, - AUTH_TYPE type, bool* pass) { +int32_t __catalogChkAuth(SCatalog* pCtg, SRequestConnInfo* pConn, const char* user, const char* dbFName, AUTH_TYPE type, + bool* pass) { *pass = true; return 0; } -int32_t __catalogGetUdfInfo(SCatalog* pCtg, SRequestConnInfo *pConn, const char* funcName, - SFuncInfo* pInfo) { +int32_t __catalogGetUdfInfo(SCatalog* pCtg, SRequestConnInfo* pConn, const char* funcName, SFuncInfo* pInfo) { return g_mockCatalogService->catalogGetUdfInfo(funcName, pInfo); } -int32_t __catalogRefreshGetTableMeta(SCatalog* pCatalog, SRequestConnInfo *pConn, - const SName* pTableName, STableMeta** pTableMeta, int32_t isSTable) { +int32_t __catalogRefreshGetTableMeta(SCatalog* pCatalog, SRequestConnInfo* pConn, const SName* pTableName, + STableMeta** pTableMeta, int32_t isSTable) { return g_mockCatalogService->catalogGetTableMeta(pTableName, pTableMeta); } diff --git a/source/libs/parser/test/parInitialATest.cpp b/source/libs/parser/test/parInitialATest.cpp index 3c1d931f37..cc23fbbc60 100644 --- a/source/libs/parser/test/parInitialATest.cpp +++ b/source/libs/parser/test/parInitialATest.cpp @@ -163,9 +163,9 @@ TEST_F(ParserInitialATest, alterSTable) { run("ALTER TABLE st1 DROP COLUMN c1"); clearAlterStbReq(); - setAlterStbReqFunc("st1", TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES, 1, "c1", TSDB_DATA_TYPE_VARCHAR, - 20 + VARSTR_HEADER_SIZE); - run("ALTER TABLE st1 MODIFY COLUMN c1 VARCHAR(20)"); + setAlterStbReqFunc("st1", TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES, 1, "c2", TSDB_DATA_TYPE_VARCHAR, + 30 + VARSTR_HEADER_SIZE); + run("ALTER TABLE st1 MODIFY COLUMN c2 VARCHAR(30)"); clearAlterStbReq(); // setAlterStbReqFunc("st1", TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME, 2, "c1", 0, 0, "cc1"); @@ -179,9 +179,9 @@ TEST_F(ParserInitialATest, alterSTable) { run("ALTER TABLE st1 DROP TAG tag1"); clearAlterStbReq(); - setAlterStbReqFunc("st1", TSDB_ALTER_TABLE_UPDATE_TAG_BYTES, 1, "tag1", TSDB_DATA_TYPE_VARCHAR, - 20 + VARSTR_HEADER_SIZE); - run("ALTER TABLE st1 MODIFY TAG tag1 VARCHAR(20)"); + setAlterStbReqFunc("st1", TSDB_ALTER_TABLE_UPDATE_TAG_BYTES, 1, "tag2", TSDB_DATA_TYPE_VARCHAR, + 30 + VARSTR_HEADER_SIZE); + run("ALTER TABLE st1 MODIFY TAG tag2 VARCHAR(30)"); clearAlterStbReq(); setAlterStbReqFunc("st1", TSDB_ALTER_TABLE_UPDATE_TAG_NAME, 2, "tag1", 0, 0, "tag11"); @@ -196,6 +196,10 @@ TEST_F(ParserInitialATest, alterSTableSemanticCheck) { useDb("root", "test"); run("ALTER TABLE st1 RENAME COLUMN c1 cc1", TSDB_CODE_PAR_INVALID_ALTER_TABLE); + + run("ALTER TABLE st1 MODIFY COLUMN c2 NCHAR(10)", TSDB_CODE_PAR_INVALID_MODIFY_COL); + + run("ALTER TABLE st1 MODIFY TAG tag2 NCHAR(10)", TSDB_CODE_PAR_INVALID_MODIFY_COL); } TEST_F(ParserInitialATest, alterTable) { @@ -336,6 +340,8 @@ TEST_F(ParserInitialATest, alterTableSemanticCheck) { useDb("root", "test"); run("ALTER TABLE st1s1 RENAME COLUMN c1 cc1", TSDB_CODE_PAR_INVALID_ALTER_TABLE); + + run("ALTER TABLE st1s1 SET TAG tag2 = '123456789012345678901'", TSDB_CODE_PAR_WRONG_VALUE_TYPE); } TEST_F(ParserInitialATest, alterUser) { diff --git a/source/libs/parser/test/parInitialCTest.cpp b/source/libs/parser/test/parInitialCTest.cpp index 919d4c041d..6cdc1e115e 100644 --- a/source/libs/parser/test/parInitialCTest.cpp +++ b/source/libs/parser/test/parInitialCTest.cpp @@ -548,12 +548,14 @@ TEST_F(ParserInitialCTest, createTable) { "a14 NCHAR(30), a15 VARCHAR(50)) " "TTL 100 COMMENT 'test create table' SMA(c1, c2, c3) ROLLUP (MIN) FILE_FACTOR 0.1"); - run("CREATE TABLE IF NOT EXISTS t1 USING st1 TAGS(1, 'wxy')"); + run("CREATE TABLE IF NOT EXISTS t1 USING st1 TAGS(1, 'wxy', NOW)"); run("CREATE TABLE " "IF NOT EXISTS test.t1 USING test.st1 (tag1, tag2) TAGS(1, 'abc') " "IF NOT EXISTS test.t2 USING test.st1 (tag1, tag2) TAGS(2, 'abc') " "IF NOT EXISTS test.t3 USING test.st1 (tag1, tag2) TAGS(3, 'abc') "); + + // run("CREATE TABLE IF NOT EXISTS t1 USING st1 TAGS(1, 'wxy', NOW + 1S)"); } TEST_F(ParserInitialCTest, createTopic) { diff --git a/source/libs/parser/test/parInsertTest.cpp b/source/libs/parser/test/parInsertTest.cpp index 29edca0d40..3ebf9a417b 100644 --- a/source/libs/parser/test/parInsertTest.cpp +++ b/source/libs/parser/test/parInsertTest.cpp @@ -245,8 +245,8 @@ TEST_F(InsertTest, autoCreateTableTest) { setDatabase("root", "test"); bind( - "insert into st1s1 using st1 tags(1, 'wxy') values (now, 1, \"beijing\")(now+1s, 2, \"shanghai\")(now+2s, 3, " - "\"guangzhou\")"); + "insert into st1s1 using st1 tags(1, 'wxy', now) " + "values (now, 1, \"beijing\")(now+1s, 2, \"shanghai\")(now+2s, 3, \"guangzhou\")"); ASSERT_EQ(run(), TSDB_CODE_SUCCESS); dumpReslut(); checkReslut(1, 3); @@ -257,8 +257,8 @@ TEST_F(InsertTest, autoCreateTableTest) { ASSERT_EQ(run(), TSDB_CODE_SUCCESS); bind( - "insert into st1s1 using st1 tags(1, 'wxy') values (now, 1, \"beijing\")(now+1s, 2, \"shanghai\")(now+2s, 3, " - "\"guangzhou\")"); + "insert into st1s1 using st1 tags(1, 'wxy', now) " + "values (now, 1, \"beijing\")(now+1s, 2, \"shanghai\")(now+2s, 3, \"guangzhou\")"); ASSERT_EQ(runAsync(), TSDB_CODE_SUCCESS); bind( diff --git a/source/libs/parser/test/parSelectTest.cpp b/source/libs/parser/test/parSelectTest.cpp index 51d302fe12..c87520e262 100644 --- a/source/libs/parser/test/parSelectTest.cpp +++ b/source/libs/parser/test/parSelectTest.cpp @@ -219,6 +219,7 @@ TEST_F(ParserSelectTest, intervalSemanticCheck) { run("SELECT HISTOGRAM(c1, 'log_bin', '{\"start\": -33,\"factor\": 55,\"count\": 5,\"infinity\": false}', 1) FROM t1 " "WHERE ts > TIMESTAMP '2022-04-01 00:00:00' and ts < TIMESTAMP '2022-04-30 23:59:59' INTERVAL(10s) FILL(NULL)", TSDB_CODE_PAR_FILL_NOT_ALLOWED_FUNC); + run("SELECT _WSTARTTS, _WENDTS, _WDURATION, sum(c1) FROM t1", TSDB_CODE_PAR_INVALID_WINDOW_PC); } TEST_F(ParserSelectTest, subquery) { @@ -231,13 +232,22 @@ TEST_F(ParserSelectTest, subquery) { run("SELECT SUM(a) FROM (SELECT MAX(c1) a, ts FROM st1s1 PARTITION BY TBNAME INTERVAL(1m)) INTERVAL(1n)"); run("SELECT SUM(a) FROM (SELECT MAX(c1) a, _wstartts FROM st1s1 PARTITION BY TBNAME INTERVAL(1m)) INTERVAL(1n)"); + + run("SELECT _C0 FROM (SELECT _ROWTS, ts FROM st1s1)"); + + run("SELECT ts FROM (SELECT t1.ts FROM st1s1 t1)"); } TEST_F(ParserSelectTest, subquerySemanticCheck) { useDb("root", "test"); - run("SELECT SUM(a) FROM (SELECT MAX(c1) a FROM st1s1 INTERVAL(1m)) INTERVAL(1n)", TSDB_CODE_PAR_NOT_ALLOWED_WIN_QUERY, - PARSER_STAGE_TRANSLATE); + run("SELECT SUM(a) FROM (SELECT MAX(c1) a FROM st1s1 INTERVAL(1m)) INTERVAL(1n)", + TSDB_CODE_PAR_NOT_ALLOWED_WIN_QUERY); + + run("SELECT ts FROM (SELECT t1.ts AS ts, t2.ts FROM st1s1 t1, st1s2 t2 WHERE t1.ts = t2.ts)", + TSDB_CODE_PAR_AMBIGUOUS_COLUMN); + + run("SELECT ts FROM (SELECT ts AS c1 FROM st1s1 t1)", TSDB_CODE_PAR_INVALID_COLUMN); } TEST_F(ParserSelectTest, semanticCheck) { diff --git a/source/libs/parser/test/parTestUtil.cpp b/source/libs/parser/test/parTestUtil.cpp index 13ab116552..663f456cb8 100644 --- a/source/libs/parser/test/parTestUtil.cpp +++ b/source/libs/parser/test/parTestUtil.cpp @@ -62,7 +62,7 @@ int32_t getLogLevel() { return g_logLevel; } class ParserTestBaseImpl { public: - ParserTestBaseImpl(ParserTestBase* pBase) : pBase_(pBase) {} + ParserTestBaseImpl(ParserTestBase* pBase) : pBase_(pBase), sqlNo_(0) {} void login(const std::string& user) { caseEnv_.user_ = user; } @@ -73,6 +73,7 @@ class ParserTestBaseImpl { } void run(const string& sql, int32_t expect, ParserStage checkStage) { + ++sqlNo_; if (caseEnv_.nsql_ > 0) { --(caseEnv_.nsql_); return; @@ -174,7 +175,7 @@ class ParserTestBaseImpl { } void dump() { - cout << "==========================================sql : [" << stmtEnv_.sql_ << "]" << endl; + cout << "========================================== " << sqlNo_ << " sql : [" << stmtEnv_.sql_ << "]" << endl; if (!res_.parsedAst_.empty()) { cout << "raw syntax tree : " << endl; cout << res_.parsedAst_ << endl; @@ -425,6 +426,7 @@ class ParserTestBaseImpl { stmtEnv stmtEnv_; stmtRes res_; ParserTestBase* pBase_; + int32_t sqlNo_; }; ParserTestBase::ParserTestBase() : impl_(new ParserTestBaseImpl(this)) {} diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index 2fc79255af..9e4967dd25 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -416,6 +416,8 @@ static int32_t stbSplCreatePartAggNode(SAggLogicNode* pMergeAgg, SLogicNode** pO pMergeAgg->node.pTargets = NULL; SNodeList* pChildren = pMergeAgg->node.pChildren; pMergeAgg->node.pChildren = NULL; + SNode* pConditions = pMergeAgg->node.pConditions; + pMergeAgg->node.pConditions = NULL; int32_t code = TSDB_CODE_SUCCESS; SAggLogicNode* pPartAgg = (SAggLogicNode*)nodesCloneNode((SNode*)pMergeAgg); @@ -434,6 +436,7 @@ static int32_t stbSplCreatePartAggNode(SAggLogicNode* pMergeAgg, SLogicNode** pO } } if (TSDB_CODE_SUCCESS == code) { + pMergeAgg->node.pConditions = pConditions; pMergeAgg->node.pTargets = pTargets; pPartAgg->node.pChildren = pChildren; diff --git a/source/libs/planner/test/planGroupByTest.cpp b/source/libs/planner/test/planGroupByTest.cpp index 78d0f7b21f..9d937d7c91 100644 --- a/source/libs/planner/test/planGroupByTest.cpp +++ b/source/libs/planner/test/planGroupByTest.cpp @@ -82,4 +82,6 @@ TEST_F(PlanGroupByTest, stable) { run("SELECT COUNT(*) FROM st1"); run("SELECT COUNT(*) FROM st1 GROUP BY c1"); + + run("SELECT SUM(c1) FROM st1 GROUP BY c2 HAVING SUM(c1) IS NOT NULL"); } diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index db8a055362..2023680abb 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -2,20 +2,20 @@ #======================b1-start=============== # ---- user -#./test.sh -f tsim/user/basic1.sim -#./test.sh -f tsim/user/pass_alter.sim -#./test.sh -f tsim/user/pass_len.sim -#./test.sh -f tsim/user/user_len.sim -#./test.sh -f tsim/user/privilege1.sim -#./test.sh -f tsim/user/privilege2.sim# +./test.sh -f tsim/user/basic1.sim +./test.sh -f tsim/user/pass_alter.sim +./test.sh -f tsim/user/pass_len.sim +./test.sh -f tsim/user/user_len.sim +./test.sh -f tsim/user/privilege1.sim +./test.sh -f tsim/user/privilege2.sim ## ---- db -#./test.sh -f tsim/db/create_all_options.sim -#./test.sh -f tsim/db/alter_option.sim -#./test.sh -f tsim/db/basic1.sim -#./test.sh -f tsim/db/basic2.sim -#./test.sh -f tsim/db/basic3.sim -#./test.sh -f tsim/db/basic6.sim +./test.sh -f tsim/db/create_all_options.sim +./test.sh -f tsim/db/alter_option.sim +./test.sh -f tsim/db/basic1.sim +./test.sh -f tsim/db/basic2.sim +./test.sh -f tsim/db/basic3.sim +./test.sh -f tsim/db/basic6.sim ./test.sh -f tsim/db/basic7.sim ./test.sh -f tsim/db/error1.sim ./test.sh -f tsim/db/taosdlog.sim From 300a3a2c0221f362839730358bfdbc8d92028be5 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 14 Jun 2022 15:46:16 +0800 Subject: [PATCH 66/78] enh: drop sma while drop stb and db --- source/dnode/mnode/impl/inc/mndSma.h | 2 + source/dnode/mnode/impl/src/mndDb.c | 1 + source/dnode/mnode/impl/src/mndSma.c | 139 ++++++++++++++++++++++----- source/dnode/mnode/impl/src/mndStb.c | 33 ++++--- 4 files changed, 137 insertions(+), 38 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndSma.h b/source/dnode/mnode/impl/inc/mndSma.h index 4a80f619d3..5d13f8b74d 100644 --- a/source/dnode/mnode/impl/inc/mndSma.h +++ b/source/dnode/mnode/impl/inc/mndSma.h @@ -26,6 +26,8 @@ int32_t mndInitSma(SMnode *pMnode); void mndCleanupSma(SMnode *pMnode); SSmaObj *mndAcquireSma(SMnode *pMnode, char *smaName); void mndReleaseSma(SMnode *pMnode, SSmaObj *pSma); +int32_t mndDropSmasByStb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb); +int32_t mndDropSmasByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index cfe363fdf0..43263af573 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -935,6 +935,7 @@ static int32_t mndDropDb(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb) { if (mndDropOffsetByDB(pMnode, pTrans, pDb) != 0) goto _OVER; if (mndDropSubByDB(pMnode, pTrans, pDb) != 0) goto _OVER; if (mndDropTopicByDB(pMnode, pTrans, pDb) != 0) goto _OVER; + if (mndDropSmasByDb(pMnode, pTrans, pDb) != 0) goto _OVER; if (mndSetDropDbRedoActions(pMnode, pTrans, pDb) != 0) goto _OVER; SUserObj *pUser = mndAcquireUser(pMnode, pDb->createUser); diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index c19b558f19..cc57076c0a 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -38,10 +38,10 @@ static int32_t mndSmaActionInsert(SSdb *pSdb, SSmaObj *pSma); static int32_t mndSmaActionDelete(SSdb *pSdb, SSmaObj *pSpSmatb); static int32_t mndSmaActionUpdate(SSdb *pSdb, SSmaObj *pOld, SSmaObj *pNew); static int32_t mndSmaGetVgEpSet(SMnode *pMnode, SDbObj *pDb, SVgEpSet **ppVgEpSet, int32_t *numOfVgroups); -static int32_t mndProcessMCreateSmaReq(SRpcMsg *pReq); -static int32_t mndProcessMDropSmaReq(SRpcMsg *pReq); +static int32_t mndProcessCreateSmaReq(SRpcMsg *pReq); +static int32_t mndProcessDropSmaReq(SRpcMsg *pReq); static int32_t mndProcessGetSmaReq(SRpcMsg *pReq); -static int32_t mndProcessGetTbSmaReq(SRpcMsg *pReq); +static int32_t mndProcessGetTbSmaReq(SRpcMsg *pReq); static int32_t mndRetrieveSma(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextSma(SMnode *pMnode, void *pIter); @@ -56,8 +56,8 @@ int32_t mndInitSma(SMnode *pMnode) { .deleteFp = (SdbDeleteFp)mndSmaActionDelete, }; - mndSetMsgHandle(pMnode, TDMT_MND_CREATE_SMA, mndProcessMCreateSmaReq); - mndSetMsgHandle(pMnode, TDMT_MND_DROP_SMA, mndProcessMDropSmaReq); + mndSetMsgHandle(pMnode, TDMT_MND_CREATE_SMA, mndProcessCreateSmaReq); + mndSetMsgHandle(pMnode, TDMT_MND_DROP_SMA, mndProcessDropSmaReq); mndSetMsgHandle(pMnode, TDMT_VND_CREATE_SMA_RSP, mndTransProcessRsp); mndSetMsgHandle(pMnode, TDMT_VND_DROP_SMA_RSP, mndTransProcessRsp); mndSetMsgHandle(pMnode, TDMT_MND_GET_INDEX, mndProcessGetSmaReq); @@ -79,7 +79,6 @@ static SSdbRaw *mndSmaActionEncode(SSmaObj *pSma) { if (pRaw == NULL) goto _OVER; int32_t dataPos = 0; - SDB_SET_BINARY(pRaw, dataPos, pSma->name, TSDB_TABLE_FNAME_LEN, _OVER) SDB_SET_BINARY(pRaw, dataPos, pSma->stb, TSDB_TABLE_FNAME_LEN, _OVER) SDB_SET_BINARY(pRaw, dataPos, pSma->db, TSDB_DB_FNAME_LEN, _OVER) @@ -100,6 +99,7 @@ static SSdbRaw *mndSmaActionEncode(SSmaObj *pSma) { SDB_SET_INT32(pRaw, dataPos, pSma->tagsFilterLen, _OVER) SDB_SET_INT32(pRaw, dataPos, pSma->sqlLen, _OVER) SDB_SET_INT32(pRaw, dataPos, pSma->astLen, _OVER) + if (pSma->exprLen > 0) { SDB_SET_BINARY(pRaw, dataPos, pSma->expr, pSma->exprLen, _OVER) } @@ -115,6 +115,7 @@ static SSdbRaw *mndSmaActionEncode(SSmaObj *pSma) { SDB_SET_RESERVE(pRaw, dataPos, TSDB_SMA_RESERVE_SIZE, _OVER) SDB_SET_DATALEN(pRaw, dataPos, _OVER) + terrno = 0; _OVER: @@ -193,6 +194,7 @@ static SSdbRow *mndSmaActionDecode(SSdbRaw *pRaw) { } SDB_GET_RESERVE(pRaw, dataPos, TSDB_SMA_RESERVE_SIZE, _OVER) + terrno = 0; _OVER: @@ -383,6 +385,25 @@ static int32_t mndSetCreateSmaVgroupCommitLogs(SMnode *pMnode, STrans *pTrans, S return 0; } +static int32_t mndSetUpdateSmaStbCommitLogs(SMnode *pMnode, STrans *pTrans, SStbObj *pStb) { + SStbObj stbObj = {0}; + taosRLockLatch(&pStb->lock); + memcpy(&stbObj, pStb, sizeof(SStbObj)); + taosRUnLockLatch(&pStb->lock); + stbObj.pColumns = NULL; + stbObj.pTags = NULL; + stbObj.updateTime = taosGetTimestampMs(); + stbObj.lock = 0; + stbObj.smaVer++; + + SSdbRaw *pCommitRaw = mndStbActionEncode(&stbObj); + if (pCommitRaw == NULL) return -1; + if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; + if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) return -1; + + return 0; +} + static int32_t mndSetCreateSmaRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SSmaObj *pSma) { SSdb *pSdb = pMnode->pSdb; SVgObj *pVgroup = NULL; @@ -457,7 +478,6 @@ static int32_t mndSetCreateSmaVgroupRedoActions(SMnode *pMnode, STrans *pTrans, pSma->schemaTag.pSchema[0].flags = 0; snprintf(pSma->schemaTag.pSchema[0].name, TSDB_COL_NAME_LEN, "groupId"); - int32_t smaContLen = 0; void *pSmaReq = mndBuildVCreateSmaReq(pMnode, pVgroup, pSma, &smaContLen); if (pSmaReq == NULL) return -1; @@ -559,6 +579,7 @@ static int32_t mndCreateSma(SMnode *pMnode, SRpcMsg *pReq, SMCreateSmaReq *pCrea if (mndSetCreateSmaVgroupRedoLogs(pMnode, pTrans, &streamObj.fixedSinkVg) != 0) goto _OVER; if (mndSetCreateSmaCommitLogs(pMnode, pTrans, &smaObj) != 0) goto _OVER; if (mndSetCreateSmaVgroupCommitLogs(pMnode, pTrans, &streamObj.fixedSinkVg) != 0) goto _OVER; + if (mndSetUpdateSmaStbCommitLogs(pMnode, pTrans, pStb) != 0) goto _OVER; if (mndSetCreateSmaRedoActions(pMnode, pTrans, pDb, &smaObj) != 0) goto _OVER; if (mndSetCreateSmaVgroupRedoActions(pMnode, pTrans, pDb, &streamObj.fixedSinkVg, &smaObj) != 0) goto _OVER; if (mndAddStreamToTrans(pMnode, &streamObj, pCreate->ast, STREAM_TRIGGER_AT_ONCE, 0, pTrans) != 0) goto _OVER; @@ -599,7 +620,7 @@ static int32_t mndCheckCreateSmaReq(SMCreateSmaReq *pCreate) { return 0; } -static int32_t mndProcessMCreateSmaReq(SRpcMsg *pReq) { +static int32_t mndProcessCreateSmaReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; int32_t code = -1; SStbObj *pStb = NULL; @@ -781,13 +802,17 @@ static int32_t mndSetDropSmaVgroupRedoActions(SMnode *pMnode, STrans *pTrans, SD } static int32_t mndDropSma(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SSmaObj *pSma) { - int32_t code = -1; - SVgObj *pVgroup = NULL; - STrans *pTrans = NULL; + int32_t code = -1; + SVgObj *pVgroup = NULL; + SStbObj *pStb = NULL; + STrans *pTrans = NULL; pVgroup = mndAcquireVgroup(pMnode, pSma->dstVgId); if (pVgroup == NULL) goto _OVER; + pStb = mndAcquireStb(pMnode, pSma->stb); + if (pStb == NULL) goto _OVER; + pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB, pReq); if (pTrans == NULL) goto _OVER; @@ -798,6 +823,7 @@ static int32_t mndDropSma(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SSmaObj *p if (mndSetDropSmaVgroupRedoLogs(pMnode, pTrans, pVgroup) != 0) goto _OVER; if (mndSetDropSmaCommitLogs(pMnode, pTrans, pSma) != 0) goto _OVER; if (mndSetDropSmaVgroupCommitLogs(pMnode, pTrans, pVgroup) != 0) goto _OVER; + if (mndSetUpdateSmaStbCommitLogs(pMnode, pTrans, pStb) != 0) goto _OVER; if (mndSetDropSmaRedoActions(pMnode, pTrans, pDb, pSma) != 0) goto _OVER; if (mndSetDropSmaVgroupRedoActions(pMnode, pTrans, pDb, pVgroup) != 0) goto _OVER; if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; @@ -807,10 +833,78 @@ static int32_t mndDropSma(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SSmaObj *p _OVER: mndTransDrop(pTrans); mndReleaseVgroup(pMnode, pVgroup); + mndReleaseStb(pMnode, pStb); return code; } -static int32_t mndProcessMDropSmaReq(SRpcMsg *pReq) { +int32_t mndDropSmasByStb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) { + SSdb *pSdb = pMnode->pSdb; + SSmaObj *pSma = NULL; + void *pIter = NULL; + SVgObj *pVgroup = NULL; + int32_t code = -1; + + while (1) { + pIter = sdbFetch(pSdb, SDB_SMA, pIter, (void **)&pSma); + if (pIter == NULL) break; + + if (pSma->stbUid == pStb->uid) { + pVgroup = mndAcquireVgroup(pMnode, pSma->dstVgId); + if (pVgroup == NULL) goto _OVER; + if (mndSetDropSmaVgroupCommitLogs(pMnode, pTrans, pVgroup) != 0) goto _OVER; + if (mndSetDropSmaVgroupRedoActions(pMnode, pTrans, pDb, pVgroup) != 0) goto _OVER; + if (mndSetDropSmaCommitLogs(pMnode, pTrans, pSma) != 0) goto _OVER; + if (mndSetDropSmaRedoActions(pMnode, pTrans, pDb, pSma) != 0) goto _OVER; + mndReleaseVgroup(pMnode, pVgroup); + pVgroup = NULL; + } + + sdbRelease(pSdb, pSma); + } + + code = 0; + +_OVER: + sdbCancelFetch(pSdb, pIter); + sdbRelease(pSdb, pSma); + mndReleaseVgroup(pMnode, pVgroup); + return code; +} + +int32_t mndDropSmasByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) { + SSdb *pSdb = pMnode->pSdb; + SSmaObj *pSma = NULL; + void *pIter = NULL; + SVgObj *pVgroup = NULL; + int32_t code = -1; + + while (1) { + pIter = sdbFetch(pSdb, SDB_SMA, pIter, (void **)&pSma); + if (pIter == NULL) break; + + if (pSma->dbUid == pDb->uid) { + pVgroup = mndAcquireVgroup(pMnode, pSma->dstVgId); + if (pVgroup == NULL) goto _OVER; + if (mndSetDropSmaVgroupCommitLogs(pMnode, pTrans, pVgroup) != 0) goto _OVER; + if (mndSetDropSmaVgroupRedoActions(pMnode, pTrans, pDb, pVgroup) != 0) goto _OVER; + if (mndSetDropSmaCommitLogs(pMnode, pTrans, pSma) != 0) goto _OVER; + mndReleaseVgroup(pMnode, pVgroup); + pVgroup = NULL; + } + + sdbRelease(pSdb, pSma); + } + + code = 0; + +_OVER: + sdbCancelFetch(pSdb, pIter); + sdbRelease(pSdb, pSma); + mndReleaseVgroup(pMnode, pVgroup); + return code; +} + +static int32_t mndProcessDropSmaReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; int32_t code = -1; SUserObj *pUser = NULL; @@ -900,10 +994,10 @@ static int32_t mndGetSma(SMnode *pMnode, SUserIndexReq *indexReq, SUserIndexRsp } static int32_t mndGetTableSma(SMnode *pMnode, STableIndexReq *indexReq, STableIndexRsp *rsp, bool *exist) { - int32_t code = 0; - SSmaObj *pSma = NULL; - SSdb *pSdb = pMnode->pSdb; - void *pIter = NULL; + int32_t code = 0; + SSmaObj *pSma = NULL; + SSdb *pSdb = pMnode->pSdb; + void *pIter = NULL; STableIndexInfo info; while (1) { @@ -922,14 +1016,14 @@ static int32_t mndGetTableSma(SMnode *pMnode, STableIndexReq *indexReq, STableIn info.dstTbUid = pSma->dstTbUid; info.dstVgId = pSma->dstVgId; - SVgObj* pVg = mndAcquireVgroup(pMnode, pSma->dstVgId); + SVgObj *pVg = mndAcquireVgroup(pMnode, pSma->dstVgId); if (pVg == NULL) { code = -1; sdbRelease(pSdb, pSma); return code; } info.epSet = mndGetVgroupEpset(pMnode, pVg); - + info.expr = taosMemoryMalloc(pSma->exprLen + 1); if (info.expr == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -953,7 +1047,7 @@ static int32_t mndGetTableSma(SMnode *pMnode, STableIndexReq *indexReq, STableIn sdbRelease(pSdb, pSma); } - + return code; } @@ -1005,10 +1099,10 @@ _OVER: static int32_t mndProcessGetTbSmaReq(SRpcMsg *pReq) { STableIndexReq indexReq = {0}; - SMnode *pMnode = pReq->info.node; - int32_t code = -1; + SMnode *pMnode = pReq->info.node; + int32_t code = -1; STableIndexRsp rsp = {0}; - bool exist = false; + bool exist = false; if (tDeserializeSTableIndexReq(pReq->pCont, pReq->contLen, &indexReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; @@ -1055,7 +1149,6 @@ _OVER: return code; } - static int32_t mndRetrieveSma(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { SMnode *pMnode = pReq->info.node; SSdb *pSdb = pMnode->pSdb; diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index ad0c913fb8..cb0a893436 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -23,6 +23,7 @@ #include "mndPerfSchema.h" #include "mndScheduler.h" #include "mndShow.h" +#include "mndSma.h" #include "mndTopic.h" #include "mndTrans.h" #include "mndUser.h" @@ -36,9 +37,9 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw); static int32_t mndStbActionInsert(SSdb *pSdb, SStbObj *pStb); static int32_t mndStbActionDelete(SSdb *pSdb, SStbObj *pStb); static int32_t mndStbActionUpdate(SSdb *pSdb, SStbObj *pOld, SStbObj *pNew); -static int32_t mndProcessMCreateStbReq(SRpcMsg *pReq); -static int32_t mndProcessMAlterStbReq(SRpcMsg *pReq); -static int32_t mndProcessMDropStbReq(SRpcMsg *pReq); +static int32_t mndProcessCreateStbReq(SRpcMsg *pReq); +static int32_t mndProcessAlterStbReq(SRpcMsg *pReq); +static int32_t mndProcessDropStbReq(SRpcMsg *pReq); static int32_t mndProcessTableMetaReq(SRpcMsg *pReq); static int32_t mndRetrieveStb(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextStb(SMnode *pMnode, void *pIter); @@ -54,9 +55,9 @@ int32_t mndInitStb(SMnode *pMnode) { .deleteFp = (SdbDeleteFp)mndStbActionDelete, }; - mndSetMsgHandle(pMnode, TDMT_MND_CREATE_STB, mndProcessMCreateStbReq); - mndSetMsgHandle(pMnode, TDMT_MND_ALTER_STB, mndProcessMAlterStbReq); - mndSetMsgHandle(pMnode, TDMT_MND_DROP_STB, mndProcessMDropStbReq); + mndSetMsgHandle(pMnode, TDMT_MND_CREATE_STB, mndProcessCreateStbReq); + mndSetMsgHandle(pMnode, TDMT_MND_ALTER_STB, mndProcessAlterStbReq); + mndSetMsgHandle(pMnode, TDMT_MND_DROP_STB, mndProcessDropStbReq); mndSetMsgHandle(pMnode, TDMT_VND_CREATE_STB_RSP, mndTransProcessRsp); mndSetMsgHandle(pMnode, TDMT_VND_ALTER_STB_RSP, mndTransProcessRsp); mndSetMsgHandle(pMnode, TDMT_VND_DROP_STB_RSP, mndTransProcessRsp); @@ -318,6 +319,7 @@ static int32_t mndStbActionUpdate(SSdb *pSdb, SStbObj *pOld, SStbObj *pNew) { pOld->updateTime = pNew->updateTime; pOld->tagVer = pNew->tagVer; pOld->colVer = pNew->colVer; + pOld->smaVer = pNew->smaVer; pOld->nextColId = pNew->nextColId; pOld->ttl = pNew->ttl; pOld->numOfColumns = pNew->numOfColumns; @@ -361,7 +363,7 @@ SDbObj *mndAcquireDbByStb(SMnode *pMnode, const char *stbName) { return mndAcquireDb(pMnode, db); } -static FORCE_INLINE int schemaExColIdCompare(const void *colId, const void *pSchema) { +static FORCE_INLINE int32_t schemaExColIdCompare(const void *colId, const void *pSchema) { if (*(col_id_t *)colId < ((SSchema *)pSchema)->colId) { return -1; } else if (*(col_id_t *)colId > ((SSchema *)pSchema)->colId) { @@ -395,14 +397,14 @@ static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pSt req.pRSmaParam.xFilesFactor = pStb->xFilesFactor; req.pRSmaParam.delay = pStb->delay; if (pStb->ast1Len > 0) { - if (mndConvertRsmaTask(&req.pRSmaParam.qmsg1, &req.pRSmaParam.qmsg1Len, pStb->pAst1, pStb->uid, STREAM_TRIGGER_AT_ONCE, 0, - req.pRSmaParam.xFilesFactor) != TSDB_CODE_SUCCESS) { + if (mndConvertRsmaTask(&req.pRSmaParam.qmsg1, &req.pRSmaParam.qmsg1Len, pStb->pAst1, pStb->uid, + STREAM_TRIGGER_AT_ONCE, 0, req.pRSmaParam.xFilesFactor) != TSDB_CODE_SUCCESS) { return NULL; } } if (pStb->ast2Len > 0) { - if (mndConvertRsmaTask(&req.pRSmaParam.qmsg2, &req.pRSmaParam.qmsg2Len, pStb->pAst2, pStb->uid, STREAM_TRIGGER_AT_ONCE, 0, - req.pRSmaParam.xFilesFactor) != TSDB_CODE_SUCCESS) { + if (mndConvertRsmaTask(&req.pRSmaParam.qmsg2, &req.pRSmaParam.qmsg2Len, pStb->pAst2, pStb->uid, + STREAM_TRIGGER_AT_ONCE, 0, req.pRSmaParam.xFilesFactor) != TSDB_CODE_SUCCESS) { return NULL; } } @@ -761,7 +763,7 @@ int32_t mndAddStbToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *p return 0; } -static int32_t mndProcessMCreateStbReq(SRpcMsg *pReq) { +static int32_t mndProcessCreateStbReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; int32_t code = -1; SStbObj *pStb = NULL; @@ -1296,7 +1298,7 @@ static int32_t mndBuildStbSchema(SMnode *pMnode, const char *dbFName, const char static int32_t mndBuildSMAlterStbRsp(SDbObj *pDb, const SMAlterStbReq *pAlter, SStbObj *pObj, void **pCont, int32_t *pLen) { - int ret; + int32_t ret; SEncoder ec = {0}; uint32_t contLen = 0; SMAlterStbRsp alterRsp = {0}; @@ -1415,7 +1417,7 @@ _OVER: return code; } -static int32_t mndProcessMAlterStbReq(SRpcMsg *pReq) { +static int32_t mndProcessAlterStbReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; int32_t code = -1; SDbObj *pDb = NULL; @@ -1545,6 +1547,7 @@ static int32_t mndDropStb(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SStbObj *p if (mndSetDropStbRedoLogs(pMnode, pTrans, pStb) != 0) goto _OVER; if (mndSetDropStbCommitLogs(pMnode, pTrans, pStb) != 0) goto _OVER; if (mndSetDropStbRedoActions(pMnode, pTrans, pDb, pStb) != 0) goto _OVER; + if (mndDropSmasByStb(pMnode, pTrans, pDb, pStb) != 0) goto _OVER; if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; code = 0; @@ -1554,7 +1557,7 @@ _OVER: return code; } -static int32_t mndProcessMDropStbReq(SRpcMsg *pReq) { +static int32_t mndProcessDropStbReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; int32_t code = -1; SUserObj *pUser = NULL; From a17c6e3c33b895865e120ba146b86c7c208f20ae Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 14 Jun 2022 15:47:56 +0800 Subject: [PATCH 67/78] fix bugs --- source/libs/function/src/builtinsimpl.c | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index fe9b4e1c3a..5ee9dbdee6 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -774,8 +774,8 @@ _avg_over: } static void avgTransferInfo(SAvgRes* pInput, SAvgRes* pOutput) { - int16_t type = pInput->type; - if (IS_INTEGER_TYPE(type)) { + pOutput->type = pInput->type; + if (IS_INTEGER_TYPE(pOutput->type)) { pOutput->sum.isum += pInput->sum.isum; } else { pOutput->sum.dsum += pInput->sum.dsum; @@ -792,11 +792,10 @@ int32_t avgFunctionMerge(SqlFunctionCtx* pCtx) { ASSERT(pCol->info.type == TSDB_DATA_TYPE_BINARY); SAvgRes* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); - SAvgRes* pInputInfo; int32_t start = pInput->startRowIndex; char* data = colDataGetData(pCol, start); - pInputInfo = (SAvgRes*)varDataVal(data); + SAvgRes* pInputInfo = (SAvgRes*)varDataVal(data); avgTransferInfo(pInputInfo, pInfo); @@ -889,8 +888,8 @@ int32_t avgCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) { int32_t avgFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { SInputColumnInfoData* pInput = &pCtx->input; - int32_t type = pInput->pData[0]->info.type; SAvgRes* pAvgRes = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); + int32_t type = pAvgRes->type; if (IS_INTEGER_TYPE(type)) { pAvgRes->result = pAvgRes->sum.isum / ((double)pAvgRes->count); @@ -3292,11 +3291,10 @@ int32_t spreadFunctionMerge(SqlFunctionCtx* pCtx) { ASSERT(pCol->info.type == TSDB_DATA_TYPE_BINARY); SSpreadInfo* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); - SSpreadInfo* pInputInfo; int32_t start = pInput->startRowIndex; char* data = colDataGetData(pCol, start); - pInputInfo = (SSpreadInfo*)varDataVal(data); + SSpreadInfo* pInputInfo = (SSpreadInfo*)varDataVal(data); spreadTransferInfo(pInputInfo, pInfo); From ac2a4678f0effd2c445e4b0a2b84415a783b6bef Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Tue, 14 Jun 2022 15:56:58 +0800 Subject: [PATCH 68/78] fix: some problems of parser --- source/libs/parser/src/parTranslater.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 365704f988..3c3c0e4a61 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -4511,7 +4511,8 @@ static int32_t translateTagVal(STranslateContext* pCxt, uint8_t precision, SSche ? pCxt->errCode : TSDB_CODE_SUCCESS); } else { - return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)pNode)->aliasName); + // return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)pNode)->aliasName); + return TSDB_CODE_FAILED; } } From c9509acf222bc3777a4bca3fea29d1c1cf1ffa0a Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 14 Jun 2022 16:01:32 +0800 Subject: [PATCH 69/78] refactor: mnode pre process msg --- include/dnode/mnode/mnode.h | 20 +++++++++++++++++--- source/dnode/mgmt/mgmt_mnode/src/mmWorker.c | 3 ++- source/dnode/mnode/impl/src/mndQuery.c | 7 +++---- 3 files changed, 22 insertions(+), 8 deletions(-) diff --git a/include/dnode/mnode/mnode.h b/include/dnode/mnode/mnode.h index 99b704826d..21d8405f58 100644 --- a/include/dnode/mnode/mnode.h +++ b/include/dnode/mnode/mnode.h @@ -59,7 +59,13 @@ void mndClose(SMnode *pMnode); * @param pMnode The mnode object. */ int32_t mndStart(SMnode *pMnode); -void mndStop(SMnode *pMnode); + +/** + * @brief Stop mnode + * + * @param pMnode The mnode object. + */ +void mndStop(SMnode *pMnode); /** * @brief Get mnode monitor info. @@ -71,17 +77,25 @@ void mndStop(SMnode *pMnode); * @return int32_t 0 for success, -1 for failure. */ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pCluster, SMonVgroupInfo *pVgroup, SMonGrantInfo *pGrant); + +/** + * @brief Get mnode loads for status msg. + * + * @param pMnode The mnode object. + * @param pLoad + * @return int32_t 0 for success, -1 for failure. + */ int32_t mndGetLoad(SMnode *pMnode, SMnodeLoad *pLoad); /** - * @brief Process the read, write, sync request. + * @brief Process the rpc, sync request. * * @param pMsg The request msg. * @return int32_t 0 for success, -1 for failure. */ int32_t mndProcessRpcMsg(SRpcMsg *pMsg); int32_t mndProcessSyncMsg(SRpcMsg *pMsg); -int32_t mndPreprocessQueryMsg(SMnode * pMnode, SRpcMsg * pMsg); +int32_t mndPreProcessMsg(SRpcMsg *pMsg); /** * @brief Generate machine code diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c index ee2df5f089..7cd7da1aa9 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c @@ -114,7 +114,8 @@ int32_t mmPutMsgToReadQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { } int32_t mmPutMsgToQueryQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { - if (mndPreprocessQueryMsg(pMgmt->pMnode, pMsg) != 0) { + pMsg->info.node = pMgmt->pMnode; + if (mndPreProcessMsg(pMsg) != 0) { dError("msg:%p, failed to pre-process in mnode since %s, type:%s", pMsg, terrstr(), TMSG_INFO(pMsg->msgType)); return -1; } diff --git a/source/dnode/mnode/impl/src/mndQuery.c b/source/dnode/mnode/impl/src/mndQuery.c index 12b39e5b78..5374f48e47 100644 --- a/source/dnode/mnode/impl/src/mndQuery.c +++ b/source/dnode/mnode/impl/src/mndQuery.c @@ -18,11 +18,10 @@ #include "mndMnode.h" #include "qworker.h" -int32_t mndPreprocessQueryMsg(SMnode * pMnode, SRpcMsg * pMsg) { - if (TDMT_VND_QUERY != pMsg->msgType) { - return 0; - } +int32_t mndPreProcessMsg(SRpcMsg *pMsg) { + if (TDMT_VND_QUERY != pMsg->msgType) return 0; + SMnode *pMnode = pMsg->info.node; return qWorkerPreprocessQueryMsg(pMnode->pQuery, pMsg); } From 72bc32af1e16ef1af3496e7ac4db30aa0747519c Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 14 Jun 2022 16:17:37 +0800 Subject: [PATCH 70/78] ehn: remove sync ref --- source/dnode/mnode/impl/src/mndMain.c | 27 --------------------------- 1 file changed, 27 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index 3b2eafced8..bd82701ae4 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -393,11 +393,6 @@ int32_t mndProcessSyncMsg(SRpcMsg *pMsg) { return TAOS_SYNC_OTHER_ERROR; } - if (mndAcquireSyncRef(pMnode) != 0) { - mError("failed to process sync msg:%p type:%s since %s", pMsg, TMSG_INFO(pMsg->msgType), terrstr()); - return TAOS_SYNC_OTHER_ERROR; - } - char logBuf[512] = {0}; char *syncNodeStr = sync2SimpleStr(pMgmt->sync); snprintf(logBuf, sizeof(logBuf), "==vnodeProcessSyncReq== msgType:%d, syncNode: %s", pMsg->msgType, syncNodeStr); @@ -501,7 +496,6 @@ int32_t mndProcessSyncMsg(SRpcMsg *pMsg) { } } - mndReleaseSyncRef(pMnode); return code; } @@ -754,24 +748,3 @@ void mndSetStop(SMnode *pMnode) { } bool mndGetStop(SMnode *pMnode) { return pMnode->stopped; } - -int32_t mndAcquireSyncRef(SMnode *pMnode) { - int32_t code = 0; - taosThreadRwlockRdlock(&pMnode->lock); - if (pMnode->stopped) { - terrno = TSDB_CODE_APP_NOT_READY; - code = -1; - } else { - int32_t ref = atomic_add_fetch_32(&pMnode->syncRef, 1); - // mTrace("mnode sync is acquired, ref:%d", ref); - } - taosThreadRwlockUnlock(&pMnode->lock); - return code; -} - -void mndReleaseSyncRef(SMnode *pMnode) { - taosThreadRwlockRdlock(&pMnode->lock); - int32_t ref = atomic_sub_fetch_32(&pMnode->syncRef, 1); - // mTrace("mnode sync is released, ref:%d", ref); - taosThreadRwlockUnlock(&pMnode->lock); -} From faf1a814c9346dd71c59c98372703f47f497005e Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Tue, 14 Jun 2022 16:31:14 +0800 Subject: [PATCH 71/78] fix: some problems of parser --- source/libs/parser/inc/sql.y | 2 +- source/libs/parser/src/sql.c | 1209 +++++++++++++++++----------------- 2 files changed, 605 insertions(+), 606 deletions(-) diff --git a/source/libs/parser/inc/sql.y b/source/libs/parser/inc/sql.y index 8097695823..83ad41aac0 100644 --- a/source/libs/parser/inc/sql.y +++ b/source/libs/parser/inc/sql.y @@ -255,7 +255,7 @@ multi_create_clause(A) ::= multi_create_clause(B) create_subtable_clause(C). create_subtable_clause(A) ::= not_exists_opt(B) full_table_name(C) USING full_table_name(D) - specific_tags_opt(E) TAGS NK_LP expression_list(F) NK_RP table_options(G). { A = createCreateSubTableClause(pCxt, B, C, D, E, F, G); } + specific_tags_opt(E) TAGS NK_LP literal_list(F) NK_RP table_options(G). { A = createCreateSubTableClause(pCxt, B, C, D, E, F, G); } %type multi_drop_clause { SNodeList* } %destructor multi_drop_clause { nodesDestroyList($$); } diff --git a/source/libs/parser/src/sql.c b/source/libs/parser/src/sql.c index bde9a2bd30..b2de892f30 100644 --- a/source/libs/parser/src/sql.c +++ b/source/libs/parser/src/sql.c @@ -215,544 +215,543 @@ typedef union { ** yy_default[] Default action for each state. ** *********** Begin parsing tables **********************************************/ -#define YY_ACTTAB_COUNT (2178) +#define YY_ACTTAB_COUNT (2152) static const YYACTIONTYPE yy_action[] = { - /* 0 */ 475, 1650, 392, 1663, 393, 1395, 1584, 1794, 1361, 352, - /* 10 */ 11, 10, 36, 34, 37, 35, 33, 32, 31, 1793, - /* 20 */ 313, 1647, 1173, 1791, 71, 510, 37, 35, 33, 32, - /* 30 */ 31, 1679, 1471, 33, 32, 31, 1643, 1649, 302, 535, - /* 40 */ 37, 35, 33, 32, 31, 1487, 1679, 1171, 528, 534, - /* 50 */ 507, 1794, 1483, 1633, 500, 936, 469, 512, 14, 1538, - /* 60 */ 36, 34, 1300, 147, 1179, 322, 296, 1791, 313, 1692, - /* 70 */ 1173, 1536, 281, 82, 1664, 537, 1666, 1667, 533, 115, - /* 80 */ 528, 1, 1571, 1732, 1314, 1794, 40, 280, 1728, 159, - /* 90 */ 499, 1276, 1663, 940, 941, 1171, 1211, 148, 1794, 1794, - /* 100 */ 132, 1791, 1375, 615, 1262, 510, 14, 1794, 36, 34, - /* 110 */ 147, 149, 1179, 1172, 1791, 1791, 313, 113, 1173, 147, - /* 120 */ 1679, 1746, 300, 1791, 400, 567, 393, 1395, 532, 2, - /* 130 */ 130, 509, 144, 1739, 1740, 410, 1744, 63, 534, 1496, - /* 140 */ 54, 358, 1633, 1171, 566, 1743, 565, 564, 563, 1307, - /* 150 */ 110, 615, 201, 1263, 14, 1197, 1174, 382, 1692, 1489, - /* 160 */ 1179, 1172, 273, 1664, 537, 1666, 1667, 533, 531, 528, - /* 170 */ 525, 1704, 133, 129, 1268, 278, 1451, 2, 55, 1197, - /* 180 */ 66, 1177, 1178, 63, 1224, 1225, 1227, 1228, 1229, 1230, - /* 190 */ 1231, 530, 526, 1239, 1240, 1241, 1242, 1243, 1244, 615, - /* 200 */ 475, 161, 160, 299, 1174, 1490, 1583, 65, 279, 1172, - /* 210 */ 79, 150, 194, 28, 311, 1257, 1258, 1259, 1260, 1261, - /* 220 */ 1265, 1266, 1267, 112, 37, 35, 33, 32, 31, 1177, - /* 230 */ 1178, 1486, 1224, 1225, 1227, 1228, 1229, 1230, 1231, 530, - /* 240 */ 526, 1239, 1240, 1241, 1242, 1243, 1244, 953, 56, 952, - /* 250 */ 55, 96, 1174, 1386, 95, 94, 93, 92, 91, 90, - /* 260 */ 89, 88, 87, 510, 569, 36, 34, 37, 35, 33, - /* 270 */ 32, 31, 1385, 313, 343, 1173, 954, 1177, 1178, 435, - /* 280 */ 1224, 1225, 1227, 1228, 1229, 1230, 1231, 530, 526, 1239, - /* 290 */ 1240, 1241, 1242, 1243, 1244, 345, 341, 445, 444, 1384, - /* 300 */ 1171, 434, 443, 1633, 1663, 111, 440, 485, 150, 439, - /* 310 */ 438, 437, 397, 36, 34, 1245, 25, 1179, 1195, 152, - /* 320 */ 1226, 313, 1633, 1173, 39, 55, 37, 35, 33, 32, - /* 330 */ 31, 27, 1679, 1425, 8, 1794, 1494, 1211, 316, 1198, - /* 340 */ 535, 37, 35, 33, 32, 31, 130, 1792, 1171, 1633, - /* 350 */ 534, 1791, 611, 610, 1633, 1496, 615, 501, 512, 130, - /* 360 */ 1538, 36, 34, 514, 567, 1179, 1172, 301, 1497, 313, - /* 370 */ 1692, 1173, 1536, 1005, 82, 1664, 537, 1666, 1667, 533, - /* 380 */ 150, 528, 9, 566, 1732, 565, 564, 563, 280, 1728, - /* 390 */ 1007, 55, 391, 445, 444, 395, 1171, 1364, 443, 1653, - /* 400 */ 1794, 111, 440, 485, 615, 439, 438, 437, 1469, 1174, - /* 410 */ 29, 243, 147, 1179, 1172, 356, 1791, 562, 96, 142, - /* 420 */ 290, 95, 94, 93, 92, 91, 90, 89, 88, 87, - /* 430 */ 9, 1532, 1494, 496, 1177, 1178, 1655, 1224, 1225, 1227, - /* 440 */ 1228, 1229, 1230, 1231, 530, 526, 1239, 1240, 1241, 1242, - /* 450 */ 1243, 1244, 615, 318, 150, 150, 1538, 1174, 319, 1182, - /* 460 */ 1574, 1576, 1172, 317, 150, 351, 130, 350, 1536, 291, - /* 470 */ 399, 289, 288, 395, 433, 1496, 442, 441, 435, 1155, - /* 480 */ 1156, 195, 1177, 1178, 1363, 1224, 1225, 1227, 1228, 1229, - /* 490 */ 1230, 1231, 530, 526, 1239, 1240, 1241, 1242, 1243, 1244, - /* 500 */ 434, 567, 1199, 1621, 1383, 1174, 502, 497, 105, 104, - /* 510 */ 103, 102, 101, 100, 99, 98, 97, 459, 36, 34, - /* 520 */ 566, 150, 565, 564, 563, 1185, 313, 1663, 1173, 1485, - /* 530 */ 1177, 1178, 1651, 1224, 1225, 1227, 1228, 1229, 1230, 1231, - /* 540 */ 530, 526, 1239, 1240, 1241, 1242, 1243, 1244, 331, 1647, - /* 550 */ 1250, 584, 1647, 1171, 1633, 1679, 1197, 1746, 1196, 1794, - /* 560 */ 277, 1472, 1195, 511, 1643, 1649, 1382, 1643, 1649, 375, - /* 570 */ 1179, 147, 387, 534, 120, 1791, 528, 1633, 1663, 528, - /* 580 */ 157, 1742, 37, 35, 33, 32, 31, 2, 585, 583, - /* 590 */ 388, 485, 346, 1692, 1324, 1381, 952, 83, 1664, 537, - /* 600 */ 1666, 1667, 533, 106, 528, 61, 1679, 1732, 60, 615, - /* 610 */ 431, 306, 1728, 143, 532, 471, 1633, 485, 410, 1172, - /* 620 */ 1494, 429, 1264, 253, 534, 235, 1524, 234, 1633, 357, - /* 630 */ 486, 1759, 1746, 493, 1322, 1323, 1325, 1326, 37, 35, - /* 640 */ 33, 32, 31, 1269, 1692, 1633, 1494, 1299, 273, 1664, - /* 650 */ 537, 1666, 1667, 533, 572, 528, 1741, 1705, 1470, 1380, - /* 660 */ 386, 1479, 1174, 381, 380, 379, 378, 377, 374, 373, + /* 0 */ 28, 231, 1663, 1650, 611, 610, 297, 1650, 358, 1485, + /* 10 */ 314, 1483, 35, 33, 352, 36, 34, 32, 31, 30, + /* 20 */ 306, 24, 1173, 1647, 533, 442, 441, 1647, 79, 1647, + /* 30 */ 1679, 36, 34, 32, 31, 30, 152, 493, 517, 1643, + /* 40 */ 1649, 115, 278, 1643, 1649, 1643, 1649, 1171, 516, 1486, + /* 50 */ 536, 533, 1633, 1494, 536, 1794, 536, 497, 14, 1746, + /* 60 */ 35, 33, 1300, 356, 1179, 1663, 114, 147, 306, 1692, + /* 70 */ 1173, 1791, 82, 1664, 519, 1666, 1667, 515, 532, 536, + /* 80 */ 1494, 1, 1732, 1743, 1794, 1314, 280, 1728, 36, 34, + /* 90 */ 32, 31, 30, 1679, 410, 1171, 1793, 1651, 1794, 520, + /* 100 */ 1791, 517, 309, 615, 112, 1583, 14, 1746, 35, 33, + /* 110 */ 149, 516, 1179, 1172, 1791, 1633, 306, 1647, 1173, 145, + /* 120 */ 1739, 1740, 532, 1744, 36, 34, 32, 31, 30, 2, + /* 130 */ 63, 1742, 1692, 1643, 1649, 84, 1664, 519, 1666, 1667, + /* 140 */ 515, 1364, 536, 1171, 536, 1732, 953, 1794, 952, 1731, + /* 150 */ 1728, 615, 1490, 72, 14, 392, 1174, 393, 1395, 148, + /* 160 */ 1179, 1172, 96, 1791, 532, 95, 94, 93, 92, 91, + /* 170 */ 90, 89, 88, 87, 1487, 954, 56, 2, 132, 201, + /* 180 */ 1375, 1177, 1178, 39, 1224, 1225, 1227, 1228, 1229, 1230, + /* 190 */ 1231, 512, 534, 1239, 1240, 1241, 1242, 1243, 1244, 615, + /* 200 */ 400, 952, 393, 1395, 1174, 55, 1197, 66, 133, 1172, + /* 210 */ 96, 150, 1451, 95, 94, 93, 92, 91, 90, 89, + /* 220 */ 88, 87, 65, 279, 1356, 38, 429, 194, 567, 1177, + /* 230 */ 1178, 1363, 1224, 1225, 1227, 1228, 1229, 1230, 1231, 512, + /* 240 */ 534, 1239, 1240, 1241, 1242, 1243, 1244, 566, 487, 565, + /* 250 */ 564, 563, 1174, 55, 63, 105, 104, 103, 102, 101, + /* 260 */ 100, 99, 98, 97, 936, 35, 33, 110, 36, 34, + /* 270 */ 32, 31, 30, 306, 1746, 1173, 1489, 1177, 1178, 1198, + /* 280 */ 1224, 1225, 1227, 1228, 1229, 1230, 1231, 512, 534, 1239, + /* 290 */ 1240, 1241, 1242, 1243, 1244, 55, 445, 444, 1741, 1538, + /* 300 */ 1171, 443, 940, 941, 111, 440, 296, 1355, 439, 438, + /* 310 */ 437, 1536, 569, 35, 33, 1245, 1679, 1179, 26, 310, + /* 320 */ 1386, 306, 391, 1173, 486, 395, 290, 130, 36, 34, + /* 330 */ 32, 31, 30, 1196, 8, 150, 1496, 1043, 559, 558, + /* 340 */ 557, 1047, 556, 1049, 1050, 555, 1052, 552, 1171, 1058, + /* 350 */ 549, 1060, 1061, 546, 543, 150, 615, 450, 482, 618, + /* 360 */ 485, 35, 33, 585, 583, 1179, 1172, 142, 397, 306, + /* 370 */ 1633, 1173, 458, 248, 1195, 291, 1324, 289, 288, 1532, + /* 380 */ 433, 318, 9, 150, 435, 107, 193, 399, 1574, 1576, + /* 390 */ 395, 607, 603, 599, 595, 247, 1171, 1472, 453, 157, + /* 400 */ 533, 129, 1361, 447, 615, 1425, 434, 572, 192, 1174, + /* 410 */ 1794, 1385, 357, 1179, 1172, 479, 1322, 1323, 1325, 1326, + /* 420 */ 80, 1571, 1792, 242, 61, 150, 1791, 60, 159, 1494, + /* 430 */ 9, 488, 483, 51, 1177, 1178, 50, 1224, 1225, 1227, + /* 440 */ 1228, 1229, 1230, 1231, 512, 534, 1239, 1240, 1241, 1242, + /* 450 */ 1243, 1244, 615, 55, 410, 316, 529, 1174, 54, 319, + /* 460 */ 322, 1633, 1172, 130, 150, 445, 444, 130, 1538, 562, + /* 470 */ 443, 382, 1496, 111, 440, 311, 1496, 439, 438, 437, + /* 480 */ 1536, 474, 1177, 1178, 203, 1224, 1225, 1227, 1228, 1229, + /* 490 */ 1230, 1231, 512, 534, 1239, 1240, 1241, 1242, 1243, 1244, + /* 500 */ 1384, 1794, 1147, 1199, 197, 1174, 1575, 1576, 36, 34, + /* 510 */ 32, 31, 30, 147, 1307, 161, 160, 1791, 35, 33, + /* 520 */ 1197, 1663, 1155, 1156, 195, 1470, 306, 351, 1173, 350, + /* 530 */ 1177, 1178, 493, 1224, 1225, 1227, 1228, 1229, 1230, 1231, + /* 540 */ 512, 534, 1239, 1240, 1241, 1242, 1243, 1244, 253, 1679, + /* 550 */ 1633, 1524, 533, 1171, 1383, 1621, 533, 517, 1382, 1250, + /* 560 */ 277, 114, 1195, 1005, 367, 1197, 1479, 516, 106, 375, + /* 570 */ 1179, 1633, 387, 466, 533, 431, 497, 1422, 1211, 469, + /* 580 */ 1007, 1494, 569, 150, 1381, 1494, 368, 2, 1692, 343, + /* 590 */ 388, 82, 1664, 519, 1666, 1667, 515, 1481, 536, 112, + /* 600 */ 331, 1732, 1477, 1494, 1633, 280, 1728, 198, 1633, 615, + /* 610 */ 345, 341, 533, 495, 144, 1739, 1740, 1794, 1744, 1172, + /* 620 */ 1794, 940, 941, 1538, 106, 1376, 11, 10, 222, 147, + /* 630 */ 317, 436, 147, 1791, 1633, 1536, 1791, 591, 590, 589, + /* 640 */ 321, 1494, 588, 587, 586, 116, 581, 580, 579, 578, + /* 650 */ 577, 576, 575, 574, 123, 570, 32, 31, 30, 1380, + /* 660 */ 386, 1299, 1174, 381, 380, 379, 378, 377, 374, 373, /* 670 */ 372, 371, 370, 366, 365, 364, 363, 362, 361, 360, - /* 680 */ 359, 1379, 26, 1378, 281, 1422, 1481, 1177, 1178, 1200, - /* 690 */ 1224, 1225, 1227, 1228, 1229, 1230, 1231, 530, 526, 1239, - /* 700 */ 1240, 1241, 1242, 1243, 1244, 485, 131, 485, 507, 1633, - /* 710 */ 573, 259, 1466, 507, 1538, 569, 1262, 367, 1356, 320, - /* 720 */ 485, 1226, 1377, 257, 53, 1197, 1537, 52, 1374, 1373, - /* 730 */ 457, 1633, 106, 1633, 1494, 7, 1494, 115, 1663, 436, - /* 740 */ 1575, 1576, 115, 455, 162, 591, 590, 589, 321, 1494, - /* 750 */ 588, 587, 586, 116, 581, 580, 579, 578, 577, 576, - /* 760 */ 575, 574, 123, 570, 485, 1263, 1679, 1372, 1477, 55, - /* 770 */ 198, 485, 1633, 485, 511, 113, 368, 485, 1633, 1633, - /* 780 */ 113, 940, 941, 409, 534, 1491, 1268, 516, 1633, 1610, - /* 790 */ 145, 1739, 1740, 1494, 1744, 146, 1739, 1740, 1371, 1744, - /* 800 */ 1494, 1355, 1494, 529, 1692, 81, 1494, 1663, 83, 1664, - /* 810 */ 537, 1666, 1667, 533, 218, 528, 519, 1633, 1732, 1370, - /* 820 */ 1369, 524, 306, 1728, 143, 28, 311, 1257, 1258, 1259, - /* 830 */ 1260, 1261, 1265, 1266, 1267, 1679, 59, 58, 355, 1751, - /* 840 */ 1295, 156, 1760, 535, 1368, 1367, 349, 185, 1633, 187, - /* 850 */ 183, 1366, 186, 534, 1412, 1298, 561, 1633, 189, 276, - /* 860 */ 485, 188, 339, 1181, 337, 333, 329, 153, 324, 1633, - /* 870 */ 1633, 485, 467, 1692, 1407, 1663, 446, 83, 1664, 537, - /* 880 */ 1666, 1667, 533, 483, 528, 485, 485, 1732, 1405, 1494, - /* 890 */ 1226, 306, 1728, 1807, 1633, 1633, 448, 484, 244, 150, - /* 900 */ 1494, 1633, 1766, 1679, 11, 10, 191, 1663, 1295, 190, - /* 910 */ 451, 535, 466, 38, 1494, 1494, 977, 208, 1358, 1359, - /* 920 */ 1376, 534, 118, 119, 238, 1633, 120, 78, 46, 1184, - /* 930 */ 221, 38, 1452, 978, 38, 1679, 507, 74, 494, 460, - /* 940 */ 1663, 1692, 229, 535, 1254, 83, 1664, 537, 1666, 1667, - /* 950 */ 533, 1680, 528, 534, 1128, 1732, 38, 1633, 209, 306, - /* 960 */ 1728, 1807, 1396, 478, 215, 115, 428, 1036, 1679, 1321, - /* 970 */ 1789, 224, 1270, 1692, 517, 1232, 535, 83, 1664, 537, - /* 980 */ 1666, 1667, 533, 512, 528, 541, 534, 1732, 1663, 119, - /* 990 */ 1633, 306, 1728, 1807, 512, 1762, 1533, 252, 120, 121, - /* 1000 */ 508, 237, 1750, 113, 240, 520, 1692, 242, 3, 119, - /* 1010 */ 265, 1664, 537, 1666, 1667, 533, 1679, 528, 232, 1739, - /* 1020 */ 506, 5, 505, 323, 535, 1794, 1064, 1195, 326, 330, - /* 1030 */ 1068, 286, 1005, 287, 534, 249, 1794, 149, 1633, 1074, - /* 1040 */ 1072, 1791, 512, 369, 1573, 1139, 158, 376, 149, 384, - /* 1050 */ 122, 383, 1791, 385, 1692, 389, 1201, 390, 265, 1664, - /* 1060 */ 537, 1666, 1667, 533, 398, 528, 1663, 1204, 401, 165, - /* 1070 */ 402, 1203, 167, 1043, 559, 558, 557, 1047, 556, 1049, - /* 1080 */ 1050, 555, 1052, 552, 1794, 1058, 549, 1060, 1061, 546, - /* 1090 */ 543, 403, 1205, 170, 1679, 404, 147, 406, 172, 407, - /* 1100 */ 1791, 1202, 535, 408, 175, 62, 1663, 411, 178, 430, - /* 1110 */ 432, 1484, 534, 182, 1179, 1480, 1633, 86, 184, 124, - /* 1120 */ 295, 1615, 125, 1482, 1614, 250, 1478, 126, 461, 127, - /* 1130 */ 196, 462, 1692, 199, 1679, 473, 84, 1664, 537, 1666, - /* 1140 */ 1667, 533, 535, 528, 468, 202, 1732, 472, 465, 205, - /* 1150 */ 1731, 1728, 534, 491, 1582, 476, 1633, 470, 479, 1581, - /* 1160 */ 213, 70, 480, 298, 251, 1663, 1200, 219, 481, 495, - /* 1170 */ 6, 1753, 1692, 1773, 211, 1772, 84, 1664, 537, 1666, - /* 1180 */ 1667, 533, 504, 528, 1663, 1495, 1732, 488, 1295, 1199, - /* 1190 */ 523, 1728, 489, 1679, 490, 1763, 223, 305, 498, 114, - /* 1200 */ 41, 535, 518, 521, 307, 230, 19, 72, 539, 254, - /* 1210 */ 228, 534, 1679, 1467, 231, 1633, 246, 136, 260, 47, - /* 1220 */ 535, 137, 1747, 256, 258, 614, 275, 266, 1627, 1626, - /* 1230 */ 534, 1692, 57, 1625, 1633, 134, 1664, 537, 1666, 1667, - /* 1240 */ 533, 1622, 528, 1713, 325, 1810, 327, 1166, 328, 1167, - /* 1250 */ 1692, 618, 154, 236, 84, 1664, 537, 1666, 1667, 533, - /* 1260 */ 1663, 528, 332, 1790, 1732, 248, 515, 239, 1620, 1729, - /* 1270 */ 304, 303, 522, 334, 241, 336, 1619, 107, 513, 1808, - /* 1280 */ 1187, 335, 338, 607, 603, 599, 595, 247, 1679, 1618, - /* 1290 */ 340, 1617, 1663, 297, 342, 1616, 535, 344, 1600, 155, - /* 1300 */ 347, 348, 1173, 1142, 1141, 1180, 534, 1594, 1593, 353, - /* 1310 */ 1633, 1592, 80, 354, 1591, 216, 1111, 1566, 1565, 1564, - /* 1320 */ 1679, 1563, 1179, 1562, 1561, 487, 1692, 1171, 535, 1560, - /* 1330 */ 274, 1664, 537, 1666, 1667, 533, 1559, 528, 534, 450, - /* 1340 */ 1558, 1557, 1633, 1556, 1179, 1555, 1554, 1553, 482, 1552, - /* 1350 */ 1551, 1550, 1549, 117, 458, 1548, 1663, 1547, 1692, 1546, - /* 1360 */ 1545, 492, 274, 1664, 537, 1666, 1667, 533, 193, 528, - /* 1370 */ 1544, 1183, 1543, 474, 108, 1113, 203, 1542, 1541, 1540, - /* 1380 */ 453, 1539, 1424, 615, 1679, 447, 1392, 943, 163, 942, - /* 1390 */ 192, 1391, 535, 1172, 1147, 140, 197, 1608, 1602, 109, - /* 1400 */ 394, 164, 534, 1590, 169, 396, 1633, 171, 1589, 1663, - /* 1410 */ 1579, 1473, 1423, 174, 1188, 51, 1421, 971, 50, 1419, - /* 1420 */ 414, 1663, 1692, 1417, 1415, 413, 269, 1664, 537, 1666, - /* 1430 */ 1667, 533, 412, 528, 416, 417, 1174, 1679, 418, 1191, - /* 1440 */ 421, 420, 422, 424, 425, 535, 426, 1404, 1403, 1679, - /* 1450 */ 526, 1239, 1240, 1390, 310, 534, 1475, 535, 45, 1633, - /* 1460 */ 1078, 1177, 1178, 503, 1077, 1474, 582, 534, 1004, 1003, - /* 1470 */ 1413, 1633, 181, 1002, 1663, 1692, 1001, 584, 998, 134, - /* 1480 */ 1664, 537, 1666, 1667, 533, 997, 528, 1692, 996, 292, - /* 1490 */ 1408, 274, 1664, 537, 1666, 1667, 533, 293, 528, 449, - /* 1500 */ 1406, 294, 1679, 1389, 452, 454, 1388, 312, 456, 180, - /* 1510 */ 535, 85, 1607, 1149, 49, 1601, 463, 1588, 1587, 1586, - /* 1520 */ 534, 141, 1663, 1809, 1633, 128, 1578, 427, 423, 419, - /* 1530 */ 415, 179, 204, 464, 200, 67, 15, 1663, 1577, 210, - /* 1540 */ 1692, 207, 206, 477, 274, 1664, 537, 1666, 1667, 533, - /* 1550 */ 1679, 528, 48, 68, 214, 314, 64, 212, 535, 177, - /* 1560 */ 74, 69, 4, 220, 38, 1679, 217, 1318, 534, 222, - /* 1570 */ 1189, 1320, 1633, 535, 44, 1313, 226, 1663, 135, 225, - /* 1580 */ 16, 227, 23, 534, 1653, 73, 17, 1633, 1692, 1292, - /* 1590 */ 24, 1291, 274, 1664, 537, 1666, 1667, 533, 233, 528, - /* 1600 */ 43, 1652, 138, 1692, 18, 1679, 1349, 261, 1664, 537, - /* 1610 */ 1666, 1667, 533, 535, 528, 42, 176, 1663, 168, 13, - /* 1620 */ 173, 1338, 405, 534, 1344, 10, 1343, 1633, 308, 1348, - /* 1630 */ 1347, 309, 20, 1255, 1695, 1219, 1663, 1236, 1234, 139, - /* 1640 */ 166, 527, 30, 1692, 1233, 1679, 12, 268, 1664, 537, - /* 1650 */ 1666, 1667, 533, 535, 528, 21, 151, 536, 538, 1042, - /* 1660 */ 560, 22, 540, 534, 1679, 315, 542, 1633, 1065, 1062, - /* 1670 */ 545, 544, 535, 547, 1059, 548, 550, 553, 1053, 551, - /* 1680 */ 1057, 1051, 534, 1692, 554, 75, 1633, 270, 1664, 537, - /* 1690 */ 1666, 1667, 533, 1663, 528, 1056, 1073, 76, 77, 1070, - /* 1700 */ 969, 1071, 1692, 568, 993, 1055, 262, 1664, 537, 1666, - /* 1710 */ 1667, 533, 1054, 528, 1011, 571, 245, 991, 990, 989, - /* 1720 */ 986, 1679, 1008, 988, 1420, 1663, 987, 985, 984, 535, - /* 1730 */ 1006, 981, 980, 979, 976, 1418, 975, 974, 592, 534, - /* 1740 */ 593, 594, 598, 1633, 596, 597, 1416, 601, 600, 602, - /* 1750 */ 1414, 605, 606, 1679, 604, 1402, 608, 609, 1401, 1692, - /* 1760 */ 1387, 535, 613, 271, 1664, 537, 1666, 1667, 533, 612, - /* 1770 */ 528, 534, 616, 1663, 1175, 1633, 255, 617, 1362, 1362, - /* 1780 */ 1362, 1362, 1362, 1362, 1362, 1362, 1362, 1362, 1663, 1362, - /* 1790 */ 1362, 1692, 1362, 1362, 1362, 263, 1664, 537, 1666, 1667, - /* 1800 */ 533, 1679, 528, 1362, 1362, 1362, 1362, 1362, 1362, 535, - /* 1810 */ 1362, 1362, 1362, 1362, 1362, 1362, 1679, 1362, 1362, 534, - /* 1820 */ 1362, 1663, 1362, 1633, 535, 1362, 1362, 1362, 1362, 1362, - /* 1830 */ 1362, 1362, 1362, 1362, 534, 1362, 1663, 1362, 1633, 1692, - /* 1840 */ 1362, 1362, 1362, 272, 1664, 537, 1666, 1667, 533, 1679, - /* 1850 */ 528, 1362, 1362, 1663, 1692, 1362, 1362, 535, 264, 1664, - /* 1860 */ 537, 1666, 1667, 533, 1679, 528, 1362, 534, 1362, 1362, - /* 1870 */ 1362, 1633, 535, 1362, 1362, 1362, 1362, 1362, 1362, 1362, - /* 1880 */ 1362, 1679, 534, 1362, 1362, 1362, 1633, 1692, 1362, 535, - /* 1890 */ 1362, 1675, 1664, 537, 1666, 1667, 533, 1362, 528, 534, - /* 1900 */ 1362, 1362, 1692, 1633, 1362, 1362, 1674, 1664, 537, 1666, - /* 1910 */ 1667, 533, 1362, 528, 1362, 1362, 1663, 1362, 1362, 1692, - /* 1920 */ 1362, 1362, 1362, 1673, 1664, 537, 1666, 1667, 533, 1362, - /* 1930 */ 528, 1663, 1362, 1362, 1362, 1362, 1362, 1362, 1362, 1362, - /* 1940 */ 1362, 1362, 1362, 1362, 1679, 1362, 1362, 1362, 1362, 1362, - /* 1950 */ 1362, 1362, 535, 1362, 1362, 1362, 1362, 1362, 1362, 1679, - /* 1960 */ 1362, 1362, 534, 1362, 1362, 1362, 1633, 535, 1362, 1362, - /* 1970 */ 1362, 1362, 1362, 1362, 1362, 1362, 1362, 534, 1362, 1362, - /* 1980 */ 1362, 1633, 1692, 1663, 1362, 1362, 284, 1664, 537, 1666, - /* 1990 */ 1667, 533, 1362, 528, 1362, 1362, 1362, 1692, 1362, 1362, - /* 2000 */ 1362, 283, 1664, 537, 1666, 1667, 533, 1362, 528, 1362, - /* 2010 */ 1362, 1679, 1362, 1362, 1362, 1663, 1362, 1362, 1362, 535, - /* 2020 */ 1362, 1362, 1362, 1362, 1362, 1362, 1362, 1362, 1362, 534, - /* 2030 */ 1362, 1362, 1362, 1633, 1362, 1362, 1362, 1362, 1362, 1362, - /* 2040 */ 1362, 1362, 1362, 1679, 1362, 1362, 1362, 1362, 1362, 1692, - /* 2050 */ 1362, 535, 1362, 285, 1664, 537, 1666, 1667, 533, 1362, - /* 2060 */ 528, 534, 1362, 1663, 1362, 1633, 1362, 1362, 1362, 1362, - /* 2070 */ 1362, 1362, 507, 1362, 1362, 1362, 1362, 1362, 1362, 1362, - /* 2080 */ 1362, 1692, 1362, 1362, 1362, 282, 1664, 537, 1666, 1667, - /* 2090 */ 533, 1679, 528, 1362, 1362, 1362, 1362, 1362, 1362, 535, - /* 2100 */ 1362, 115, 1362, 1362, 1362, 1362, 1362, 1362, 1362, 534, - /* 2110 */ 1362, 1362, 1362, 1633, 1362, 1362, 1362, 1362, 1362, 512, - /* 2120 */ 1362, 1362, 1362, 1362, 1362, 1362, 1362, 1362, 1362, 1692, - /* 2130 */ 1362, 1362, 1362, 267, 1664, 537, 1666, 1667, 533, 113, - /* 2140 */ 528, 1362, 1362, 1362, 1362, 1362, 1362, 1362, 1362, 1362, - /* 2150 */ 1362, 1362, 1362, 1362, 232, 1739, 506, 1362, 505, 1362, - /* 2160 */ 1362, 1794, 1362, 1362, 1362, 1362, 1362, 1362, 1362, 1362, - /* 2170 */ 1362, 1362, 1362, 147, 1362, 1362, 1362, 1791, + /* 680 */ 359, 499, 1663, 520, 1379, 1226, 1197, 1177, 1178, 1584, + /* 690 */ 1224, 1225, 1227, 1228, 1229, 1230, 1231, 512, 534, 1239, + /* 700 */ 1240, 1241, 1242, 1243, 1244, 1264, 131, 1276, 1378, 1633, + /* 710 */ 1679, 259, 573, 533, 1466, 7, 511, 533, 496, 1471, + /* 720 */ 1200, 533, 459, 257, 53, 409, 1269, 52, 516, 1491, + /* 730 */ 1226, 533, 1633, 1610, 1633, 533, 36, 34, 32, 31, + /* 740 */ 30, 1377, 1494, 244, 162, 1374, 1494, 467, 1663, 1692, + /* 750 */ 1494, 501, 83, 1664, 519, 1666, 1667, 515, 1633, 536, + /* 760 */ 1494, 1373, 1732, 1794, 1494, 25, 299, 1728, 143, 55, + /* 770 */ 36, 34, 32, 31, 30, 147, 1679, 1469, 533, 1791, + /* 780 */ 223, 533, 281, 130, 517, 475, 1759, 435, 533, 1372, + /* 790 */ 530, 1633, 1497, 531, 516, 1633, 185, 1371, 1633, 183, + /* 800 */ 320, 476, 1370, 1369, 1663, 81, 1211, 1494, 1368, 434, + /* 810 */ 1494, 1633, 567, 504, 1262, 1692, 281, 1494, 274, 1664, + /* 820 */ 519, 1666, 1667, 515, 509, 536, 1751, 1295, 493, 561, + /* 830 */ 457, 566, 1679, 565, 564, 563, 59, 58, 355, 1633, + /* 840 */ 514, 156, 1452, 455, 1538, 207, 349, 1633, 1262, 584, + /* 850 */ 516, 1226, 1633, 1633, 1633, 1367, 1537, 114, 1633, 276, + /* 860 */ 1173, 226, 339, 1263, 337, 333, 329, 153, 324, 1298, + /* 870 */ 567, 1692, 1412, 1182, 273, 1664, 519, 1666, 1667, 515, + /* 880 */ 513, 536, 510, 1704, 1268, 1171, 1366, 1407, 1295, 566, + /* 890 */ 346, 565, 564, 563, 446, 112, 120, 1263, 187, 150, + /* 900 */ 189, 186, 1179, 188, 191, 1633, 1663, 190, 46, 448, + /* 910 */ 146, 1739, 1740, 1405, 1744, 11, 10, 1653, 1268, 1358, + /* 920 */ 1359, 480, 1181, 27, 304, 1257, 1258, 1259, 1260, 1261, + /* 930 */ 1265, 1266, 1267, 210, 1679, 451, 1633, 471, 502, 1185, + /* 940 */ 78, 615, 496, 37, 37, 460, 37, 1254, 233, 1321, + /* 950 */ 74, 1172, 516, 217, 1655, 1680, 1633, 27, 304, 1257, + /* 960 */ 1258, 1259, 1260, 1261, 1265, 1266, 1267, 118, 1663, 1396, + /* 970 */ 119, 1533, 120, 1692, 212, 46, 83, 1664, 519, 1666, + /* 980 */ 1667, 515, 977, 536, 1270, 1232, 1732, 1128, 1184, 235, + /* 990 */ 299, 1728, 143, 541, 1174, 428, 1679, 1762, 494, 978, + /* 1000 */ 1663, 225, 505, 119, 517, 228, 120, 230, 525, 3, + /* 1010 */ 1760, 241, 5, 1036, 516, 121, 252, 119, 1633, 1177, + /* 1020 */ 1178, 323, 1195, 326, 330, 286, 287, 1005, 1679, 249, + /* 1030 */ 1139, 369, 1573, 376, 1064, 1692, 517, 158, 83, 1664, + /* 1040 */ 519, 1666, 1667, 515, 1068, 536, 516, 1074, 1732, 384, + /* 1050 */ 1633, 389, 299, 1728, 1807, 383, 1072, 1201, 122, 385, + /* 1060 */ 1663, 390, 1204, 1766, 401, 398, 165, 1692, 402, 167, + /* 1070 */ 83, 1664, 519, 1666, 1667, 515, 1203, 536, 1205, 404, + /* 1080 */ 1732, 403, 170, 406, 299, 1728, 1807, 172, 1679, 407, + /* 1090 */ 1202, 175, 1663, 408, 62, 1789, 517, 411, 178, 430, + /* 1100 */ 432, 1484, 182, 86, 1480, 184, 516, 295, 124, 1179, + /* 1110 */ 1633, 125, 1482, 1478, 1615, 126, 127, 1614, 196, 461, + /* 1120 */ 1679, 199, 250, 202, 465, 462, 1200, 1692, 517, 468, + /* 1130 */ 83, 1664, 519, 1666, 1667, 515, 472, 536, 516, 205, + /* 1140 */ 1732, 481, 1633, 1663, 299, 1728, 1807, 497, 470, 478, + /* 1150 */ 473, 523, 1773, 1772, 298, 1750, 490, 208, 1763, 1692, + /* 1160 */ 211, 6, 264, 1664, 519, 1666, 1667, 515, 1753, 536, + /* 1170 */ 484, 1679, 216, 1663, 477, 113, 1295, 218, 1199, 517, + /* 1180 */ 40, 300, 1747, 506, 503, 18, 527, 1582, 1794, 516, + /* 1190 */ 137, 521, 522, 1633, 219, 1581, 526, 308, 497, 237, + /* 1200 */ 149, 1679, 528, 224, 1791, 251, 1663, 1495, 1713, 517, + /* 1210 */ 1692, 1810, 1790, 264, 1664, 519, 1666, 1667, 515, 516, + /* 1220 */ 536, 239, 71, 1633, 73, 539, 246, 254, 500, 614, + /* 1230 */ 136, 227, 1467, 229, 1679, 507, 1663, 265, 47, 1794, + /* 1240 */ 1692, 256, 517, 84, 1664, 519, 1666, 1667, 515, 258, + /* 1250 */ 536, 147, 516, 1732, 275, 1791, 1633, 508, 1728, 266, + /* 1260 */ 1627, 1626, 57, 1625, 1679, 325, 1622, 327, 328, 332, + /* 1270 */ 1166, 1167, 517, 1692, 154, 1620, 134, 1664, 519, 1666, + /* 1280 */ 1667, 515, 516, 536, 334, 335, 1633, 1663, 336, 1619, + /* 1290 */ 338, 1618, 340, 1617, 342, 1616, 1663, 344, 1600, 155, + /* 1300 */ 1142, 347, 1141, 1692, 348, 1594, 84, 1664, 519, 1666, + /* 1310 */ 1667, 515, 1593, 536, 353, 1679, 1732, 354, 1592, 498, + /* 1320 */ 1808, 1729, 1591, 517, 1679, 1111, 1566, 1565, 1564, 1563, + /* 1330 */ 1562, 1561, 517, 516, 1560, 1559, 1558, 1633, 1557, 1556, + /* 1340 */ 1555, 1554, 516, 1553, 1552, 1551, 1633, 1550, 1549, 117, + /* 1350 */ 1663, 1548, 1547, 1546, 1692, 1545, 1544, 269, 1664, 519, + /* 1360 */ 1666, 1667, 515, 1692, 536, 1543, 134, 1664, 519, 1666, + /* 1370 */ 1667, 515, 1113, 536, 1542, 1541, 1540, 1539, 1679, 1424, + /* 1380 */ 1392, 943, 163, 108, 942, 1391, 517, 1608, 1602, 1590, + /* 1390 */ 1663, 171, 1589, 394, 489, 140, 516, 164, 109, 169, + /* 1400 */ 1633, 396, 1579, 303, 45, 174, 1663, 1473, 1423, 1421, + /* 1410 */ 1809, 1419, 1417, 412, 413, 414, 417, 1692, 1679, 971, + /* 1420 */ 274, 1664, 519, 1666, 1667, 515, 514, 536, 416, 420, + /* 1430 */ 418, 421, 422, 1415, 1679, 426, 516, 424, 1404, 425, + /* 1440 */ 1633, 1403, 517, 1390, 1475, 1077, 181, 1078, 1474, 1413, + /* 1450 */ 1004, 582, 516, 1003, 1663, 584, 1633, 1692, 292, 305, + /* 1460 */ 273, 1664, 519, 1666, 1667, 515, 1408, 536, 1002, 1705, + /* 1470 */ 449, 1001, 998, 1692, 997, 996, 274, 1664, 519, 1666, + /* 1480 */ 1667, 515, 1679, 536, 293, 1406, 1663, 294, 1389, 452, + /* 1490 */ 517, 180, 1388, 454, 456, 85, 1607, 1149, 1601, 463, + /* 1500 */ 516, 1588, 1587, 141, 1633, 1586, 1578, 307, 49, 427, + /* 1510 */ 423, 419, 415, 179, 1679, 67, 1663, 204, 464, 4, + /* 1520 */ 37, 1692, 517, 206, 274, 1664, 519, 1666, 1667, 515, + /* 1530 */ 15, 536, 516, 128, 209, 43, 1633, 1320, 64, 200, + /* 1540 */ 135, 177, 213, 215, 1679, 22, 48, 1653, 214, 1313, + /* 1550 */ 68, 23, 517, 1692, 42, 1292, 260, 1664, 519, 1666, + /* 1560 */ 1667, 515, 516, 536, 221, 1291, 1633, 1663, 138, 1349, + /* 1570 */ 17, 1338, 1344, 1343, 301, 1348, 10, 1347, 302, 19, + /* 1580 */ 1255, 139, 1234, 1692, 151, 29, 268, 1664, 519, 1666, + /* 1590 */ 1667, 515, 12, 536, 1233, 1679, 20, 1219, 176, 16, + /* 1600 */ 168, 518, 173, 517, 405, 41, 13, 1663, 1577, 238, + /* 1610 */ 74, 524, 1652, 516, 232, 21, 234, 1633, 1318, 1189, + /* 1620 */ 236, 240, 166, 69, 70, 243, 1695, 540, 1236, 535, + /* 1630 */ 44, 1071, 1065, 538, 1692, 1679, 315, 270, 1664, 519, + /* 1640 */ 1666, 1667, 515, 517, 536, 542, 544, 1663, 1062, 1059, + /* 1650 */ 545, 547, 548, 516, 550, 1053, 553, 1633, 551, 1057, + /* 1660 */ 1051, 1056, 1663, 554, 1055, 1054, 75, 1042, 76, 560, + /* 1670 */ 1073, 77, 1070, 568, 1692, 1679, 969, 261, 1664, 519, + /* 1680 */ 1666, 1667, 515, 517, 536, 993, 1011, 245, 991, 571, + /* 1690 */ 1679, 986, 1008, 516, 990, 989, 988, 1633, 517, 987, + /* 1700 */ 985, 984, 1006, 981, 980, 979, 976, 975, 516, 974, + /* 1710 */ 1420, 592, 1633, 1663, 1692, 593, 594, 271, 1664, 519, + /* 1720 */ 1666, 1667, 515, 1418, 536, 1663, 597, 596, 598, 1692, + /* 1730 */ 1416, 600, 262, 1664, 519, 1666, 1667, 515, 601, 536, + /* 1740 */ 602, 1679, 604, 605, 606, 1402, 608, 609, 1401, 517, + /* 1750 */ 1414, 1387, 613, 1679, 612, 1175, 255, 616, 617, 516, + /* 1760 */ 1362, 517, 1362, 1633, 1362, 1663, 1362, 1362, 1362, 1362, + /* 1770 */ 1362, 516, 1362, 1362, 1362, 1633, 1663, 1362, 1362, 1362, + /* 1780 */ 1692, 1362, 1362, 272, 1664, 519, 1666, 1667, 515, 1362, + /* 1790 */ 536, 1362, 1692, 1679, 1362, 263, 1664, 519, 1666, 1667, + /* 1800 */ 515, 517, 536, 1362, 1679, 1362, 1362, 1362, 1362, 1362, + /* 1810 */ 1362, 516, 517, 1362, 1362, 1633, 1362, 1362, 1362, 1362, + /* 1820 */ 1362, 1362, 516, 1362, 1362, 1362, 1633, 1663, 1362, 1362, + /* 1830 */ 1362, 1362, 1692, 1362, 1362, 1675, 1664, 519, 1666, 1667, + /* 1840 */ 515, 1362, 536, 1692, 1362, 1362, 1674, 1664, 519, 1666, + /* 1850 */ 1667, 515, 1362, 536, 1362, 1679, 1362, 1663, 1362, 1362, + /* 1860 */ 1362, 1362, 1362, 517, 1362, 1362, 1362, 1362, 1362, 1362, + /* 1870 */ 1362, 1362, 1362, 516, 1362, 1362, 1362, 1633, 1362, 1362, + /* 1880 */ 1362, 1362, 1362, 1362, 1362, 1679, 1362, 1663, 1362, 1362, + /* 1890 */ 1362, 1362, 1362, 517, 1692, 1362, 1362, 1673, 1664, 519, + /* 1900 */ 1666, 1667, 515, 516, 536, 1362, 1362, 1633, 1362, 1362, + /* 1910 */ 1362, 1362, 1362, 1362, 1362, 1679, 1362, 1362, 1362, 1663, + /* 1920 */ 1362, 1362, 1362, 517, 1692, 1362, 1362, 284, 1664, 519, + /* 1930 */ 1666, 1667, 515, 516, 536, 1362, 1362, 1633, 1362, 1362, + /* 1940 */ 1362, 1362, 1362, 313, 312, 1362, 1362, 1679, 1362, 1362, + /* 1950 */ 1362, 1362, 1663, 1187, 1692, 517, 1362, 283, 1664, 519, + /* 1960 */ 1666, 1667, 515, 1362, 536, 516, 1362, 1362, 1362, 1633, + /* 1970 */ 1362, 1362, 1362, 1362, 1362, 1362, 1362, 1362, 1180, 1362, + /* 1980 */ 1679, 1362, 1362, 1362, 1362, 1663, 1692, 1362, 517, 285, + /* 1990 */ 1664, 519, 1666, 1667, 515, 1179, 536, 1362, 516, 1362, + /* 2000 */ 1362, 1362, 1633, 1362, 1362, 1362, 1362, 1362, 1362, 1362, + /* 2010 */ 1362, 1362, 1362, 1679, 1362, 1362, 493, 1362, 1362, 1692, + /* 2020 */ 1362, 517, 282, 1664, 519, 1666, 1667, 515, 1362, 536, + /* 2030 */ 1362, 516, 1362, 1362, 537, 1633, 1362, 1362, 1362, 1362, + /* 2040 */ 1362, 1362, 1362, 1362, 1183, 114, 493, 1362, 1362, 1362, + /* 2050 */ 1362, 1362, 1692, 1362, 1362, 267, 1664, 519, 1666, 1667, + /* 2060 */ 515, 1362, 536, 1362, 497, 1362, 1362, 1362, 1362, 1362, + /* 2070 */ 1362, 1362, 1362, 1362, 1362, 114, 1362, 1362, 1362, 1362, + /* 2080 */ 1362, 1362, 1362, 112, 1362, 1362, 1362, 1188, 1362, 1362, + /* 2090 */ 1362, 1362, 1362, 1362, 497, 1362, 1362, 1362, 220, 1739, + /* 2100 */ 492, 1362, 491, 1362, 1362, 1794, 1362, 1362, 1362, 1362, + /* 2110 */ 1362, 1362, 1191, 112, 1362, 1362, 1362, 149, 1362, 1362, + /* 2120 */ 1362, 1791, 1362, 534, 1239, 1240, 1362, 1362, 220, 1739, + /* 2130 */ 492, 1362, 491, 1362, 1362, 1794, 1362, 1362, 1362, 1362, + /* 2140 */ 1362, 1362, 1362, 1362, 1362, 1362, 1362, 147, 1362, 1362, + /* 2150 */ 1362, 1791, }; static const YYCODETYPE yy_lookahead[] = { - /* 0 */ 289, 273, 246, 243, 248, 249, 295, 339, 240, 297, - /* 10 */ 1, 2, 12, 13, 12, 13, 14, 15, 16, 351, - /* 20 */ 20, 293, 22, 355, 253, 20, 12, 13, 14, 15, - /* 30 */ 16, 271, 0, 14, 15, 16, 308, 309, 310, 279, - /* 40 */ 12, 13, 14, 15, 16, 274, 271, 47, 320, 289, - /* 50 */ 250, 339, 272, 293, 279, 4, 297, 297, 58, 271, - /* 60 */ 12, 13, 14, 351, 64, 297, 278, 355, 20, 309, - /* 70 */ 22, 283, 58, 313, 314, 315, 316, 317, 318, 279, - /* 80 */ 320, 81, 279, 323, 82, 339, 81, 327, 328, 286, - /* 90 */ 315, 82, 243, 42, 43, 47, 82, 351, 339, 339, - /* 100 */ 242, 355, 244, 103, 90, 20, 58, 339, 12, 13, - /* 110 */ 351, 351, 64, 113, 355, 355, 20, 317, 22, 351, - /* 120 */ 271, 311, 263, 355, 246, 93, 248, 249, 279, 81, - /* 130 */ 271, 331, 332, 333, 334, 57, 336, 255, 289, 280, - /* 140 */ 3, 250, 293, 47, 112, 335, 114, 115, 116, 14, - /* 150 */ 268, 103, 55, 139, 58, 20, 156, 75, 309, 277, - /* 160 */ 64, 113, 313, 314, 315, 316, 317, 318, 319, 320, - /* 170 */ 321, 322, 256, 145, 160, 284, 260, 81, 81, 20, - /* 180 */ 83, 181, 182, 255, 184, 185, 186, 187, 188, 189, + /* 0 */ 324, 325, 243, 273, 251, 252, 276, 273, 250, 273, + /* 10 */ 276, 272, 12, 13, 298, 12, 13, 14, 15, 16, + /* 20 */ 20, 2, 22, 293, 250, 257, 258, 293, 253, 293, + /* 30 */ 271, 12, 13, 14, 15, 16, 262, 250, 279, 309, + /* 40 */ 310, 266, 284, 309, 310, 309, 310, 47, 289, 274, + /* 50 */ 320, 250, 293, 279, 320, 339, 320, 298, 58, 311, + /* 60 */ 12, 13, 14, 262, 64, 243, 279, 351, 20, 310, + /* 70 */ 22, 355, 313, 314, 315, 316, 317, 318, 20, 320, + /* 80 */ 279, 81, 323, 335, 339, 82, 327, 328, 12, 13, + /* 90 */ 14, 15, 16, 271, 57, 47, 351, 273, 339, 289, + /* 100 */ 355, 279, 292, 103, 317, 295, 58, 311, 12, 13, + /* 110 */ 351, 289, 64, 113, 355, 293, 20, 293, 22, 332, + /* 120 */ 333, 334, 20, 336, 12, 13, 14, 15, 16, 81, + /* 130 */ 255, 335, 310, 309, 310, 313, 314, 315, 316, 317, + /* 140 */ 318, 0, 320, 47, 320, 323, 20, 339, 22, 327, + /* 150 */ 328, 103, 277, 253, 58, 246, 156, 248, 249, 351, + /* 160 */ 64, 113, 21, 355, 20, 24, 25, 26, 27, 28, + /* 170 */ 29, 30, 31, 32, 274, 49, 4, 81, 242, 55, + /* 180 */ 244, 181, 182, 81, 184, 185, 186, 187, 188, 189, /* 190 */ 190, 191, 192, 193, 194, 195, 196, 197, 198, 103, - /* 200 */ 289, 119, 120, 292, 156, 277, 295, 165, 166, 113, - /* 210 */ 253, 211, 170, 199, 200, 201, 202, 203, 204, 205, - /* 220 */ 206, 207, 208, 266, 12, 13, 14, 15, 16, 181, - /* 230 */ 182, 274, 184, 185, 186, 187, 188, 189, 190, 191, - /* 240 */ 192, 193, 194, 195, 196, 197, 198, 20, 4, 22, - /* 250 */ 81, 21, 156, 243, 24, 25, 26, 27, 28, 29, - /* 260 */ 30, 31, 32, 20, 57, 12, 13, 12, 13, 14, - /* 270 */ 15, 16, 243, 20, 151, 22, 49, 181, 182, 93, + /* 200 */ 246, 22, 248, 249, 156, 81, 20, 83, 256, 113, + /* 210 */ 21, 211, 260, 24, 25, 26, 27, 28, 29, 30, + /* 220 */ 31, 32, 165, 166, 148, 81, 47, 170, 93, 181, + /* 230 */ 182, 0, 184, 185, 186, 187, 188, 189, 190, 191, + /* 240 */ 192, 193, 194, 195, 196, 197, 198, 112, 20, 114, + /* 250 */ 115, 116, 156, 81, 255, 24, 25, 26, 27, 28, + /* 260 */ 29, 30, 31, 32, 4, 12, 13, 268, 12, 13, + /* 270 */ 14, 15, 16, 20, 311, 22, 277, 181, 182, 20, /* 280 */ 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, - /* 290 */ 194, 195, 196, 197, 198, 172, 173, 60, 61, 243, - /* 300 */ 47, 115, 65, 293, 243, 68, 69, 250, 211, 72, - /* 310 */ 73, 74, 14, 12, 13, 14, 2, 64, 20, 262, - /* 320 */ 185, 20, 293, 22, 81, 81, 12, 13, 14, 15, - /* 330 */ 16, 2, 271, 0, 81, 339, 279, 82, 263, 20, - /* 340 */ 279, 12, 13, 14, 15, 16, 271, 351, 47, 293, - /* 350 */ 289, 355, 251, 252, 293, 280, 103, 20, 297, 271, - /* 360 */ 271, 12, 13, 226, 93, 64, 113, 278, 280, 20, - /* 370 */ 309, 22, 283, 47, 313, 314, 315, 316, 317, 318, - /* 380 */ 211, 320, 81, 112, 323, 114, 115, 116, 327, 328, - /* 390 */ 64, 81, 247, 60, 61, 250, 47, 0, 65, 44, - /* 400 */ 339, 68, 69, 250, 103, 72, 73, 74, 0, 156, - /* 410 */ 324, 325, 351, 64, 113, 262, 355, 92, 21, 270, - /* 420 */ 35, 24, 25, 26, 27, 28, 29, 30, 31, 32, - /* 430 */ 81, 282, 279, 143, 181, 182, 81, 184, 185, 186, + /* 290 */ 194, 195, 196, 197, 198, 81, 60, 61, 335, 271, + /* 300 */ 47, 65, 42, 43, 68, 69, 278, 231, 72, 73, + /* 310 */ 74, 283, 57, 12, 13, 14, 271, 64, 2, 263, + /* 320 */ 243, 20, 247, 22, 279, 250, 35, 271, 12, 13, + /* 330 */ 14, 15, 16, 20, 81, 211, 280, 94, 95, 96, + /* 340 */ 97, 98, 99, 100, 101, 102, 103, 104, 47, 106, + /* 350 */ 107, 108, 109, 110, 111, 211, 103, 4, 143, 19, + /* 360 */ 315, 12, 13, 257, 258, 64, 113, 270, 14, 20, + /* 370 */ 293, 22, 19, 33, 20, 84, 181, 86, 87, 282, + /* 380 */ 89, 281, 81, 211, 93, 45, 33, 247, 288, 289, + /* 390 */ 250, 51, 52, 53, 54, 55, 47, 0, 45, 55, + /* 400 */ 250, 145, 240, 50, 103, 0, 115, 64, 55, 156, + /* 410 */ 339, 243, 262, 64, 113, 220, 221, 222, 223, 224, + /* 420 */ 80, 279, 351, 83, 80, 211, 355, 83, 286, 279, + /* 430 */ 81, 216, 217, 80, 181, 182, 83, 184, 185, 186, /* 440 */ 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, - /* 450 */ 197, 198, 103, 281, 211, 211, 271, 156, 263, 47, - /* 460 */ 288, 289, 113, 278, 211, 155, 271, 157, 283, 84, - /* 470 */ 247, 86, 87, 250, 89, 280, 257, 258, 93, 167, - /* 480 */ 168, 169, 181, 182, 0, 184, 185, 186, 187, 188, + /* 450 */ 197, 198, 103, 81, 57, 263, 116, 156, 3, 263, + /* 460 */ 298, 293, 113, 271, 211, 60, 61, 271, 271, 92, + /* 470 */ 65, 75, 280, 68, 69, 278, 280, 72, 73, 74, + /* 480 */ 283, 141, 181, 182, 144, 184, 185, 186, 187, 188, /* 490 */ 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, - /* 500 */ 115, 93, 20, 0, 243, 156, 216, 217, 24, 25, - /* 510 */ 26, 27, 28, 29, 30, 31, 32, 297, 12, 13, - /* 520 */ 112, 211, 114, 115, 116, 113, 20, 243, 22, 273, - /* 530 */ 181, 182, 273, 184, 185, 186, 187, 188, 189, 190, - /* 540 */ 191, 192, 193, 194, 195, 196, 197, 198, 45, 293, - /* 550 */ 14, 41, 293, 47, 293, 271, 20, 311, 20, 339, - /* 560 */ 18, 0, 20, 279, 308, 309, 243, 308, 309, 27, - /* 570 */ 64, 351, 30, 289, 41, 355, 320, 293, 243, 320, - /* 580 */ 55, 335, 12, 13, 14, 15, 16, 81, 257, 258, - /* 590 */ 48, 250, 82, 309, 181, 243, 22, 313, 314, 315, - /* 600 */ 316, 317, 318, 262, 320, 80, 271, 323, 83, 103, - /* 610 */ 269, 327, 328, 329, 279, 82, 293, 250, 57, 113, - /* 620 */ 279, 47, 139, 264, 289, 341, 267, 145, 293, 262, - /* 630 */ 346, 347, 311, 220, 221, 222, 223, 224, 12, 13, - /* 640 */ 14, 15, 16, 160, 309, 293, 279, 4, 313, 314, - /* 650 */ 315, 316, 317, 318, 64, 320, 335, 322, 0, 243, - /* 660 */ 118, 272, 156, 121, 122, 123, 124, 125, 126, 127, + /* 500 */ 243, 339, 162, 20, 164, 156, 288, 289, 12, 13, + /* 510 */ 14, 15, 16, 351, 14, 119, 120, 355, 12, 13, + /* 520 */ 20, 243, 167, 168, 169, 0, 20, 155, 22, 157, + /* 530 */ 181, 182, 250, 184, 185, 186, 187, 188, 189, 190, + /* 540 */ 191, 192, 193, 194, 195, 196, 197, 198, 264, 271, + /* 550 */ 293, 267, 250, 47, 243, 0, 250, 279, 243, 14, + /* 560 */ 18, 279, 20, 47, 262, 20, 272, 289, 262, 27, + /* 570 */ 64, 293, 30, 302, 250, 269, 298, 0, 82, 298, + /* 580 */ 64, 279, 57, 211, 243, 279, 262, 81, 310, 151, + /* 590 */ 48, 313, 314, 315, 316, 317, 318, 272, 320, 317, + /* 600 */ 45, 323, 272, 279, 293, 327, 328, 272, 293, 103, + /* 610 */ 172, 173, 250, 331, 332, 333, 334, 339, 336, 113, + /* 620 */ 339, 42, 43, 271, 262, 244, 1, 2, 145, 351, + /* 630 */ 278, 269, 351, 355, 293, 283, 355, 60, 61, 62, + /* 640 */ 63, 279, 65, 66, 67, 68, 69, 70, 71, 72, + /* 650 */ 73, 74, 75, 76, 77, 78, 14, 15, 16, 243, + /* 660 */ 118, 4, 156, 121, 122, 123, 124, 125, 126, 127, /* 670 */ 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, - /* 680 */ 138, 243, 199, 243, 58, 0, 272, 181, 182, 20, + /* 680 */ 138, 226, 243, 289, 243, 185, 20, 181, 182, 295, /* 690 */ 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, - /* 700 */ 194, 195, 196, 197, 198, 250, 18, 250, 250, 293, - /* 710 */ 259, 23, 261, 250, 271, 57, 90, 262, 148, 262, - /* 720 */ 250, 185, 243, 35, 36, 20, 283, 39, 243, 243, - /* 730 */ 21, 293, 262, 293, 279, 37, 279, 279, 243, 269, - /* 740 */ 288, 289, 279, 34, 56, 60, 61, 62, 63, 279, - /* 750 */ 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, - /* 760 */ 75, 76, 77, 78, 250, 139, 271, 243, 272, 81, - /* 770 */ 272, 250, 293, 250, 279, 317, 262, 250, 293, 293, - /* 780 */ 317, 42, 43, 262, 289, 262, 160, 41, 293, 262, - /* 790 */ 332, 333, 334, 279, 336, 332, 333, 334, 243, 336, - /* 800 */ 279, 231, 279, 272, 309, 117, 279, 243, 313, 314, - /* 810 */ 315, 316, 317, 318, 145, 320, 41, 293, 323, 243, - /* 820 */ 243, 58, 327, 328, 329, 199, 200, 201, 202, 203, - /* 830 */ 204, 205, 206, 207, 208, 271, 148, 149, 150, 209, - /* 840 */ 210, 153, 347, 279, 243, 243, 158, 85, 293, 85, - /* 850 */ 88, 243, 88, 289, 0, 212, 272, 293, 85, 171, - /* 860 */ 250, 88, 174, 47, 176, 177, 178, 179, 180, 293, - /* 870 */ 293, 250, 262, 309, 0, 243, 22, 313, 314, 315, - /* 880 */ 316, 317, 318, 262, 320, 250, 250, 323, 0, 279, - /* 890 */ 185, 327, 328, 329, 293, 293, 22, 262, 262, 211, - /* 900 */ 279, 293, 338, 271, 1, 2, 85, 243, 210, 88, - /* 910 */ 22, 279, 301, 41, 279, 279, 47, 41, 196, 197, - /* 920 */ 244, 289, 41, 41, 358, 293, 41, 81, 41, 113, - /* 930 */ 41, 41, 260, 64, 41, 271, 250, 91, 349, 305, - /* 940 */ 243, 309, 343, 279, 181, 313, 314, 315, 316, 317, - /* 950 */ 318, 271, 320, 289, 82, 323, 41, 293, 82, 327, - /* 960 */ 328, 329, 249, 82, 82, 279, 251, 82, 271, 82, - /* 970 */ 338, 82, 82, 309, 228, 82, 279, 313, 314, 315, - /* 980 */ 316, 317, 318, 297, 320, 41, 289, 323, 243, 41, - /* 990 */ 293, 327, 328, 329, 297, 312, 282, 82, 41, 41, - /* 1000 */ 337, 352, 338, 317, 352, 230, 309, 352, 340, 41, - /* 1010 */ 313, 314, 315, 316, 317, 318, 271, 320, 332, 333, - /* 1020 */ 334, 213, 336, 307, 279, 339, 82, 20, 250, 45, - /* 1030 */ 82, 306, 47, 257, 289, 299, 339, 351, 293, 82, - /* 1040 */ 82, 355, 297, 250, 250, 154, 40, 287, 351, 139, - /* 1050 */ 82, 285, 355, 285, 309, 250, 20, 245, 313, 314, - /* 1060 */ 315, 316, 317, 318, 245, 320, 243, 20, 303, 255, - /* 1070 */ 289, 20, 255, 94, 95, 96, 97, 98, 99, 100, - /* 1080 */ 101, 102, 103, 104, 339, 106, 107, 108, 109, 110, - /* 1090 */ 111, 296, 20, 255, 271, 298, 351, 296, 255, 279, - /* 1100 */ 355, 20, 279, 290, 255, 255, 243, 250, 255, 245, - /* 1110 */ 271, 271, 289, 271, 64, 271, 293, 250, 271, 271, - /* 1120 */ 245, 293, 271, 271, 293, 303, 271, 271, 163, 271, - /* 1130 */ 253, 302, 309, 253, 271, 290, 313, 314, 315, 316, - /* 1140 */ 317, 318, 279, 320, 250, 253, 323, 279, 289, 253, - /* 1150 */ 327, 328, 289, 218, 294, 293, 293, 296, 142, 294, - /* 1160 */ 253, 253, 291, 293, 267, 243, 20, 294, 290, 219, - /* 1170 */ 225, 345, 309, 348, 279, 348, 313, 314, 315, 316, - /* 1180 */ 317, 318, 147, 320, 243, 279, 323, 214, 210, 20, - /* 1190 */ 327, 328, 293, 271, 293, 312, 294, 293, 293, 279, - /* 1200 */ 40, 279, 227, 229, 232, 307, 81, 81, 275, 250, - /* 1210 */ 344, 289, 271, 261, 330, 293, 253, 304, 265, 300, - /* 1220 */ 279, 342, 311, 254, 241, 245, 265, 265, 0, 0, - /* 1230 */ 289, 309, 40, 0, 293, 313, 314, 315, 316, 317, - /* 1240 */ 318, 0, 320, 326, 72, 359, 47, 47, 175, 47, - /* 1250 */ 309, 19, 47, 353, 313, 314, 315, 316, 317, 318, - /* 1260 */ 243, 320, 175, 354, 323, 33, 354, 353, 0, 328, - /* 1270 */ 12, 13, 354, 47, 353, 175, 0, 45, 356, 357, - /* 1280 */ 22, 47, 175, 51, 52, 53, 54, 55, 271, 0, - /* 1290 */ 47, 0, 243, 276, 47, 0, 279, 47, 0, 81, - /* 1300 */ 160, 159, 22, 113, 156, 47, 289, 0, 0, 152, - /* 1310 */ 293, 0, 80, 151, 0, 83, 44, 0, 0, 0, - /* 1320 */ 271, 0, 64, 0, 0, 276, 309, 47, 279, 0, - /* 1330 */ 313, 314, 315, 316, 317, 318, 0, 320, 289, 4, - /* 1340 */ 0, 0, 293, 0, 64, 0, 0, 0, 116, 0, - /* 1350 */ 0, 0, 0, 40, 19, 0, 243, 0, 309, 0, - /* 1360 */ 0, 103, 313, 314, 315, 316, 317, 318, 33, 320, - /* 1370 */ 0, 113, 0, 141, 37, 22, 144, 0, 0, 0, - /* 1380 */ 45, 0, 0, 103, 271, 50, 0, 14, 40, 14, - /* 1390 */ 55, 0, 279, 113, 162, 41, 164, 0, 0, 37, - /* 1400 */ 44, 38, 289, 0, 37, 44, 293, 147, 0, 243, - /* 1410 */ 0, 0, 0, 37, 156, 80, 0, 59, 83, 0, - /* 1420 */ 37, 243, 309, 0, 0, 45, 313, 314, 315, 316, - /* 1430 */ 317, 318, 47, 320, 47, 45, 156, 271, 37, 181, - /* 1440 */ 45, 47, 37, 47, 45, 279, 37, 0, 0, 271, - /* 1450 */ 192, 193, 194, 0, 276, 289, 0, 279, 90, 293, - /* 1460 */ 47, 181, 182, 350, 22, 0, 41, 289, 47, 47, - /* 1470 */ 0, 293, 88, 47, 243, 309, 47, 41, 47, 313, - /* 1480 */ 314, 315, 316, 317, 318, 47, 320, 309, 47, 22, - /* 1490 */ 0, 313, 314, 315, 316, 317, 318, 22, 320, 48, - /* 1500 */ 0, 22, 271, 0, 47, 22, 0, 276, 22, 33, - /* 1510 */ 279, 20, 0, 47, 145, 0, 22, 0, 0, 0, - /* 1520 */ 289, 45, 243, 357, 293, 161, 0, 51, 52, 53, - /* 1530 */ 54, 55, 37, 145, 142, 81, 81, 243, 0, 81, - /* 1540 */ 309, 82, 140, 143, 313, 314, 315, 316, 317, 318, - /* 1550 */ 271, 320, 145, 81, 140, 276, 80, 37, 279, 83, - /* 1560 */ 91, 81, 41, 82, 41, 271, 44, 82, 289, 81, - /* 1570 */ 22, 82, 293, 279, 41, 82, 41, 243, 81, 81, - /* 1580 */ 215, 44, 81, 289, 44, 81, 215, 293, 309, 82, - /* 1590 */ 41, 82, 313, 314, 315, 316, 317, 318, 44, 320, - /* 1600 */ 41, 44, 44, 309, 41, 271, 82, 313, 314, 315, - /* 1610 */ 316, 317, 318, 279, 320, 209, 140, 243, 142, 215, - /* 1620 */ 144, 82, 146, 289, 47, 2, 47, 293, 47, 47, - /* 1630 */ 47, 47, 41, 181, 81, 22, 243, 82, 82, 44, - /* 1640 */ 164, 81, 81, 309, 82, 271, 81, 313, 314, 315, - /* 1650 */ 316, 317, 318, 279, 320, 81, 44, 183, 92, 22, - /* 1660 */ 93, 81, 47, 289, 271, 47, 81, 293, 82, 82, - /* 1670 */ 81, 47, 279, 47, 82, 81, 47, 47, 82, 81, - /* 1680 */ 105, 82, 289, 309, 81, 81, 293, 313, 314, 315, - /* 1690 */ 316, 317, 318, 243, 320, 105, 47, 81, 81, 22, - /* 1700 */ 59, 113, 309, 58, 47, 105, 313, 314, 315, 316, - /* 1710 */ 317, 318, 105, 320, 64, 79, 41, 47, 47, 47, - /* 1720 */ 22, 271, 64, 47, 0, 243, 47, 47, 47, 279, - /* 1730 */ 47, 47, 47, 47, 47, 0, 47, 47, 47, 289, - /* 1740 */ 45, 37, 37, 293, 47, 45, 0, 45, 47, 37, - /* 1750 */ 0, 45, 37, 271, 47, 0, 47, 46, 0, 309, - /* 1760 */ 0, 279, 21, 313, 314, 315, 316, 317, 318, 22, - /* 1770 */ 320, 289, 21, 243, 22, 293, 22, 20, 360, 360, - /* 1780 */ 360, 360, 360, 360, 360, 360, 360, 360, 243, 360, - /* 1790 */ 360, 309, 360, 360, 360, 313, 314, 315, 316, 317, - /* 1800 */ 318, 271, 320, 360, 360, 360, 360, 360, 360, 279, - /* 1810 */ 360, 360, 360, 360, 360, 360, 271, 360, 360, 289, - /* 1820 */ 360, 243, 360, 293, 279, 360, 360, 360, 360, 360, - /* 1830 */ 360, 360, 360, 360, 289, 360, 243, 360, 293, 309, - /* 1840 */ 360, 360, 360, 313, 314, 315, 316, 317, 318, 271, - /* 1850 */ 320, 360, 360, 243, 309, 360, 360, 279, 313, 314, - /* 1860 */ 315, 316, 317, 318, 271, 320, 360, 289, 360, 360, - /* 1870 */ 360, 293, 279, 360, 360, 360, 360, 360, 360, 360, - /* 1880 */ 360, 271, 289, 360, 360, 360, 293, 309, 360, 279, - /* 1890 */ 360, 313, 314, 315, 316, 317, 318, 360, 320, 289, - /* 1900 */ 360, 360, 309, 293, 360, 360, 313, 314, 315, 316, - /* 1910 */ 317, 318, 360, 320, 360, 360, 243, 360, 360, 309, - /* 1920 */ 360, 360, 360, 313, 314, 315, 316, 317, 318, 360, - /* 1930 */ 320, 243, 360, 360, 360, 360, 360, 360, 360, 360, - /* 1940 */ 360, 360, 360, 360, 271, 360, 360, 360, 360, 360, - /* 1950 */ 360, 360, 279, 360, 360, 360, 360, 360, 360, 271, - /* 1960 */ 360, 360, 289, 360, 360, 360, 293, 279, 360, 360, - /* 1970 */ 360, 360, 360, 360, 360, 360, 360, 289, 360, 360, - /* 1980 */ 360, 293, 309, 243, 360, 360, 313, 314, 315, 316, - /* 1990 */ 317, 318, 360, 320, 360, 360, 360, 309, 360, 360, - /* 2000 */ 360, 313, 314, 315, 316, 317, 318, 360, 320, 360, - /* 2010 */ 360, 271, 360, 360, 360, 243, 360, 360, 360, 279, - /* 2020 */ 360, 360, 360, 360, 360, 360, 360, 360, 360, 289, - /* 2030 */ 360, 360, 360, 293, 360, 360, 360, 360, 360, 360, - /* 2040 */ 360, 360, 360, 271, 360, 360, 360, 360, 360, 309, - /* 2050 */ 360, 279, 360, 313, 314, 315, 316, 317, 318, 360, - /* 2060 */ 320, 289, 360, 243, 360, 293, 360, 360, 360, 360, - /* 2070 */ 360, 360, 250, 360, 360, 360, 360, 360, 360, 360, - /* 2080 */ 360, 309, 360, 360, 360, 313, 314, 315, 316, 317, - /* 2090 */ 318, 271, 320, 360, 360, 360, 360, 360, 360, 279, - /* 2100 */ 360, 279, 360, 360, 360, 360, 360, 360, 360, 289, - /* 2110 */ 360, 360, 360, 293, 360, 360, 360, 360, 360, 297, - /* 2120 */ 360, 360, 360, 360, 360, 360, 360, 360, 360, 309, - /* 2130 */ 360, 360, 360, 313, 314, 315, 316, 317, 318, 317, - /* 2140 */ 320, 360, 360, 360, 360, 360, 360, 360, 360, 360, - /* 2150 */ 360, 360, 360, 360, 332, 333, 334, 360, 336, 360, - /* 2160 */ 360, 339, 360, 360, 360, 360, 360, 360, 360, 360, - /* 2170 */ 360, 360, 360, 351, 360, 360, 360, 355, + /* 700 */ 194, 195, 196, 197, 198, 139, 18, 82, 243, 293, + /* 710 */ 271, 23, 259, 250, 261, 37, 272, 250, 279, 0, + /* 720 */ 20, 250, 298, 35, 36, 262, 160, 39, 289, 262, + /* 730 */ 185, 250, 293, 262, 293, 250, 12, 13, 14, 15, + /* 740 */ 16, 243, 279, 262, 56, 243, 279, 262, 243, 310, + /* 750 */ 279, 41, 313, 314, 315, 316, 317, 318, 293, 320, + /* 760 */ 279, 243, 323, 339, 279, 199, 327, 328, 329, 81, + /* 770 */ 12, 13, 14, 15, 16, 351, 271, 0, 250, 355, + /* 780 */ 341, 250, 58, 271, 279, 346, 347, 93, 250, 243, + /* 790 */ 262, 293, 280, 262, 289, 293, 85, 243, 293, 88, + /* 800 */ 262, 296, 243, 243, 243, 117, 82, 279, 243, 115, + /* 810 */ 279, 293, 93, 41, 90, 310, 58, 279, 313, 314, + /* 820 */ 315, 316, 317, 318, 58, 320, 209, 210, 250, 272, + /* 830 */ 21, 112, 271, 114, 115, 116, 148, 149, 150, 293, + /* 840 */ 279, 153, 260, 34, 271, 145, 158, 293, 90, 41, + /* 850 */ 289, 185, 293, 293, 293, 243, 283, 279, 293, 171, + /* 860 */ 22, 358, 174, 139, 176, 177, 178, 179, 180, 212, + /* 870 */ 93, 310, 0, 47, 313, 314, 315, 316, 317, 318, + /* 880 */ 319, 320, 321, 322, 160, 47, 243, 0, 210, 112, + /* 890 */ 82, 114, 115, 116, 22, 317, 41, 139, 85, 211, + /* 900 */ 85, 88, 64, 88, 85, 293, 243, 88, 41, 22, + /* 910 */ 332, 333, 334, 0, 336, 1, 2, 44, 160, 196, + /* 920 */ 197, 349, 47, 199, 200, 201, 202, 203, 204, 205, + /* 930 */ 206, 207, 208, 41, 271, 22, 293, 82, 228, 113, + /* 940 */ 81, 103, 279, 41, 41, 306, 41, 181, 41, 82, + /* 950 */ 91, 113, 289, 343, 81, 271, 293, 199, 200, 201, + /* 960 */ 202, 203, 204, 205, 206, 207, 208, 41, 243, 249, + /* 970 */ 41, 282, 41, 310, 82, 41, 313, 314, 315, 316, + /* 980 */ 317, 318, 47, 320, 82, 82, 323, 82, 113, 82, + /* 990 */ 327, 328, 329, 41, 156, 251, 271, 312, 337, 64, + /* 1000 */ 243, 352, 230, 41, 279, 352, 41, 352, 82, 340, + /* 1010 */ 347, 82, 213, 82, 289, 41, 82, 41, 293, 181, + /* 1020 */ 182, 308, 20, 250, 45, 307, 257, 47, 271, 300, + /* 1030 */ 154, 250, 250, 287, 82, 310, 279, 40, 313, 314, + /* 1040 */ 315, 316, 317, 318, 82, 320, 289, 82, 323, 139, + /* 1050 */ 293, 250, 327, 328, 329, 285, 82, 20, 82, 285, + /* 1060 */ 243, 245, 20, 338, 304, 245, 255, 310, 289, 255, + /* 1070 */ 313, 314, 315, 316, 317, 318, 20, 320, 20, 299, + /* 1080 */ 323, 297, 255, 297, 327, 328, 329, 255, 271, 279, + /* 1090 */ 20, 255, 243, 290, 255, 338, 279, 250, 255, 245, + /* 1100 */ 271, 271, 271, 250, 271, 271, 289, 245, 271, 64, + /* 1110 */ 293, 271, 271, 271, 293, 271, 271, 293, 253, 163, + /* 1120 */ 271, 253, 304, 253, 289, 303, 20, 310, 279, 250, + /* 1130 */ 313, 314, 315, 316, 317, 318, 279, 320, 289, 253, + /* 1140 */ 323, 219, 293, 243, 327, 328, 329, 298, 297, 293, + /* 1150 */ 290, 218, 348, 348, 293, 338, 147, 294, 312, 310, + /* 1160 */ 294, 225, 313, 314, 315, 316, 317, 318, 345, 320, + /* 1170 */ 293, 271, 344, 243, 214, 279, 210, 308, 20, 279, + /* 1180 */ 40, 232, 311, 229, 227, 81, 291, 294, 339, 289, + /* 1190 */ 342, 293, 293, 293, 330, 294, 142, 293, 298, 279, + /* 1200 */ 351, 271, 290, 353, 355, 267, 243, 279, 326, 279, + /* 1210 */ 310, 359, 354, 313, 314, 315, 316, 317, 318, 289, + /* 1220 */ 320, 253, 253, 293, 81, 275, 253, 250, 354, 245, + /* 1230 */ 305, 353, 261, 353, 271, 354, 243, 265, 301, 339, + /* 1240 */ 310, 254, 279, 313, 314, 315, 316, 317, 318, 241, + /* 1250 */ 320, 351, 289, 323, 265, 355, 293, 327, 328, 265, + /* 1260 */ 0, 0, 40, 0, 271, 72, 0, 47, 175, 175, + /* 1270 */ 47, 47, 279, 310, 47, 0, 313, 314, 315, 316, + /* 1280 */ 317, 318, 289, 320, 47, 47, 293, 243, 175, 0, + /* 1290 */ 175, 0, 47, 0, 47, 0, 243, 47, 0, 81, + /* 1300 */ 113, 160, 156, 310, 159, 0, 313, 314, 315, 316, + /* 1310 */ 317, 318, 0, 320, 152, 271, 323, 151, 0, 356, + /* 1320 */ 357, 328, 0, 279, 271, 44, 0, 0, 0, 0, + /* 1330 */ 0, 0, 279, 289, 0, 0, 0, 293, 0, 0, + /* 1340 */ 0, 0, 289, 0, 0, 0, 293, 0, 0, 40, + /* 1350 */ 243, 0, 0, 0, 310, 0, 0, 313, 314, 315, + /* 1360 */ 316, 317, 318, 310, 320, 0, 313, 314, 315, 316, + /* 1370 */ 317, 318, 22, 320, 0, 0, 0, 0, 271, 0, + /* 1380 */ 0, 14, 40, 37, 14, 0, 279, 0, 0, 0, + /* 1390 */ 243, 147, 0, 44, 350, 41, 289, 38, 37, 37, + /* 1400 */ 293, 44, 0, 296, 90, 37, 243, 0, 0, 0, + /* 1410 */ 357, 0, 0, 47, 45, 37, 45, 310, 271, 59, + /* 1420 */ 313, 314, 315, 316, 317, 318, 279, 320, 47, 47, + /* 1430 */ 37, 45, 37, 0, 271, 37, 289, 47, 0, 45, + /* 1440 */ 293, 0, 279, 0, 0, 22, 88, 47, 0, 0, + /* 1450 */ 47, 41, 289, 47, 243, 41, 293, 310, 22, 296, + /* 1460 */ 313, 314, 315, 316, 317, 318, 0, 320, 47, 322, + /* 1470 */ 48, 47, 47, 310, 47, 47, 313, 314, 315, 316, + /* 1480 */ 317, 318, 271, 320, 22, 0, 243, 22, 0, 47, + /* 1490 */ 279, 33, 0, 22, 22, 20, 0, 47, 0, 22, + /* 1500 */ 289, 0, 0, 45, 293, 0, 0, 296, 145, 51, + /* 1510 */ 52, 53, 54, 55, 271, 81, 243, 37, 145, 41, + /* 1520 */ 41, 310, 279, 140, 313, 314, 315, 316, 317, 318, + /* 1530 */ 215, 320, 289, 161, 82, 41, 293, 82, 80, 142, + /* 1540 */ 81, 83, 81, 44, 271, 81, 145, 44, 41, 82, + /* 1550 */ 81, 41, 279, 310, 41, 82, 313, 314, 315, 316, + /* 1560 */ 317, 318, 289, 320, 44, 82, 293, 243, 44, 82, + /* 1570 */ 41, 82, 47, 47, 47, 47, 2, 47, 47, 41, + /* 1580 */ 181, 44, 82, 310, 44, 81, 313, 314, 315, 316, + /* 1590 */ 317, 318, 81, 320, 82, 271, 81, 22, 140, 215, + /* 1600 */ 142, 183, 144, 279, 146, 209, 215, 243, 0, 37, + /* 1610 */ 91, 143, 44, 289, 82, 81, 81, 293, 82, 22, + /* 1620 */ 81, 140, 164, 81, 81, 44, 81, 47, 82, 81, + /* 1630 */ 81, 113, 82, 92, 310, 271, 47, 313, 314, 315, + /* 1640 */ 316, 317, 318, 279, 320, 81, 47, 243, 82, 82, + /* 1650 */ 81, 47, 81, 289, 47, 82, 47, 293, 81, 105, + /* 1660 */ 82, 105, 243, 81, 105, 105, 81, 22, 81, 93, + /* 1670 */ 47, 81, 22, 58, 310, 271, 59, 313, 314, 315, + /* 1680 */ 316, 317, 318, 279, 320, 47, 64, 41, 47, 79, + /* 1690 */ 271, 22, 64, 289, 47, 47, 47, 293, 279, 47, + /* 1700 */ 47, 47, 47, 47, 47, 47, 47, 47, 289, 47, + /* 1710 */ 0, 47, 293, 243, 310, 45, 37, 313, 314, 315, + /* 1720 */ 316, 317, 318, 0, 320, 243, 45, 47, 37, 310, + /* 1730 */ 0, 47, 313, 314, 315, 316, 317, 318, 45, 320, + /* 1740 */ 37, 271, 47, 45, 37, 0, 47, 46, 0, 279, + /* 1750 */ 0, 0, 21, 271, 22, 22, 22, 21, 20, 289, + /* 1760 */ 360, 279, 360, 293, 360, 243, 360, 360, 360, 360, + /* 1770 */ 360, 289, 360, 360, 360, 293, 243, 360, 360, 360, + /* 1780 */ 310, 360, 360, 313, 314, 315, 316, 317, 318, 360, + /* 1790 */ 320, 360, 310, 271, 360, 313, 314, 315, 316, 317, + /* 1800 */ 318, 279, 320, 360, 271, 360, 360, 360, 360, 360, + /* 1810 */ 360, 289, 279, 360, 360, 293, 360, 360, 360, 360, + /* 1820 */ 360, 360, 289, 360, 360, 360, 293, 243, 360, 360, + /* 1830 */ 360, 360, 310, 360, 360, 313, 314, 315, 316, 317, + /* 1840 */ 318, 360, 320, 310, 360, 360, 313, 314, 315, 316, + /* 1850 */ 317, 318, 360, 320, 360, 271, 360, 243, 360, 360, + /* 1860 */ 360, 360, 360, 279, 360, 360, 360, 360, 360, 360, + /* 1870 */ 360, 360, 360, 289, 360, 360, 360, 293, 360, 360, + /* 1880 */ 360, 360, 360, 360, 360, 271, 360, 243, 360, 360, + /* 1890 */ 360, 360, 360, 279, 310, 360, 360, 313, 314, 315, + /* 1900 */ 316, 317, 318, 289, 320, 360, 360, 293, 360, 360, + /* 1910 */ 360, 360, 360, 360, 360, 271, 360, 360, 360, 243, + /* 1920 */ 360, 360, 360, 279, 310, 360, 360, 313, 314, 315, + /* 1930 */ 316, 317, 318, 289, 320, 360, 360, 293, 360, 360, + /* 1940 */ 360, 360, 360, 12, 13, 360, 360, 271, 360, 360, + /* 1950 */ 360, 360, 243, 22, 310, 279, 360, 313, 314, 315, + /* 1960 */ 316, 317, 318, 360, 320, 289, 360, 360, 360, 293, + /* 1970 */ 360, 360, 360, 360, 360, 360, 360, 360, 47, 360, + /* 1980 */ 271, 360, 360, 360, 360, 243, 310, 360, 279, 313, + /* 1990 */ 314, 315, 316, 317, 318, 64, 320, 360, 289, 360, + /* 2000 */ 360, 360, 293, 360, 360, 360, 360, 360, 360, 360, + /* 2010 */ 360, 360, 360, 271, 360, 360, 250, 360, 360, 310, + /* 2020 */ 360, 279, 313, 314, 315, 316, 317, 318, 360, 320, + /* 2030 */ 360, 289, 360, 360, 103, 293, 360, 360, 360, 360, + /* 2040 */ 360, 360, 360, 360, 113, 279, 250, 360, 360, 360, + /* 2050 */ 360, 360, 310, 360, 360, 313, 314, 315, 316, 317, + /* 2060 */ 318, 360, 320, 360, 298, 360, 360, 360, 360, 360, + /* 2070 */ 360, 360, 360, 360, 360, 279, 360, 360, 360, 360, + /* 2080 */ 360, 360, 360, 317, 360, 360, 360, 156, 360, 360, + /* 2090 */ 360, 360, 360, 360, 298, 360, 360, 360, 332, 333, + /* 2100 */ 334, 360, 336, 360, 360, 339, 360, 360, 360, 360, + /* 2110 */ 360, 360, 181, 317, 360, 360, 360, 351, 360, 360, + /* 2120 */ 360, 355, 360, 192, 193, 194, 360, 360, 332, 333, + /* 2130 */ 334, 360, 336, 360, 360, 339, 360, 360, 360, 360, + /* 2140 */ 360, 360, 360, 360, 360, 360, 360, 351, 360, 360, + /* 2150 */ 360, 355, 360, 360, 360, 360, 360, 360, 360, 360, + /* 2160 */ 360, 360, 360, 360, 360, 360, 360, 360, 360, 360, + /* 2170 */ 360, 360, 360, 360, 360, 360, 360, 360, 360, 360, + /* 2180 */ 360, 360, 360, 360, }; #define YY_SHIFT_COUNT (618) #define YY_SHIFT_MIN (0) -#define YY_SHIFT_MAX (1760) +#define YY_SHIFT_MAX (1931) static const unsigned short int yy_shift_ofst[] = { /* 0 */ 688, 0, 0, 48, 96, 96, 96, 96, 253, 253, /* 10 */ 96, 96, 301, 349, 506, 349, 349, 349, 349, 349, /* 20 */ 349, 349, 349, 349, 349, 349, 349, 349, 349, 349, - /* 30 */ 349, 349, 349, 349, 349, 349, 349, 349, 349, 243, - /* 40 */ 243, 5, 5, 5, 1258, 1258, 1258, 310, 97, 169, - /* 50 */ 85, 85, 51, 51, 244, 169, 169, 85, 85, 85, - /* 60 */ 85, 85, 85, 85, 78, 85, 85, 159, 319, 159, - /* 70 */ 85, 85, 159, 337, 85, 159, 159, 319, 159, 85, - /* 80 */ 207, 542, 14, 626, 626, 230, 237, 1280, 1280, 1280, - /* 90 */ 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, 1280, - /* 100 */ 1280, 1280, 1280, 1280, 1280, 1280, 385, 227, 298, 298, - /* 110 */ 561, 326, 658, 482, 482, 482, 326, 538, 319, 159, - /* 120 */ 159, 319, 325, 590, 979, 979, 979, 979, 979, 979, - /* 130 */ 979, 1232, 397, 333, 570, 413, 42, 290, 135, 536, - /* 140 */ 739, 574, 186, 669, 630, 698, 630, 137, 137, 137, - /* 150 */ 643, 705, 808, 1007, 984, 985, 891, 1007, 1007, 1006, - /* 160 */ 910, 910, 1007, 1036, 1036, 1047, 78, 319, 78, 1051, - /* 170 */ 1072, 78, 1051, 78, 538, 1081, 78, 78, 1007, 78, - /* 180 */ 1036, 159, 159, 159, 159, 159, 159, 159, 159, 159, - /* 190 */ 159, 159, 1007, 1036, 1050, 1050, 1047, 207, 965, 319, - /* 200 */ 207, 1007, 1051, 207, 538, 1081, 207, 935, 1050, 935, - /* 210 */ 1050, 1016, 538, 1081, 207, 325, 207, 538, 1146, 950, - /* 220 */ 935, 1050, 1050, 950, 935, 1050, 1050, 159, 945, 1035, - /* 230 */ 973, 808, 978, 538, 1169, 1160, 974, 975, 972, 974, - /* 240 */ 975, 974, 975, 1125, 1126, 590, 1007, 207, 1036, 2178, - /* 250 */ 2178, 2178, 2178, 2178, 2178, 2178, 685, 1476, 484, 1335, - /* 260 */ 32, 2, 314, 329, 28, 255, 408, 212, 212, 212, - /* 270 */ 212, 212, 212, 212, 212, 271, 123, 525, 82, 312, - /* 280 */ 9, 483, 19, 19, 19, 19, 503, 510, 762, 764, - /* 290 */ 773, 821, 854, 874, 888, 709, 533, 872, 876, 881, - /* 300 */ 882, 885, 887, 412, 816, 889, 903, 722, 746, 775, - /* 310 */ 890, 763, 893, 355, 915, 944, 948, 957, 958, 968, - /* 320 */ 846, 869, 1228, 1229, 1192, 1233, 1172, 1241, 1199, 1073, - /* 330 */ 1200, 1202, 1205, 1087, 1268, 1226, 1234, 1100, 1276, 1107, - /* 340 */ 1289, 1243, 1291, 1247, 1295, 1250, 1298, 1218, 1140, 1142, - /* 350 */ 1190, 1148, 1307, 1308, 1157, 1162, 1311, 1314, 1272, 1317, - /* 360 */ 1318, 1319, 1321, 1323, 1324, 1329, 1336, 1340, 1341, 1343, - /* 370 */ 1345, 1346, 1347, 1349, 1350, 1351, 1352, 1313, 1355, 1357, - /* 380 */ 1359, 1360, 1370, 1372, 1353, 1377, 1378, 1379, 1381, 1382, - /* 390 */ 1386, 1348, 1337, 1354, 1373, 1356, 1375, 1361, 1391, 1363, - /* 400 */ 1362, 1397, 1398, 1403, 1367, 1260, 1408, 1410, 1376, 1411, - /* 410 */ 1358, 1412, 1416, 1385, 1380, 1383, 1419, 1387, 1390, 1401, - /* 420 */ 1423, 1394, 1395, 1405, 1424, 1396, 1399, 1409, 1447, 1448, - /* 430 */ 1453, 1456, 1368, 1384, 1413, 1442, 1465, 1421, 1422, 1426, - /* 440 */ 1429, 1425, 1436, 1431, 1438, 1441, 1470, 1467, 1490, 1475, - /* 450 */ 1451, 1500, 1479, 1457, 1503, 1483, 1506, 1486, 1491, 1512, - /* 460 */ 1369, 1466, 1515, 1364, 1494, 1388, 1392, 1517, 1518, 1519, - /* 470 */ 1407, 1526, 1454, 1495, 1402, 1455, 1459, 1458, 1400, 1472, - /* 480 */ 1538, 1520, 1414, 1480, 1469, 1522, 1521, 1523, 1365, 1481, - /* 490 */ 1485, 1488, 1548, 1533, 1489, 1497, 1498, 1501, 1493, 1535, - /* 500 */ 1537, 1540, 1504, 1549, 1371, 1507, 1509, 1554, 1406, 1559, - /* 510 */ 1557, 1558, 1524, 1563, 1404, 1539, 1577, 1579, 1581, 1582, - /* 520 */ 1583, 1584, 1539, 1623, 1452, 1591, 1553, 1555, 1560, 1556, - /* 530 */ 1561, 1562, 1595, 1565, 1574, 1612, 1613, 1474, 1580, 1566, - /* 540 */ 1586, 1615, 1618, 1585, 1587, 1624, 1589, 1592, 1626, 1594, - /* 550 */ 1596, 1629, 1598, 1599, 1630, 1603, 1575, 1590, 1600, 1607, - /* 560 */ 1637, 1567, 1604, 1616, 1649, 1617, 1588, 1677, 1641, 1645, - /* 570 */ 1657, 1650, 1636, 1675, 1670, 1671, 1672, 1676, 1679, 1698, - /* 580 */ 1680, 1681, 1658, 1425, 1683, 1436, 1684, 1685, 1686, 1687, - /* 590 */ 1689, 1690, 1724, 1691, 1695, 1704, 1735, 1697, 1700, 1705, - /* 600 */ 1746, 1701, 1702, 1712, 1750, 1707, 1706, 1715, 1755, 1709, - /* 610 */ 1711, 1758, 1760, 1747, 1741, 1752, 1754, 1751, 1757, + /* 30 */ 349, 349, 349, 349, 349, 349, 349, 349, 144, 144, + /* 40 */ 102, 102, 102, 1931, 1931, 1931, 1931, 372, 124, 214, + /* 50 */ 58, 58, 260, 260, 172, 214, 214, 58, 58, 58, + /* 60 */ 58, 58, 58, 58, 37, 58, 58, 186, 228, 259, + /* 70 */ 186, 58, 58, 186, 58, 186, 186, 259, 186, 58, + /* 80 */ 255, 542, 724, 758, 758, 189, 236, 838, 838, 838, + /* 90 */ 838, 838, 838, 838, 838, 838, 838, 838, 838, 838, + /* 100 */ 838, 838, 838, 838, 838, 838, 291, 126, 354, 354, + /* 110 */ 397, 516, 483, 483, 483, 525, 516, 313, 259, 186, + /* 120 */ 186, 259, 377, 343, 243, 243, 243, 243, 243, 243, + /* 130 */ 243, 340, 141, 405, 76, 195, 57, 215, 500, 545, + /* 140 */ 579, 179, 694, 700, 617, 678, 617, 455, 455, 455, + /* 150 */ 657, 666, 799, 1002, 979, 980, 876, 1002, 1002, 997, + /* 160 */ 910, 910, 1002, 1037, 1037, 1042, 37, 259, 37, 1056, + /* 170 */ 1058, 37, 1056, 37, 313, 1070, 37, 37, 1002, 37, + /* 180 */ 1037, 186, 186, 186, 186, 186, 186, 186, 186, 186, + /* 190 */ 186, 186, 1002, 1037, 1045, 1045, 1042, 255, 956, 259, + /* 200 */ 255, 1002, 1056, 255, 313, 1070, 255, 1106, 922, 933, + /* 210 */ 1045, 922, 933, 1045, 1045, 186, 936, 1009, 960, 799, + /* 220 */ 966, 313, 1158, 1140, 954, 957, 949, 954, 957, 954, + /* 230 */ 957, 1104, 933, 1045, 1045, 933, 1045, 1054, 313, 1070, + /* 240 */ 255, 377, 255, 313, 1143, 343, 1002, 255, 1037, 2152, + /* 250 */ 2152, 2152, 2152, 2152, 2152, 2152, 577, 1458, 231, 353, + /* 260 */ 3, 19, 316, 256, 496, 719, 777, 112, 112, 112, + /* 270 */ 112, 112, 112, 112, 112, 135, 438, 344, 396, 355, + /* 280 */ 625, 566, 642, 642, 642, 642, 555, 808, 711, 813, + /* 290 */ 815, 819, 872, 887, 913, 809, 855, 867, 892, 914, + /* 300 */ 723, 710, 772, 902, 766, 903, 873, 905, 907, 926, + /* 310 */ 929, 931, 826, 875, 934, 952, 962, 965, 974, 976, + /* 320 */ 859, 935, 1260, 1261, 1222, 1263, 1193, 1266, 1220, 1093, + /* 330 */ 1223, 1224, 1227, 1094, 1275, 1237, 1238, 1113, 1289, 1115, + /* 340 */ 1291, 1245, 1293, 1247, 1295, 1250, 1298, 1218, 1141, 1145, + /* 350 */ 1187, 1146, 1305, 1312, 1162, 1166, 1318, 1322, 1281, 1326, + /* 360 */ 1327, 1328, 1329, 1330, 1331, 1334, 1335, 1336, 1338, 1339, + /* 370 */ 1340, 1341, 1343, 1344, 1345, 1347, 1348, 1309, 1351, 1352, + /* 380 */ 1353, 1355, 1356, 1365, 1350, 1374, 1375, 1376, 1377, 1379, + /* 390 */ 1380, 1342, 1346, 1354, 1367, 1349, 1370, 1357, 1385, 1359, + /* 400 */ 1361, 1387, 1388, 1389, 1362, 1244, 1392, 1402, 1368, 1407, + /* 410 */ 1360, 1408, 1409, 1366, 1369, 1378, 1411, 1381, 1371, 1393, + /* 420 */ 1412, 1382, 1386, 1395, 1433, 1390, 1394, 1398, 1438, 1441, + /* 430 */ 1443, 1444, 1314, 1358, 1400, 1423, 1448, 1403, 1406, 1421, + /* 440 */ 1424, 1410, 1414, 1425, 1427, 1428, 1449, 1436, 1466, 1462, + /* 450 */ 1422, 1485, 1465, 1442, 1488, 1471, 1492, 1472, 1475, 1496, + /* 460 */ 1363, 1450, 1498, 1372, 1477, 1373, 1397, 1501, 1502, 1505, + /* 470 */ 1401, 1506, 1434, 1480, 1383, 1478, 1479, 1315, 1452, 1494, + /* 480 */ 1455, 1459, 1461, 1464, 1467, 1507, 1499, 1503, 1469, 1510, + /* 490 */ 1384, 1473, 1483, 1520, 1396, 1513, 1524, 1487, 1529, 1391, + /* 500 */ 1489, 1525, 1526, 1527, 1528, 1530, 1531, 1489, 1574, 1399, + /* 510 */ 1538, 1500, 1504, 1512, 1537, 1511, 1515, 1540, 1575, 1418, + /* 520 */ 1534, 1532, 1536, 1535, 1539, 1468, 1542, 1608, 1572, 1481, + /* 530 */ 1543, 1519, 1568, 1581, 1545, 1546, 1548, 1597, 1549, 1541, + /* 540 */ 1550, 1580, 1589, 1564, 1566, 1599, 1569, 1567, 1604, 1571, + /* 550 */ 1573, 1607, 1577, 1578, 1609, 1582, 1554, 1556, 1559, 1560, + /* 560 */ 1645, 1576, 1585, 1587, 1623, 1590, 1518, 1650, 1617, 1615, + /* 570 */ 1638, 1622, 1610, 1646, 1641, 1647, 1648, 1649, 1652, 1669, + /* 580 */ 1653, 1654, 1628, 1410, 1655, 1414, 1656, 1657, 1658, 1659, + /* 590 */ 1660, 1662, 1710, 1664, 1670, 1679, 1723, 1680, 1681, 1691, + /* 600 */ 1730, 1684, 1693, 1703, 1750, 1695, 1698, 1707, 1745, 1699, + /* 610 */ 1701, 1748, 1751, 1732, 1731, 1733, 1734, 1736, 1738, }; #define YY_REDUCE_COUNT (255) -#define YY_REDUCE_MIN (-332) -#define YY_REDUCE_MAX (1822) +#define YY_REDUCE_MIN (-324) +#define YY_REDUCE_MAX (1796) static const short yy_reduce_ofst[] = { - /* 0 */ -232, -240, 61, 284, 495, 564, 632, 664, 697, 745, - /* 10 */ 823, 863, -151, 922, 941, 1017, 1049, 1113, 1166, 1178, - /* 20 */ 335, 1231, 1279, 1294, 1334, 1374, 1393, 1450, 1482, 1530, - /* 30 */ 1545, 1578, 1593, 1610, 1673, 1688, 1740, 1772, 1820, 686, - /* 40 */ 1822, -200, 458, 463, -272, 256, 259, -288, -241, 220, - /* 50 */ 341, 470, -244, -122, -332, -254, -4, 57, 153, 367, - /* 60 */ 455, 514, 521, 523, -118, 527, 610, -212, -89, -141, - /* 70 */ 621, 635, 89, -225, 636, 75, 185, 172, 195, 457, - /* 80 */ -43, -109, 86, 86, 86, -142, -84, 10, 29, 56, - /* 90 */ 261, 323, 352, 416, 438, 440, 479, 485, 486, 524, - /* 100 */ 555, 576, 577, 601, 602, 608, 149, 101, 145, 223, - /* 110 */ -72, 219, -229, -190, 246, 321, 331, -197, -289, 88, - /* 120 */ 443, 452, 359, 451, -220, 389, 414, 496, 498, 531, - /* 130 */ 584, 611, 676, 672, 566, 589, 634, 599, 680, 680, - /* 140 */ 713, 715, 714, 683, 663, 663, 663, 649, 652, 655, - /* 150 */ 668, 680, 716, 778, 725, 776, 736, 793, 794, 760, - /* 160 */ 766, 768, 805, 812, 819, 765, 814, 781, 817, 795, - /* 170 */ 797, 838, 801, 843, 820, 813, 849, 850, 857, 853, - /* 180 */ 864, 839, 840, 842, 844, 847, 848, 851, 852, 855, - /* 190 */ 856, 858, 867, 875, 828, 831, 822, 877, 829, 859, - /* 200 */ 880, 894, 861, 892, 868, 845, 896, 860, 862, 865, - /* 210 */ 870, 871, 895, 878, 907, 897, 908, 906, 883, 825, - /* 220 */ 873, 899, 901, 827, 902, 904, 905, 680, 826, 866, - /* 230 */ 879, 898, 663, 920, 911, 884, 909, 900, 886, 912, - /* 240 */ 914, 918, 921, 917, 933, 952, 959, 963, 980, 919, - /* 250 */ 913, 953, 961, 962, 969, 983, + /* 0 */ 162, -241, 278, 439, 663, 725, 757, 817, 849, 900, + /* 10 */ -178, 930, 561, 963, 993, 505, 1044, 1053, 1107, 1147, + /* 20 */ 1163, 1211, 1243, 1273, 1324, 1364, 1404, 1419, 1470, 1482, + /* 30 */ 1522, 1533, 1584, 1614, 1644, 1676, 1709, 1742, 1766, 1796, + /* 40 */ 282, -213, 578, -270, -266, -264, -176, -284, 281, 424, + /* 50 */ 306, 362, -91, -46, -255, -192, 71, -226, -199, 150, + /* 60 */ 302, 324, 463, 467, -1, 471, 485, 28, 45, -190, + /* 70 */ 56, 528, 531, 197, 481, 192, 352, 100, 196, 538, + /* 80 */ -225, -242, -324, -324, -324, -64, -48, 77, 168, 257, + /* 90 */ 311, 315, 341, 416, 441, 465, 498, 502, 518, 546, + /* 100 */ 554, 559, 560, 565, 612, 643, 97, -247, 75, 140, + /* 110 */ -125, -232, -252, -204, -37, -100, 106, 142, 394, 512, + /* 120 */ 573, 218, 284, 453, -261, 294, 325, 330, 335, 444, + /* 130 */ 557, 271, 381, 582, 503, 572, 639, 610, 684, 684, + /* 140 */ 720, 744, 689, 685, 661, 661, 661, 649, 653, 655, + /* 150 */ 669, 684, 713, 773, 718, 769, 729, 781, 782, 746, + /* 160 */ 770, 774, 801, 816, 820, 760, 811, 779, 814, 784, + /* 170 */ 780, 827, 786, 832, 810, 803, 836, 839, 847, 843, + /* 180 */ 854, 829, 830, 831, 833, 834, 837, 840, 841, 842, + /* 190 */ 844, 845, 853, 862, 821, 824, 818, 865, 822, 835, + /* 200 */ 868, 879, 851, 870, 857, 860, 886, 846, 804, 863, + /* 210 */ 856, 805, 866, 861, 877, 684, 823, 828, 848, 869, + /* 220 */ 661, 896, 871, 864, 858, 850, 852, 874, 878, 881, + /* 230 */ 880, 882, 893, 898, 899, 901, 904, 895, 920, 912, + /* 240 */ 968, 938, 969, 928, 950, 971, 977, 973, 984, 937, + /* 250 */ 925, 972, 989, 994, 987, 1008, }; static const YYACTIONTYPE yy_default[] = { /* 0 */ 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, @@ -766,7 +765,7 @@ static const YYACTIONTYPE yy_default[] = { /* 80 */ 1427, 1567, 1360, 1734, 1360, 1360, 1360, 1360, 1360, 1360, /* 90 */ 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, /* 100 */ 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, - /* 110 */ 1429, 1360, 1427, 1745, 1745, 1745, 1360, 1360, 1360, 1360, + /* 110 */ 1429, 1360, 1745, 1745, 1745, 1427, 1360, 1360, 1360, 1360, /* 120 */ 1360, 1360, 1523, 1360, 1360, 1360, 1360, 1360, 1360, 1360, /* 130 */ 1360, 1603, 1360, 1360, 1811, 1360, 1609, 1769, 1360, 1360, /* 140 */ 1360, 1360, 1476, 1761, 1737, 1751, 1738, 1796, 1796, 1796, @@ -775,18 +774,18 @@ static const YYACTIONTYPE yy_default[] = { /* 170 */ 1360, 1429, 1360, 1429, 1360, 1360, 1429, 1429, 1360, 1429, /* 180 */ 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, /* 190 */ 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1427, 1605, 1360, - /* 200 */ 1427, 1360, 1360, 1427, 1360, 1360, 1427, 1774, 1360, 1774, - /* 210 */ 1360, 1580, 1360, 1360, 1427, 1360, 1427, 1360, 1360, 1776, - /* 220 */ 1774, 1360, 1360, 1776, 1774, 1360, 1360, 1360, 1788, 1784, - /* 230 */ 1767, 1765, 1751, 1360, 1360, 1360, 1802, 1798, 1814, 1802, - /* 240 */ 1798, 1802, 1798, 1360, 1492, 1360, 1360, 1427, 1360, 1597, + /* 200 */ 1427, 1360, 1360, 1427, 1360, 1360, 1427, 1360, 1776, 1774, + /* 210 */ 1360, 1776, 1774, 1360, 1360, 1360, 1788, 1784, 1767, 1765, + /* 220 */ 1751, 1360, 1360, 1360, 1802, 1798, 1814, 1802, 1798, 1802, + /* 230 */ 1798, 1360, 1774, 1360, 1360, 1774, 1360, 1580, 1360, 1360, + /* 240 */ 1427, 1360, 1427, 1360, 1492, 1360, 1360, 1427, 1360, 1597, /* 250 */ 1611, 1526, 1526, 1526, 1430, 1365, 1360, 1360, 1360, 1360, /* 260 */ 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1678, 1787, 1786, /* 270 */ 1710, 1709, 1708, 1706, 1677, 1488, 1360, 1360, 1360, 1360, /* 280 */ 1360, 1360, 1671, 1672, 1670, 1669, 1360, 1360, 1360, 1360, - /* 290 */ 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, - /* 300 */ 1360, 1360, 1360, 1360, 1360, 1360, 1735, 1360, 1799, 1803, - /* 310 */ 1360, 1360, 1360, 1654, 1360, 1360, 1360, 1360, 1360, 1360, + /* 290 */ 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1735, + /* 300 */ 1360, 1799, 1803, 1360, 1360, 1360, 1654, 1360, 1360, 1360, + /* 310 */ 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, /* 320 */ 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, /* 330 */ 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, /* 340 */ 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, @@ -802,13 +801,13 @@ static const YYACTIONTYPE yy_default[] = { /* 440 */ 1360, 1457, 1456, 1360, 1360, 1360, 1360, 1360, 1360, 1360, /* 450 */ 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, /* 460 */ 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, - /* 470 */ 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, - /* 480 */ 1360, 1360, 1360, 1360, 1360, 1360, 1758, 1768, 1360, 1360, - /* 490 */ 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, - /* 500 */ 1360, 1654, 1360, 1785, 1360, 1744, 1740, 1360, 1360, 1736, - /* 510 */ 1653, 1360, 1360, 1797, 1360, 1360, 1360, 1360, 1360, 1360, - /* 520 */ 1360, 1360, 1360, 1730, 1360, 1703, 1694, 1360, 1360, 1360, - /* 530 */ 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1665, 1360, 1360, + /* 470 */ 1360, 1360, 1360, 1360, 1360, 1758, 1768, 1360, 1360, 1360, + /* 480 */ 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1654, 1360, 1785, + /* 490 */ 1360, 1744, 1740, 1360, 1360, 1736, 1360, 1360, 1797, 1360, + /* 500 */ 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1730, 1360, + /* 510 */ 1703, 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1665, + /* 520 */ 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, + /* 530 */ 1360, 1360, 1653, 1360, 1694, 1360, 1360, 1360, 1360, 1360, /* 540 */ 1360, 1360, 1360, 1520, 1360, 1360, 1360, 1360, 1360, 1360, /* 550 */ 1360, 1360, 1360, 1360, 1360, 1360, 1505, 1503, 1502, 1501, /* 560 */ 1360, 1498, 1360, 1360, 1360, 1360, 1360, 1360, 1360, 1360, @@ -1439,7 +1438,7 @@ static const char *const yyTokenName[] = { /* 273 */ "signed_literal", /* 274 */ "create_subtable_clause", /* 275 */ "specific_tags_opt", - /* 276 */ "expression_list", + /* 276 */ "literal_list", /* 277 */ "drop_table_clause", /* 278 */ "col_name_list", /* 279 */ "table_name", @@ -1459,21 +1458,21 @@ static const char *const yyTokenName[] = { /* 293 */ "duration_literal", /* 294 */ "sliding_opt", /* 295 */ "func", - /* 296 */ "topic_name", - /* 297 */ "query_expression", - /* 298 */ "cgroup_name", - /* 299 */ "analyze_opt", - /* 300 */ "explain_options", - /* 301 */ "agg_func_opt", - /* 302 */ "bufsize_opt", - /* 303 */ "stream_name", - /* 304 */ "stream_options", - /* 305 */ "into_opt", - /* 306 */ "dnode_list", - /* 307 */ "where_clause_opt", - /* 308 */ "signed", - /* 309 */ "literal_func", - /* 310 */ "literal_list", + /* 296 */ "expression_list", + /* 297 */ "topic_name", + /* 298 */ "query_expression", + /* 299 */ "cgroup_name", + /* 300 */ "analyze_opt", + /* 301 */ "explain_options", + /* 302 */ "agg_func_opt", + /* 303 */ "bufsize_opt", + /* 304 */ "stream_name", + /* 305 */ "stream_options", + /* 306 */ "into_opt", + /* 307 */ "dnode_list", + /* 308 */ "where_clause_opt", + /* 309 */ "signed", + /* 310 */ "literal_func", /* 311 */ "table_alias", /* 312 */ "column_alias", /* 313 */ "expression", @@ -1655,7 +1654,7 @@ static const char *const yyRuleName[] = { /* 122 */ "alter_table_clause ::= full_table_name SET TAG column_name NK_EQ signed_literal", /* 123 */ "multi_create_clause ::= create_subtable_clause", /* 124 */ "multi_create_clause ::= multi_create_clause create_subtable_clause", - /* 125 */ "create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_tags_opt TAGS NK_LP expression_list NK_RP table_options", + /* 125 */ "create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_tags_opt TAGS NK_LP literal_list NK_RP table_options", /* 126 */ "multi_drop_clause ::= drop_table_clause", /* 127 */ "multi_drop_clause ::= multi_drop_clause drop_table_clause", /* 128 */ "drop_table_clause ::= exists_opt full_table_name", @@ -2133,13 +2132,13 @@ static void yy_destructor( case 293: /* duration_literal */ case 294: /* sliding_opt */ case 295: /* func */ - case 297: /* query_expression */ - case 300: /* explain_options */ - case 304: /* stream_options */ - case 305: /* into_opt */ - case 307: /* where_clause_opt */ - case 308: /* signed */ - case 309: /* literal_func */ + case 298: /* query_expression */ + case 301: /* explain_options */ + case 305: /* stream_options */ + case 306: /* into_opt */ + case 308: /* where_clause_opt */ + case 309: /* signed */ + case 310: /* literal_func */ case 313: /* expression */ case 314: /* pseudo_column */ case 315: /* column_reference */ @@ -2175,7 +2174,7 @@ static void yy_destructor( case 241: /* account_options */ case 242: /* alter_account_options */ case 244: /* alter_account_option */ - case 302: /* bufsize_opt */ + case 303: /* bufsize_opt */ { } @@ -2189,9 +2188,9 @@ static void yy_destructor( case 279: /* table_name */ case 289: /* function_name */ case 290: /* index_name */ - case 296: /* topic_name */ - case 298: /* cgroup_name */ - case 303: /* stream_name */ + case 297: /* topic_name */ + case 299: /* cgroup_name */ + case 304: /* stream_name */ case 311: /* table_alias */ case 312: /* column_alias */ case 318: /* star_func */ @@ -2210,8 +2209,8 @@ static void yy_destructor( break; case 253: /* not_exists_opt */ case 255: /* exists_opt */ - case 299: /* analyze_opt */ - case 301: /* agg_func_opt */ + case 300: /* analyze_opt */ + case 302: /* agg_func_opt */ case 340: /* set_quantifier_opt */ { @@ -2226,12 +2225,12 @@ static void yy_destructor( case 267: /* tags_def */ case 268: /* multi_drop_clause */ case 275: /* specific_tags_opt */ - case 276: /* expression_list */ + case 276: /* literal_list */ case 278: /* col_name_list */ case 281: /* func_name_list */ case 292: /* func_list */ - case 306: /* dnode_list */ - case 310: /* literal_list */ + case 296: /* expression_list */ + case 307: /* dnode_list */ case 319: /* star_func_para_list */ case 321: /* other_para_list */ case 341: /* select_list */ @@ -2701,7 +2700,7 @@ static const struct { { 269, -6 }, /* (122) alter_table_clause ::= full_table_name SET TAG column_name NK_EQ signed_literal */ { 266, -1 }, /* (123) multi_create_clause ::= create_subtable_clause */ { 266, -2 }, /* (124) multi_create_clause ::= multi_create_clause create_subtable_clause */ - { 274, -10 }, /* (125) create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_tags_opt TAGS NK_LP expression_list NK_RP table_options */ + { 274, -10 }, /* (125) create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_tags_opt TAGS NK_LP literal_list NK_RP table_options */ { 268, -1 }, /* (126) multi_drop_clause ::= drop_table_clause */ { 268, -2 }, /* (127) multi_drop_clause ::= multi_drop_clause drop_table_clause */ { 277, -2 }, /* (128) drop_table_clause ::= exists_opt full_table_name */ @@ -2808,27 +2807,27 @@ static const struct { { 240, -2 }, /* (229) cmd ::= DESCRIBE full_table_name */ { 240, -3 }, /* (230) cmd ::= RESET QUERY CACHE */ { 240, -4 }, /* (231) cmd ::= EXPLAIN analyze_opt explain_options query_expression */ - { 299, 0 }, /* (232) analyze_opt ::= */ - { 299, -1 }, /* (233) analyze_opt ::= ANALYZE */ - { 300, 0 }, /* (234) explain_options ::= */ - { 300, -3 }, /* (235) explain_options ::= explain_options VERBOSE NK_BOOL */ - { 300, -3 }, /* (236) explain_options ::= explain_options RATIO NK_FLOAT */ + { 300, 0 }, /* (232) analyze_opt ::= */ + { 300, -1 }, /* (233) analyze_opt ::= ANALYZE */ + { 301, 0 }, /* (234) explain_options ::= */ + { 301, -3 }, /* (235) explain_options ::= explain_options VERBOSE NK_BOOL */ + { 301, -3 }, /* (236) explain_options ::= explain_options RATIO NK_FLOAT */ { 240, -6 }, /* (237) cmd ::= COMPACT VNODES IN NK_LP integer_list NK_RP */ { 240, -10 }, /* (238) cmd ::= CREATE agg_func_opt FUNCTION not_exists_opt function_name AS NK_STRING OUTPUTTYPE type_name bufsize_opt */ { 240, -4 }, /* (239) cmd ::= DROP FUNCTION exists_opt function_name */ - { 301, 0 }, /* (240) agg_func_opt ::= */ - { 301, -1 }, /* (241) agg_func_opt ::= AGGREGATE */ - { 302, 0 }, /* (242) bufsize_opt ::= */ - { 302, -2 }, /* (243) bufsize_opt ::= BUFSIZE NK_INTEGER */ + { 302, 0 }, /* (240) agg_func_opt ::= */ + { 302, -1 }, /* (241) agg_func_opt ::= AGGREGATE */ + { 303, 0 }, /* (242) bufsize_opt ::= */ + { 303, -2 }, /* (243) bufsize_opt ::= BUFSIZE NK_INTEGER */ { 240, -8 }, /* (244) cmd ::= CREATE STREAM not_exists_opt stream_name stream_options into_opt AS query_expression */ { 240, -4 }, /* (245) cmd ::= DROP STREAM exists_opt stream_name */ - { 305, 0 }, /* (246) into_opt ::= */ - { 305, -2 }, /* (247) into_opt ::= INTO full_table_name */ - { 304, 0 }, /* (248) stream_options ::= */ - { 304, -3 }, /* (249) stream_options ::= stream_options TRIGGER AT_ONCE */ - { 304, -3 }, /* (250) stream_options ::= stream_options TRIGGER WINDOW_CLOSE */ - { 304, -4 }, /* (251) stream_options ::= stream_options TRIGGER MAX_DELAY duration_literal */ - { 304, -3 }, /* (252) stream_options ::= stream_options WATERMARK duration_literal */ + { 306, 0 }, /* (246) into_opt ::= */ + { 306, -2 }, /* (247) into_opt ::= INTO full_table_name */ + { 305, 0 }, /* (248) stream_options ::= */ + { 305, -3 }, /* (249) stream_options ::= stream_options TRIGGER AT_ONCE */ + { 305, -3 }, /* (250) stream_options ::= stream_options TRIGGER WINDOW_CLOSE */ + { 305, -4 }, /* (251) stream_options ::= stream_options TRIGGER MAX_DELAY duration_literal */ + { 305, -3 }, /* (252) stream_options ::= stream_options WATERMARK duration_literal */ { 240, -3 }, /* (253) cmd ::= KILL CONNECTION NK_INTEGER */ { 240, -3 }, /* (254) cmd ::= KILL QUERY NK_INTEGER */ { 240, -3 }, /* (255) cmd ::= KILL TRANSACTION NK_INTEGER */ @@ -2836,8 +2835,8 @@ static const struct { { 240, -4 }, /* (257) cmd ::= MERGE VGROUP NK_INTEGER NK_INTEGER */ { 240, -4 }, /* (258) cmd ::= REDISTRIBUTE VGROUP NK_INTEGER dnode_list */ { 240, -3 }, /* (259) cmd ::= SPLIT VGROUP NK_INTEGER */ - { 306, -2 }, /* (260) dnode_list ::= DNODE NK_INTEGER */ - { 306, -3 }, /* (261) dnode_list ::= dnode_list DNODE NK_INTEGER */ + { 307, -2 }, /* (260) dnode_list ::= DNODE NK_INTEGER */ + { 307, -3 }, /* (261) dnode_list ::= dnode_list DNODE NK_INTEGER */ { 240, -3 }, /* (262) cmd ::= SYNCDB db_name REPLICA */ { 240, -4 }, /* (263) cmd ::= DELETE FROM full_table_name where_clause_opt */ { 240, -1 }, /* (264) cmd ::= query_expression */ @@ -2850,12 +2849,12 @@ static const struct { { 243, -1 }, /* (271) literal ::= NULL */ { 243, -1 }, /* (272) literal ::= NK_QUESTION */ { 293, -1 }, /* (273) duration_literal ::= NK_VARIABLE */ - { 308, -1 }, /* (274) signed ::= NK_INTEGER */ - { 308, -2 }, /* (275) signed ::= NK_PLUS NK_INTEGER */ - { 308, -2 }, /* (276) signed ::= NK_MINUS NK_INTEGER */ - { 308, -1 }, /* (277) signed ::= NK_FLOAT */ - { 308, -2 }, /* (278) signed ::= NK_PLUS NK_FLOAT */ - { 308, -2 }, /* (279) signed ::= NK_MINUS NK_FLOAT */ + { 309, -1 }, /* (274) signed ::= NK_INTEGER */ + { 309, -2 }, /* (275) signed ::= NK_PLUS NK_INTEGER */ + { 309, -2 }, /* (276) signed ::= NK_MINUS NK_INTEGER */ + { 309, -1 }, /* (277) signed ::= NK_FLOAT */ + { 309, -2 }, /* (278) signed ::= NK_PLUS NK_FLOAT */ + { 309, -2 }, /* (279) signed ::= NK_MINUS NK_FLOAT */ { 273, -1 }, /* (280) signed_literal ::= signed */ { 273, -1 }, /* (281) signed_literal ::= NK_STRING */ { 273, -1 }, /* (282) signed_literal ::= NK_BOOL */ @@ -2863,8 +2862,8 @@ static const struct { { 273, -1 }, /* (284) signed_literal ::= duration_literal */ { 273, -1 }, /* (285) signed_literal ::= NULL */ { 273, -1 }, /* (286) signed_literal ::= literal_func */ - { 310, -1 }, /* (287) literal_list ::= signed_literal */ - { 310, -3 }, /* (288) literal_list ::= literal_list NK_COMMA signed_literal */ + { 276, -1 }, /* (287) literal_list ::= signed_literal */ + { 276, -3 }, /* (288) literal_list ::= literal_list NK_COMMA signed_literal */ { 250, -1 }, /* (289) db_name ::= NK_ID */ { 279, -1 }, /* (290) table_name ::= NK_ID */ { 271, -1 }, /* (291) column_name ::= NK_ID */ @@ -2873,9 +2872,9 @@ static const struct { { 312, -1 }, /* (294) column_alias ::= NK_ID */ { 245, -1 }, /* (295) user_name ::= NK_ID */ { 290, -1 }, /* (296) index_name ::= NK_ID */ - { 296, -1 }, /* (297) topic_name ::= NK_ID */ - { 303, -1 }, /* (298) stream_name ::= NK_ID */ - { 298, -1 }, /* (299) cgroup_name ::= NK_ID */ + { 297, -1 }, /* (297) topic_name ::= NK_ID */ + { 304, -1 }, /* (298) stream_name ::= NK_ID */ + { 299, -1 }, /* (299) cgroup_name ::= NK_ID */ { 313, -1 }, /* (300) expression ::= literal */ { 313, -1 }, /* (301) expression ::= pseudo_column */ { 313, -1 }, /* (302) expression ::= column_reference */ @@ -2890,8 +2889,8 @@ static const struct { { 313, -3 }, /* (311) expression ::= expression NK_SLASH expression */ { 313, -3 }, /* (312) expression ::= expression NK_REM expression */ { 313, -3 }, /* (313) expression ::= column_reference NK_ARROW NK_STRING */ - { 276, -1 }, /* (314) expression_list ::= expression */ - { 276, -3 }, /* (315) expression_list ::= expression_list NK_COMMA expression */ + { 296, -1 }, /* (314) expression_list ::= expression */ + { 296, -3 }, /* (315) expression_list ::= expression_list NK_COMMA expression */ { 315, -1 }, /* (316) column_reference ::= column_name */ { 315, -3 }, /* (317) column_reference ::= table_name NK_DOT column_name */ { 314, -1 }, /* (318) pseudo_column ::= ROWTS */ @@ -2906,8 +2905,8 @@ static const struct { { 316, -4 }, /* (327) function_expression ::= star_func NK_LP star_func_para_list NK_RP */ { 316, -6 }, /* (328) function_expression ::= CAST NK_LP expression AS type_name NK_RP */ { 316, -1 }, /* (329) function_expression ::= literal_func */ - { 309, -3 }, /* (330) literal_func ::= noarg_func NK_LP NK_RP */ - { 309, -1 }, /* (331) literal_func ::= NOW */ + { 310, -3 }, /* (330) literal_func ::= noarg_func NK_LP NK_RP */ + { 310, -1 }, /* (331) literal_func ::= NOW */ { 320, -1 }, /* (332) noarg_func ::= NOW */ { 320, -1 }, /* (333) noarg_func ::= TODAY */ { 320, -1 }, /* (334) noarg_func ::= TIMEZONE */ @@ -2978,8 +2977,8 @@ static const struct { { 347, -2 }, /* (399) select_item ::= common_expression column_alias */ { 347, -3 }, /* (400) select_item ::= common_expression AS column_alias */ { 347, -3 }, /* (401) select_item ::= table_name NK_DOT NK_STAR */ - { 307, 0 }, /* (402) where_clause_opt ::= */ - { 307, -2 }, /* (403) where_clause_opt ::= WHERE search_condition */ + { 308, 0 }, /* (402) where_clause_opt ::= */ + { 308, -2 }, /* (403) where_clause_opt ::= WHERE search_condition */ { 342, 0 }, /* (404) partition_by_clause_opt ::= */ { 342, -3 }, /* (405) partition_by_clause_opt ::= PARTITION BY expression_list */ { 343, 0 }, /* (406) twindow_clause_opt ::= */ @@ -3003,7 +3002,7 @@ static const struct { { 350, -3 }, /* (424) group_by_list ::= group_by_list NK_COMMA expression */ { 345, 0 }, /* (425) having_clause_opt ::= */ { 345, -2 }, /* (426) having_clause_opt ::= HAVING search_condition */ - { 297, -4 }, /* (427) query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt */ + { 298, -4 }, /* (427) query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt */ { 351, -1 }, /* (428) query_expression_body ::= query_primary */ { 351, -4 }, /* (429) query_expression_body ::= query_expression_body UNION ALL query_expression_body */ { 351, -3 }, /* (430) query_expression_body ::= query_expression_body UNION query_expression_body */ @@ -3532,7 +3531,7 @@ static YYACTIONTYPE yy_reduce( { yylhsminor.yy424 = addNodeToList(pCxt, yymsp[-1].minor.yy424, yymsp[0].minor.yy632); } yymsp[-1].minor.yy424 = yylhsminor.yy424; break; - case 125: /* create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_tags_opt TAGS NK_LP expression_list NK_RP table_options */ + case 125: /* create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_tags_opt TAGS NK_LP literal_list NK_RP table_options */ { yylhsminor.yy632 = createCreateSubTableClause(pCxt, yymsp[-9].minor.yy137, yymsp[-8].minor.yy632, yymsp[-6].minor.yy632, yymsp[-5].minor.yy424, yymsp[-2].minor.yy424, yymsp[0].minor.yy632); } yymsp[-9].minor.yy632 = yylhsminor.yy632; break; From 04593fa44ec4e37d13832ec3e8ce4efe07da24f5 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Tue, 14 Jun 2022 16:38:33 +0800 Subject: [PATCH 72/78] fix(stream): build table name --- source/client/src/tmq.c | 12 ++++++++---- source/common/src/tdatablock.c | 1 + source/dnode/vnode/src/tq/tqSink.c | 20 +++++++++++--------- 3 files changed, 20 insertions(+), 13 deletions(-) diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index 48c85cf265..10c6c3623a 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -199,8 +199,8 @@ typedef struct { int8_t automatic; int8_t async; int8_t freeOffsets; - int8_t waitingRspNum; - int8_t totalRspNum; + int32_t waitingRspNum; + int32_t totalRspNum; tmq_resp_err_t rspErr; tmq_commit_cb* userCb; SArray* successfulOffsets; @@ -373,8 +373,9 @@ int32_t tmqCommitCb2(void* param, const SDataBuf* pBuf, int32_t code) { } else { taosArrayPush(pParamSet->successfulOffsets, &pParam->pOffset); } + // count down waiting rsp - int8_t waitingRspNum = atomic_sub_fetch_8(&pParam->params->waitingRspNum, 1); + int32_t waitingRspNum = atomic_sub_fetch_32(&pParamSet->waitingRspNum, 1); ASSERT(waitingRspNum >= 0); if (waitingRspNum == 0) { @@ -395,7 +396,8 @@ int32_t tmqCommitCb2(void* param, const SDataBuf* pBuf, int32_t code) { return 0; } -int32_t tmqComitInner2(tmq_t* tmq, int8_t automatic, int8_t async, tmq_commit_cb* userCb, void* userParam) { +int32_t tmqCommitInner2(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, int8_t automatic, int8_t async, + tmq_commit_cb* userCb, void* userParam) { int32_t code = -1; SMqCommitCbParamSet* pParamSet = taosMemoryCalloc(1, sizeof(SMqCommitCbParamSet)); @@ -466,6 +468,8 @@ int32_t tmqComitInner2(tmq_t* tmq, int8_t automatic, int8_t async, tmq_commit_cb SEpSet epSet = getEpSet_s(&tmq->pTscObj->pAppInfo->mgmtEp); int64_t transporterId = 0; asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, pMsgSendInfo); + pParamSet->waitingRspNum++; + pParamSet->totalRspNum++; } } diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index b4ece426b3..5a2aaed74e 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1708,6 +1708,7 @@ char* buildCtbNameByGroupId(const char* stbName, uint64_t groupId) { pTag->keyLen = strlen(pTag->key); pTag->type = TSDB_DATA_TYPE_UBIGINT; pTag->u = groupId; + pTag->length = sizeof(uint64_t); taosArrayPush(tags, &pTag); void* cname = taosMemoryCalloc(1, TSDB_TABLE_NAME_LEN + 1); diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index 391a008440..0cca1d2e10 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -46,11 +46,12 @@ static SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSche createTbReq.type = TSDB_CHILD_TABLE; createTbReq.ctb.suid = suid; - STagVal tagVal = {.cid = pDataBlock->info.numOfCols + 1, - .type = TSDB_DATA_TYPE_UBIGINT, - .pData = (uint8_t*)&pDataBlock->info.groupId, - .nData = sizeof(uint64_t)}; - STag* pTag = NULL; + STagVal tagVal = { + .cid = pDataBlock->info.numOfCols + 1, + .type = TSDB_DATA_TYPE_UBIGINT, + .i64 = (int64_t)pDataBlock->info.groupId, + }; + STag* pTag = NULL; taosArrayClear(tagArray); taosArrayPush(tagArray, &tagVal); tTagNew(tagArray, 1, false, &pTag); @@ -110,10 +111,11 @@ static SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSche createTbReq.type = TSDB_CHILD_TABLE; createTbReq.ctb.suid = suid; - STagVal tagVal = {.cid = pDataBlock->info.numOfCols + 1, - .type = TSDB_DATA_TYPE_UBIGINT, - .pData = (uint8_t*)&pDataBlock->info.groupId, - .nData = sizeof(uint64_t)}; + STagVal tagVal = { + .cid = pDataBlock->info.numOfCols + 1, + .type = TSDB_DATA_TYPE_UBIGINT, + .i64 = (int64_t)pDataBlock->info.groupId, + }; taosArrayClear(tagArray); taosArrayPush(tagArray, &tagVal); STag* pTag = NULL; From cd94eabc73440242b7d81ae4f5ab239b7409c146 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 14 Jun 2022 16:42:05 +0800 Subject: [PATCH 73/78] add stddev function translate partial/merge --- include/libs/function/functionMgt.h | 2 + source/libs/function/inc/builtinsimpl.h | 1 + source/libs/function/src/builtins.c | 53 +++++++++++++++++++++++++ source/libs/function/src/builtinsimpl.c | 2 + 4 files changed, 58 insertions(+) diff --git a/include/libs/function/functionMgt.h b/include/libs/function/functionMgt.h index f324d8fed1..a73b9b47f4 100644 --- a/include/libs/function/functionMgt.h +++ b/include/libs/function/functionMgt.h @@ -144,6 +144,8 @@ typedef enum EFunctionType { FUNCTION_TYPE_LAST_MERGE, FUNCTION_TYPE_AVG_PARTIAL, FUNCTION_TYPE_AVG_MERGE, + FUNCTION_TYPE_STDDEV_PARTIAL, + FUNCTION_TYPE_STDDEV_MERGE, // user defined funcion FUNCTION_TYPE_UDF = 10000 diff --git a/source/libs/function/inc/builtinsimpl.h b/source/libs/function/inc/builtinsimpl.h index 1bb30934c6..0d08f97c77 100644 --- a/source/libs/function/inc/builtinsimpl.h +++ b/source/libs/function/inc/builtinsimpl.h @@ -68,6 +68,7 @@ int32_t stddevFunction(SqlFunctionCtx* pCtx); int32_t stddevFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); int32_t stddevInvertFunction(SqlFunctionCtx* pCtx); int32_t stddevCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx); +int32_t getStddevInfoSize(); bool getLeastSQRFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool leastSQRFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo); diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 4dae296f05..0099f9e8c4 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -184,6 +184,35 @@ static int32_t translateAvgMerge(SFunctionNode* pFunc, char* pErrBuf, int32_t le return TSDB_CODE_SUCCESS; } +static int32_t translateStddevPartial(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { + if (1 != LIST_LENGTH(pFunc->pParameterList)) { + return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); + } + + uint8_t paraType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type; + if (!IS_NUMERIC_TYPE(paraType) && !IS_NULL_TYPE(paraType)) { + return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); + } + + pFunc->node.resType = (SDataType){.bytes = getStddevInfoSize() + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}; + return TSDB_CODE_SUCCESS; +} + +static int32_t translateStddevMerge(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { + if (1 != LIST_LENGTH(pFunc->pParameterList)) { + return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); + } + + uint8_t paraType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type; + if (TSDB_DATA_TYPE_BINARY != paraType) { + return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); + } + + pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes, .type = TSDB_DATA_TYPE_DOUBLE}; + + return TSDB_CODE_SUCCESS; +} + static int32_t translateWduration(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { // pseudo column do not need to check parameters pFunc->node.resType = (SDataType){.bytes = sizeof(int64_t), .type = TSDB_DATA_TYPE_BIGINT}; @@ -1523,6 +1552,30 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .invertFunc = stddevInvertFunction, .combineFunc = stddevCombine, }, + { + .name = "_stddev_partial", + .type = FUNCTION_TYPE_STDDEV_PARTIAL, + .classification = FUNC_MGT_AGG_FUNC, + .translateFunc = translateStddevPartial, + .getEnvFunc = getStddevFuncEnv, + .initFunc = stddevFunctionSetup, + .processFunc = stddevFunction, + .finalizeFunc = stddevFinalize, + .invertFunc = stddevInvertFunction, + .combineFunc = stddevCombine, + }, + { + .name = "_stddev_merge", + .type = FUNCTION_TYPE_STDDEV_MERGE, + .classification = FUNC_MGT_AGG_FUNC, + .translateFunc = translateStddevMerge, + .getEnvFunc = getStddevFuncEnv, + .initFunc = stddevFunctionSetup, + .processFunc = stddevFunction, + .finalizeFunc = stddevFinalize, + .invertFunc = stddevInvertFunction, + .combineFunc = stddevCombine, + }, { .name = "leastsquares", .type = FUNCTION_TYPE_LEASTSQUARES, diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 5ee9dbdee6..648a60c299 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -1488,6 +1488,8 @@ int32_t maxCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) { return minMaxCombine(pDestCtx, pSourceCtx, 0); } +int32_t getStddevInfoSize() { return (int32_t)sizeof(SStddevRes); } + bool getStddevFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) { pEnv->calcMemSize = sizeof(SStddevRes); return true; From 68d33b1656abd0f3267f791634da99ba31298e96 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 14 Jun 2022 16:53:59 +0800 Subject: [PATCH 74/78] add stddev function partial/merge --- source/libs/function/src/builtinsimpl.c | 56 ++++++++++++++++++++++++- 1 file changed, 55 insertions(+), 1 deletion(-) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 648a60c299..c6ab9f859d 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -89,6 +89,7 @@ typedef struct SStddevRes { double dsum; int64_t isum; }; + int16_t type; } SStddevRes; typedef struct SLeastSQRInfo { @@ -1513,6 +1514,7 @@ int32_t stddevFunction(SqlFunctionCtx* pCtx) { int32_t type = pInput->pData[0]->info.type; SStddevRes* pStddevRes = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); + pStddevRes->type = type; // computing based on the true data block SColumnInfoData* pCol = pInput->pData[0]; @@ -1629,6 +1631,39 @@ _stddev_over: return TSDB_CODE_SUCCESS; } +static void stddevTransferInfo(SStddevRes* pInput, SStddevRes* pOutput) { + pOutput->type = pInput->type; + if (IS_INTEGER_TYPE(pOutput->type)) { + pOutput->quadraticISum += pInput->quadraticISum; + pOutput->isum += pInput->isum; + } else { + pOutput->quadraticDSum += pInput->quadraticDSum; + pOutput->dsum += pInput->dsum; + } + + pOutput->count += pInput->count; + + return; +} + +int32_t stddevFunctionMerge(SqlFunctionCtx* pCtx) { + SInputColumnInfoData* pInput = &pCtx->input; + SColumnInfoData* pCol = pInput->pData[0]; + ASSERT(pCol->info.type == TSDB_DATA_TYPE_BINARY); + + SStddevRes* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); + + int32_t start = pInput->startRowIndex; + char* data = colDataGetData(pCol, start); + SStddevRes* pInputInfo = (SStddevRes*)varDataVal(data); + + stddevTransferInfo(pInputInfo, pInfo); + + SET_VAL(GET_RES_INFO(pCtx), 1, 1); + + return TSDB_CODE_SUCCESS; +} + #define LIST_STDDEV_SUB_N(sumT, T) \ do { \ T* plist = (T*)pCol->pData; \ @@ -1694,9 +1729,10 @@ int32_t stddevInvertFunction(SqlFunctionCtx* pCtx) { int32_t stddevFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { SInputColumnInfoData* pInput = &pCtx->input; - int32_t type = pInput->pData[0]->info.type; SStddevRes* pStddevRes = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); + int32_t type = pStddevRes->type; double avg; + if (IS_INTEGER_TYPE(type)) { avg = pStddevRes->isum / ((double)pStddevRes->count); pStddevRes->result = sqrt(pStddevRes->quadraticISum / ((double)pStddevRes->count) - avg * avg); @@ -1708,6 +1744,24 @@ int32_t stddevFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { return functionFinalize(pCtx, pBlock); } +int32_t stddevPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { + SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); + SStddevRes* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); + int32_t resultBytes = getStddevInfoSize(); + char* res = taosMemoryCalloc(resultBytes + VARSTR_HEADER_SIZE, sizeof(char)); + + memcpy(varDataVal(res), pInfo, resultBytes); + varDataSetLen(res, resultBytes); + + int32_t slotId = pCtx->pExpr->base.resSchema.slotId; + SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); + + colDataAppend(pCol, pBlock->info.rows, res, false); + + taosMemoryFree(res); + return pResInfo->numOfRes; +} + int32_t stddevCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) { SResultRowEntryInfo* pDResInfo = GET_RES_INFO(pDestCtx); SStddevRes* pDBuf = GET_ROWCELL_INTERBUF(pDResInfo); From 7ed7fd0fd681689f86515ed7f4f34f8d1ce61fcf Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 14 Jun 2022 17:00:32 +0800 Subject: [PATCH 75/78] enable stddev splitting --- source/libs/function/inc/builtinsimpl.h | 2 ++ source/libs/function/src/builtins.c | 6 ++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/source/libs/function/inc/builtinsimpl.h b/source/libs/function/inc/builtinsimpl.h index 0d08f97c77..f3060243ed 100644 --- a/source/libs/function/inc/builtinsimpl.h +++ b/source/libs/function/inc/builtinsimpl.h @@ -65,7 +65,9 @@ int32_t getAvgInfoSize(); bool getStddevFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool stddevFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo); int32_t stddevFunction(SqlFunctionCtx* pCtx); +int32_t stddevFunctionMerge(SqlFunctionCtx* pCtx); int32_t stddevFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); +int32_t stddevPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); int32_t stddevInvertFunction(SqlFunctionCtx* pCtx); int32_t stddevCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx); int32_t getStddevInfoSize(); diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 0099f9e8c4..4ffdf44cd6 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -1551,6 +1551,8 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .finalizeFunc = stddevFinalize, .invertFunc = stddevInvertFunction, .combineFunc = stddevCombine, + .pPartialFunc = "_stddev_partial", + .pMergeFunc = "_stddev_merge" }, { .name = "_stddev_partial", @@ -1560,7 +1562,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .getEnvFunc = getStddevFuncEnv, .initFunc = stddevFunctionSetup, .processFunc = stddevFunction, - .finalizeFunc = stddevFinalize, + .finalizeFunc = stddevPartialFinalize, .invertFunc = stddevInvertFunction, .combineFunc = stddevCombine, }, @@ -1571,7 +1573,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .translateFunc = translateStddevMerge, .getEnvFunc = getStddevFuncEnv, .initFunc = stddevFunctionSetup, - .processFunc = stddevFunction, + .processFunc = stddevFunctionMerge, .finalizeFunc = stddevFinalize, .invertFunc = stddevInvertFunction, .combineFunc = stddevCombine, From 1248b5fbb345b1b0f80b27e9fa7e52dad5f7db80 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Tue, 14 Jun 2022 16:57:42 +0800 Subject: [PATCH 76/78] enh(stream): forbid indefinite row functions for streams --- source/libs/function/src/builtins.c | 33 +++++++++++++++-------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 1c3f274ee9..ed3d838b8c 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -393,7 +393,8 @@ static int32_t translateTopBotImpl(SFunctionNode* pFunc, char* pErrBuf, int32_t pValue->notReserved = true; // set result type - pFunc->node.resType = (SDataType){.bytes = getTopBotInfoSize(pValue->datum.i) + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}; + pFunc->node.resType = + (SDataType){.bytes = getTopBotInfoSize(pValue->datum.i) + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}; } else { if (1 != numOfParams) { return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); @@ -956,11 +957,11 @@ static int32_t translateFirstLast(SFunctionNode* pFunc, char* pErrBuf, int32_t l static int32_t translateFirstLastImpl(SFunctionNode* pFunc, char* pErrBuf, int32_t len, bool isPartial) { // first(col_list) will be rewritten as first(col) - if (2 != LIST_LENGTH(pFunc->pParameterList)) { //input has two params c0,ts, is this a bug? + if (2 != LIST_LENGTH(pFunc->pParameterList)) { // input has two params c0,ts, is this a bug? return TSDB_CODE_SUCCESS; } - SNode* pPara = nodesListGetNode(pFunc->pParameterList, 0); + SNode* pPara = nodesListGetNode(pFunc->pParameterList, 0); uint8_t paraType = ((SExprNode*)pPara)->resType.type; int32_t paraBytes = ((SExprNode*)pPara)->resType.bytes; if (isPartial) { @@ -969,8 +970,8 @@ static int32_t translateFirstLastImpl(SFunctionNode* pFunc, char* pErrBuf, int32 "The parameters of first/last can only be columns"); } - pFunc->node.resType = (SDataType){.bytes = getFirstLastInfoSize(paraBytes) + VARSTR_HEADER_SIZE, - .type = TSDB_DATA_TYPE_BINARY}; + pFunc->node.resType = + (SDataType){.bytes = getFirstLastInfoSize(paraBytes) + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}; } else { if (TSDB_DATA_TYPE_BINARY != paraType) { return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); @@ -1521,7 +1522,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "percentile", .type = FUNCTION_TYPE_PERCENTILE, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_REPEAT_SCAN_FUNC, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_REPEAT_SCAN_FUNC | FUNC_MGT_FORBID_STREAM_FUNC, .translateFunc = translatePercentile, .getEnvFunc = getPercentileFuncEnv, .initFunc = percentileFunctionSetup, @@ -1682,7 +1683,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "elapsed", .type = FUNCTION_TYPE_ELAPSED, - .classification = FUNC_MGT_AGG_FUNC, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_FORBID_STREAM_FUNC, .dataRequiredFunc = statisDataRequired, .translateFunc = translateElapsed, .getEnvFunc = getElapsedFuncEnv, @@ -1803,7 +1804,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "twa", .type = FUNCTION_TYPE_TWA, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_TIMELINE_FUNC, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_FORBID_STREAM_FUNC, .translateFunc = translateInNumOutDou, .getEnvFunc = getTwaFuncEnv, .initFunc = twaFunctionSetup, @@ -1889,7 +1890,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "diff", .type = FUNCTION_TYPE_DIFF, - .classification = FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC, + .classification = FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_FORBID_STREAM_FUNC, .translateFunc = translateDiff, .getEnvFunc = getDiffFuncEnv, .initFunc = diffFunctionSetup, @@ -1899,7 +1900,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "statecount", .type = FUNCTION_TYPE_STATE_COUNT, - .classification = FUNC_MGT_INDEFINITE_ROWS_FUNC, + .classification = FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_FORBID_STREAM_FUNC, .translateFunc = translateStateCount, .getEnvFunc = getStateFuncEnv, .initFunc = functionSetup, @@ -1909,7 +1910,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "stateduration", .type = FUNCTION_TYPE_STATE_DURATION, - .classification = FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC, + .classification = FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_FORBID_STREAM_FUNC, .translateFunc = translateStateDuration, .getEnvFunc = getStateFuncEnv, .initFunc = functionSetup, @@ -1919,7 +1920,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "csum", .type = FUNCTION_TYPE_CSUM, - .classification = FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC, + .classification = FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_FORBID_STREAM_FUNC, .translateFunc = translateCsum, .getEnvFunc = getCsumFuncEnv, .initFunc = functionSetup, @@ -1929,7 +1930,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "mavg", .type = FUNCTION_TYPE_MAVG, - .classification = FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC, + .classification = FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_FORBID_STREAM_FUNC, .translateFunc = translateMavg, .getEnvFunc = getMavgFuncEnv, .initFunc = mavgFunctionSetup, @@ -1939,7 +1940,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "sample", .type = FUNCTION_TYPE_SAMPLE, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_INDEFINITE_ROWS_FUNC, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_FORBID_STREAM_FUNC, .translateFunc = translateSample, .getEnvFunc = getSampleFuncEnv, .initFunc = sampleFunctionSetup, @@ -1949,7 +1950,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "tail", .type = FUNCTION_TYPE_TAIL, - .classification = FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC, + .classification = FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_FORBID_STREAM_FUNC, .translateFunc = translateTail, .getEnvFunc = getTailFuncEnv, .initFunc = tailFunctionSetup, @@ -1959,7 +1960,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "unique", .type = FUNCTION_TYPE_UNIQUE, - .classification = FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC, + .classification = FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_FORBID_STREAM_FUNC, .translateFunc = translateUnique, .getEnvFunc = getUniqueFuncEnv, .initFunc = uniqueFunctionSetup, From 0226e39595462f29916e6850a694c26973e525f1 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Tue, 14 Jun 2022 17:13:30 +0800 Subject: [PATCH 77/78] fix: some problems of parser --- source/libs/parser/src/parTranslater.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 3c3c0e4a61..52abeadac9 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -3214,7 +3214,7 @@ static int32_t checkAlterSuperTable(STranslateContext* pCxt, SAlterTableStmt* pS if (NULL == pSchema) { code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLUMN, pStmt->colName); } else if (!IS_VAR_DATA_TYPE(pSchema->type) || pSchema->type != pStmt->dataType.type || - pSchema->bytes >= pStmt->dataType.bytes) { + pSchema->bytes >= calcTypeBytes(pStmt->dataType)) { code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_MODIFY_COL); } } From d0c658d3107d51e7ac2c383a731e03e1c3a01bc1 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 14 Jun 2022 17:19:10 +0800 Subject: [PATCH 78/78] enh: set vnode standby if drop it --- include/common/tmsgdef.h | 3 +- source/dnode/mgmt/mgmt_mnode/src/mmHandle.c | 6 +- source/dnode/mgmt/mgmt_vnode/src/vmHandle.c | 1 + source/dnode/mnode/impl/inc/mndInt.h | 2 - source/dnode/mnode/impl/src/mndMain.c | 4 +- source/dnode/mnode/impl/src/mndMnode.c | 5 +- source/dnode/mnode/impl/src/mndVgroup.c | 65 +++++++++++++++++++++ source/dnode/vnode/src/vnd/vnodeSvr.c | 4 ++ 8 files changed, 81 insertions(+), 9 deletions(-) diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 0f272aef9c..9c4df34f96 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -97,7 +97,6 @@ enum { TD_DEF_MSG_TYPE(TDMT_MND_CREATE_MNODE, "create-mnode", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_ALTER_MNODE, "alter-mnode", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_DROP_MNODE, "drop-mnode", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_SET_STANDBY, "set-mnode-standby", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_CREATE_QNODE, "create-qnode", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_ALTER_QNODE, "alter-qnode", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_DROP_QNODE, "drop-qnode", NULL, NULL) @@ -238,6 +237,8 @@ enum { TD_DEF_MSG_TYPE(TDMT_SYNC_SNAPSHOT_SEND, "sync-snapshot-send", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_SYNC_SNAPSHOT_RSP, "sync-snapshot-rsp", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_SYNC_LEADER_TRANSFER, "sync-leader-transfer", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_SYNC_SET_MNODE_STANDBY, "set-mnode-standby", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_SYNC_SET_VNODE_STANDBY, "set-vnode-standby", NULL, NULL) #if defined(TD_MSG_NUMBER_) TDMT_MAX diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c index e91f4b8cf4..8cda8fcec3 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c @@ -155,7 +155,6 @@ SArray *mmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_MNODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_ALTER_MNODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_ALTER_MNODE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_SET_STANDBY_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_MNODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_QNODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_QNODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; @@ -236,7 +235,10 @@ SArray *mmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES_REPLY, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_SNAPSHOT_SEND, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_SNAPSHOT_RSP, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_SET_STANDBY, mmPutMsgToSyncQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_SET_MNODE_STANDBY, mmPutMsgToSyncQueue, 0) == NULL) goto _OVER; + + if (dmSetMgmtHandle(pArray, TDMT_SYNC_SET_MNODE_STANDBY_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_SET_VNODE_STANDBY_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; code = 0; diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index ee120576c3..a5d7e1d4ca 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -373,6 +373,7 @@ SArray *vmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_SYNC_REQUEST_VOTE_REPLY, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_APPEND_ENTRIES_REPLY, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_SET_VNODE_STANDBY, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; code = 0; diff --git a/source/dnode/mnode/impl/inc/mndInt.h b/source/dnode/mnode/impl/inc/mndInt.h index 16220f101a..cc9bc5b634 100644 --- a/source/dnode/mnode/impl/inc/mndInt.h +++ b/source/dnode/mnode/impl/inc/mndInt.h @@ -124,8 +124,6 @@ void mndReleaseRpcRef(SMnode *pMnode); void mndSetRestore(SMnode *pMnode, bool restored); void mndSetStop(SMnode *pMnode); bool mndGetStop(SMnode *pMnode); -int32_t mndAcquireSyncRef(SMnode *pMnode); -void mndReleaseSyncRef(SMnode *pMnode); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index bd82701ae4..59599ee134 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -445,7 +445,7 @@ int32_t mndProcessSyncMsg(SRpcMsg *pMsg) { SyncSnapshotRsp *pSyncMsg = syncSnapshotRspFromRpcMsg2(pMsg); code = syncNodeOnSnapshotRspCb(pSyncNode, pSyncMsg); syncSnapshotRspDestroy(pSyncMsg); - } else if (pMsg->msgType == TDMT_MND_SET_STANDBY) { + } else if (pMsg->msgType == TDMT_SYNC_SET_MNODE_STANDBY) { code = syncSetStandby(pMgmt->sync); SRpcMsg rsp = {.code = code, .info = pMsg->info}; tmsgSendRsp(&rsp); @@ -486,7 +486,7 @@ int32_t mndProcessSyncMsg(SRpcMsg *pMsg) { SyncAppendEntriesReply *pSyncMsg = syncAppendEntriesReplyFromRpcMsg2(pMsg); code = syncNodeOnAppendEntriesReplyCb(pSyncNode, pSyncMsg); syncAppendEntriesReplyDestroy(pSyncMsg); - } else if (pMsg->msgType == TDMT_MND_SET_STANDBY) { + } else if (pMsg->msgType == TDMT_SYNC_SET_MNODE_STANDBY) { code = syncSetStandby(pMgmt->sync); SRpcMsg rsp = {.code = code, .info = pMsg->info}; tmsgSendRsp(&rsp); diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index f9749a969d..f6cef945e2 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -55,7 +55,8 @@ int32_t mndInitMnode(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_MND_ALTER_MNODE_RSP, mndTransProcessRsp); mndSetMsgHandle(pMnode, TDMT_MND_DROP_MNODE, mndProcessDropMnodeReq); mndSetMsgHandle(pMnode, TDMT_DND_DROP_MNODE_RSP, mndTransProcessRsp); - mndSetMsgHandle(pMnode, TDMT_MND_SET_STANDBY_RSP, mndTransProcessRsp); + mndSetMsgHandle(pMnode, TDMT_SYNC_SET_MNODE_STANDBY_RSP, mndTransProcessRsp); + mndSetMsgHandle(pMnode, TDMT_SYNC_SET_VNODE_STANDBY_RSP, mndTransProcessRsp); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_MNODE, mndRetrieveMnodes); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_MNODE, mndCancelGetNextMnode); @@ -511,7 +512,7 @@ static int32_t mndSetDropMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnode .epSet = dropEpSet, .pCont = pReq, .contLen = contLen, - .msgType = TDMT_MND_SET_STANDBY, + .msgType = TDMT_SYNC_SET_MNODE_STANDBY, .acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED, }; diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 6b2ee03524..804b4ef5d1 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -322,6 +322,33 @@ void *mndBuildAlterVnodeReq(SMnode *pMnode, SDbObj *pDb, SVgObj *pVgroup, int32_ return pReq; } +void *mndBuildSetVnodeStandbyReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen) { + SSetStandbyReq standbyReq = {0}; + standbyReq.dnodeId = pDnode->id; + standbyReq.standby = 1; + + int32_t contLen = tSerializeSSetStandbyReq(NULL, 0, &standbyReq); + if (contLen < 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + contLen += sizeof(SMsgHead); + void *pReq = taosMemoryMalloc(contLen); + if (pReq == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + tSerializeSSetStandbyReq((char *)pReq + sizeof(SMsgHead), contLen, &standbyReq); + SMsgHead *pHead = pReq; + pHead->contLen = htonl(contLen); + pHead->vgId = htonl(pVgroup->vgId); + + *pContLen = contLen; + return pReq; +} + void *mndBuildDropVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen) { SDropVnodeReq dropReq = {0}; dropReq.dnodeId = pDnode->id; @@ -898,6 +925,39 @@ int32_t mndAddAlterVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgO return 0; } +static int32_t mndAddSetVnodeStandByAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, + SVnodeGid *pVgid, bool isRedo) { + STransAction action = {0}; + + SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); + if (pDnode == NULL) return -1; + action.epSet = mndGetDnodeEpset(pDnode); + mndReleaseDnode(pMnode, pDnode); + + int32_t contLen = 0; + void *pReq = mndBuildSetVnodeStandbyReq(pMnode, pDnode, pDb, pVgroup, &contLen); + if (pReq == NULL) return -1; + + action.pCont = pReq; + action.contLen = contLen; + action.msgType = TDMT_DND_DROP_VNODE; + action.acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED; + + if (isRedo) { + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + taosMemoryFree(pReq); + return -1; + } + } else { + if (mndTransAppendUndoAction(pTrans, &action) != 0) { + taosMemoryFree(pReq); + return -1; + } + } + + return 0; +} + int32_t mndAddDropVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SVnodeGid *pVgid, bool isRedo) { STransAction action = {0}; @@ -952,6 +1012,7 @@ int32_t mndSetMoveVgroupInfoToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVnodeGid del = newVg.vnodeGid[vnIndex]; newVg.vnodeGid[vnIndex] = newVg.vnodeGid[newVg.replica]; memset(&newVg.vnodeGid[newVg.replica], 0, sizeof(SVnodeGid)); + if (mndAddSetVnodeStandByAction(pMnode, pTrans, pDb, pVgroup, &del, true) != 0) return -1; if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVg, TDMT_VND_ALTER_REPLICA) != 0) return -1; if (mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVg, &del, true) != 0) return -1; if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVg) != 0) return -1; @@ -1031,6 +1092,7 @@ static int32_t mndAddDecVgroupReplicaFromTrans(SMnode *pMnode, STrans *pTrans, S memcpy(pGid, &pVgroup->vnodeGid[pVgroup->replica], sizeof(SVnodeGid)); memset(&pVgroup->vnodeGid[pVgroup->replica], 0, sizeof(SVnodeGid)); + if (mndAddSetVnodeStandByAction(pMnode, pTrans, pDb, pVgroup, &delGid, true) != 0) return -1; if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, pVgroup, TDMT_VND_ALTER_REPLICA) != 0) return -1; if (mndAddDropVnodeAction(pMnode, pTrans, pDb, pVgroup, &delGid, true) != 0) return -1; if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, pVgroup) != 0) return -1; @@ -1341,12 +1403,14 @@ int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, S SVnodeGid del1 = {0}; if (mndRemoveVnodeFromVgroup(pMnode, &newVgroup, pArray, &del1) != 0) return -1; + if (mndAddSetVnodeStandByAction(pMnode, pTrans, pDb, pVgroup, &del1, true) != 0) return -1; if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup, TDMT_VND_ALTER_REPLICA) != 0) return -1; if (mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVgroup, &del1, true) != 0) return -1; if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVgroup) != 0) return -1; SVnodeGid del2 = {0}; if (mndRemoveVnodeFromVgroup(pMnode, &newVgroup, pArray, &del2) != 0) return -1; + if (mndAddSetVnodeStandByAction(pMnode, pTrans, pDb, pVgroup, &del2, true) != 0) return -1; if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVgroup, TDMT_VND_ALTER_REPLICA) != 0) return -1; if (mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVgroup, &del2, true) != 0) return -1; if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVgroup) != 0) return -1; @@ -1396,6 +1460,7 @@ static int32_t mndSplitVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SVgObj } else if (newVg1.replica == 3) { SVnodeGid del1 = {0}; if (mndRemoveVnodeFromVgroup(pMnode, &newVg1, pArray, &del1) != 0) goto _OVER; + if (mndAddSetVnodeStandByAction(pMnode, pTrans, pDb, pVgroup, &del1, true) != 0) return -1; if (mndAddAlterVnodeAction(pMnode, pTrans, pDb, &newVg1, TDMT_VND_ALTER_REPLICA) != 0) goto _OVER; if (mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVg1, &del1, true) != 0) goto _OVER; if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVg1) != 0) goto _OVER; diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 768f940d07..2964d7dc53 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -368,6 +368,10 @@ int32_t vnodeProcessSyncReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { ret = syncNodeOnAppendEntriesReplyCb(pSyncNode, pSyncMsg); syncAppendEntriesReplyDestroy(pSyncMsg); + } else if (pRpcMsg->msgType == TDMT_SYNC_SET_MNODE_STANDBY) { + ret = syncSetStandby(pVnode->sync); + SRpcMsg rsp = {.code = ret, .info = pMsg->info}; + tmsgSendRsp(&rsp); } else { vError("==vnodeProcessSyncReq== error msg type:%d", pRpcMsg->msgType); ret = TAOS_SYNC_OTHER_ERROR;