From c877418df00f36c1621a6ccc2e2adf65e0457611 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 17 Jul 2020 00:31:23 +0000 Subject: [PATCH 001/100] support multi thread update mnode and dnode --- src/client/inc/tsclient.h | 2 +- src/client/src/tscServer.c | 78 ++++++++++++++++++++++++++------------ src/client/src/tscSql.c | 2 +- src/client/src/tscSystem.c | 3 +- src/client/src/tscUtil.c | 17 +++++---- src/inc/trpc.h | 5 +++ 6 files changed, 72 insertions(+), 35 deletions(-) diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index be82eb64a8..e1180c4143 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -457,7 +457,7 @@ extern void * tscQhandle; extern int tscKeepConn[]; extern int tsInsertHeadSize; extern int tscNumOfThreads; -extern SRpcIpSet tscMgmtIpSet; +extern SRpcCorIpSet tscMgmtIpSet; extern int (*tscBuildMsg[TSDB_SQL_MAX])(SSqlObj *pSql, SSqlInfo *pInfo); diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index d8af6d5c87..e8a03a6482 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -29,7 +29,7 @@ #define TSC_MGMT_VNODE 999 -SRpcIpSet tscMgmtIpSet; +SRpcCorIpSet tscMgmtIpSet; SRpcIpSet tscDnodeIpSet; int (*tscBuildMsg[TSDB_SQL_MAX])(SSqlObj *pSql, SSqlInfo *pInfo) = {0}; @@ -44,6 +44,42 @@ void tscSaveSubscriptionProgress(void* sub); static int32_t minMsgSize() { return tsRpcHeadSize + 100; } +static void tscDumpMgmtIpSet(SRpcIpSet *ipSet) { + taosCorBeginRead(&tscMgmtIpSet.version); + *ipSet = tscMgmtIpSet.ipSet; + taosCorEndRead(&tscMgmtIpSet.version); +} + +bool tscIpSetIsEqual(SRpcIpSet *s1, SRpcIpSet *s2) { + if (s1->numOfIps != s2->numOfIps + || s1->inUse != s1->inUse) { + return false; + } + for (int32_t i = 0; i < s1->numOfIps; i++) { + if (s1->port[i] != s2->port[i] + || strncmp(s1->fqdn[i], s2->fqdn[i], TSDB_FQDN_LEN) != 0) + return false; + } + return true; +} +void tscSetMgmtIpList(SRpcIpSet *pIpSet) { + // no need to update if equal + SRpcIpSet dump; + tscDumpMgmtIpSet(&dump); + if (tscIpSetIsEqual(&dump, pIpSet)) { + return; + } + + taosCorBeginWrite(&tscMgmtIpSet.version); + SRpcIpSet *mgmtIpSet = &tscMgmtIpSet.ipSet; + mgmtIpSet->numOfIps = pIpSet->numOfIps; + mgmtIpSet->inUse = pIpSet->inUse; + for (int32_t i = 0; i < mgmtIpSet->numOfIps; ++i) { + mgmtIpSet->port[i] = htons(pIpSet->port[i]); + strncpy(mgmtIpSet->fqdn[i], pIpSet->fqdn[i], TSDB_FQDN_LEN); + } + taosCorEndWrite(&tscMgmtIpSet.version); +} static void tscSetDnodeIpList(SSqlObj* pSql, SCMVgroupInfo* pVgroupInfo) { SRpcIpSet* pIpList = &pSql->ipList; pIpList->inUse = 0; @@ -60,31 +96,17 @@ static void tscSetDnodeIpList(SSqlObj* pSql, SCMVgroupInfo* pVgroupInfo) { } void tscPrintMgmtIp() { - if (tscMgmtIpSet.numOfIps <= 0) { - tscError("invalid mnode IP list:%d", tscMgmtIpSet.numOfIps); + SRpcIpSet dump; + tscDumpMgmtIpSet(&dump); + if (dump.numOfIps <= 0) { + tscError("invalid mnode IP list:%d", dump.numOfIps); } else { - for (int i = 0; i < tscMgmtIpSet.numOfIps; ++i) { - tscDebug("mnode index:%d %s:%d", i, tscMgmtIpSet.fqdn[i], tscMgmtIpSet.port[i]); + for (int i = 0; i < dump.numOfIps; ++i) { + tscDebug("mnode index:%d %s:%d", i, dump.fqdn[i], dump.port[i]); } } } -void tscSetMgmtIpList(SRpcIpSet *pIpList) { - tscMgmtIpSet.numOfIps = pIpList->numOfIps; - tscMgmtIpSet.inUse = pIpList->inUse; - for (int32_t i = 0; i < tscMgmtIpSet.numOfIps; ++i) { - tscMgmtIpSet.port[i] = htons(pIpList->port[i]); - } -} - -void tscUpdateIpSet(void *ahandle, SRpcIpSet *pIpSet) { - tscMgmtIpSet = *pIpSet; - tscDebug("mnode IP list is changed for ufp is called, numOfIps:%d inUse:%d", tscMgmtIpSet.numOfIps, tscMgmtIpSet.inUse); - for (int32_t i = 0; i < tscMgmtIpSet.numOfIps; ++i) { - tscDebug("index:%d fqdn:%s port:%d", i, tscMgmtIpSet.fqdn[i], tscMgmtIpSet.port[i]); - } -} - /* * For each management node, try twice at least in case of poor network situation. * If the client start to connect to a non-management node from the client, and the first retry may fail due to @@ -95,7 +117,9 @@ void tscUpdateIpSet(void *ahandle, SRpcIpSet *pIpSet) { UNUSED_FUNC static int32_t tscGetMgmtConnMaxRetryTimes() { int32_t factor = 2; - return tscMgmtIpSet.numOfIps * factor; + SRpcIpSet dump; + tscDumpMgmtIpSet(&dump); + return dump.numOfIps * factor; } void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) { @@ -185,7 +209,9 @@ int tscSendMsgToServer(SSqlObj *pSql) { // set the mgmt ip list if (pSql->cmd.command >= TSDB_SQL_MGMT) { - pSql->ipList = tscMgmtIpSet; + SRpcIpSet dump; + tscDumpMgmtIpSet(&dump); + pSql->ipList = dump; } memcpy(pMsg, pSql->cmd.payload, pSql->cmd.payloadLen); @@ -239,7 +265,7 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcIpSet *pIpSet) { if (pCmd->command < TSDB_SQL_MGMT) { if (pIpSet) pSql->ipList = *pIpSet; } else { - if (pIpSet) tscMgmtIpSet = *pIpSet; + if (pIpSet) tscSetMgmtIpList(pIpSet); } if (rpcMsg->pCont == NULL) { @@ -421,7 +447,9 @@ int tscProcessSql(SSqlObj *pSql) { return pSql->res.code; } } else if (pCmd->command < TSDB_SQL_LOCAL) { - pSql->ipList = tscMgmtIpSet; + SRpcIpSet dump; + tscDumpMgmtIpSet(&dump); + pSql->ipList = dump; } else { // local handler return (*tscProcessMsgRsp[pCmd->command])(pSql); } diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index 0677463d8d..b24cfd9b27 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -63,7 +63,7 @@ SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pass, con if (ip) { if (tscSetMgmtIpListFromCfg(ip, NULL) < 0) return NULL; - if (port) tscMgmtIpSet.port[0] = port; + if (port) tscMgmtIpSet.ipSet.port[0] = port; } void *pDnodeConn = NULL; diff --git a/src/client/src/tscSystem.c b/src/client/src/tscSystem.c index 82cc8cc225..1a78ed87bd 100644 --- a/src/client/src/tscSystem.c +++ b/src/client/src/tscSystem.c @@ -41,7 +41,8 @@ int tscNumOfThreads; static pthread_once_t tscinit = PTHREAD_ONCE_INIT; void taosInitNote(int numOfNoteLines, int maxNotes, char* lable); -void tscUpdateIpSet(void *ahandle, SRpcIpSet *pIpSet); +//void tscUpdateIpSet(void *ahandle, SRpcIpSet *pIpSet); + void tscCheckDiskUsage(void *UNUSED_PARAM(para), void* UNUSED_PARAM(param)) { taosGetDisk(); diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 3e0fe0b4be..b0e6c727eb 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -2146,16 +2146,19 @@ char* strdup_throw(const char* str) { } int tscSetMgmtIpListFromCfg(const char *first, const char *second) { - tscMgmtIpSet.numOfIps = 0; - tscMgmtIpSet.inUse = 0; + // init mgmt ip set + tscMgmtIpSet.version = 0; + SRpcIpSet *mgmtIpSet = &(tscMgmtIpSet.ipSet); + mgmtIpSet->numOfIps = 0; + mgmtIpSet->inUse = 0; if (first && first[0] != 0) { if (strlen(first) >= TSDB_EP_LEN) { terrno = TSDB_CODE_TSC_INVALID_FQDN; return -1; } - taosGetFqdnPortFromEp(first, tscMgmtIpSet.fqdn[tscMgmtIpSet.numOfIps], &tscMgmtIpSet.port[tscMgmtIpSet.numOfIps]); - tscMgmtIpSet.numOfIps++; + taosGetFqdnPortFromEp(first, mgmtIpSet->fqdn[mgmtIpSet->numOfIps], &(mgmtIpSet->port[mgmtIpSet->numOfIps])); + mgmtIpSet->numOfIps++; } if (second && second[0] != 0) { @@ -2163,11 +2166,11 @@ int tscSetMgmtIpListFromCfg(const char *first, const char *second) { terrno = TSDB_CODE_TSC_INVALID_FQDN; return -1; } - taosGetFqdnPortFromEp(second, tscMgmtIpSet.fqdn[tscMgmtIpSet.numOfIps], &tscMgmtIpSet.port[tscMgmtIpSet.numOfIps]); - tscMgmtIpSet.numOfIps++; + taosGetFqdnPortFromEp(second, mgmtIpSet->fqdn[mgmtIpSet->numOfIps], &(mgmtIpSet->port[mgmtIpSet->numOfIps])); + mgmtIpSet->numOfIps++; } - if ( tscMgmtIpSet.numOfIps == 0) { + if (mgmtIpSet->numOfIps == 0) { terrno = TSDB_CODE_TSC_INVALID_FQDN; return -1; } diff --git a/src/inc/trpc.h b/src/inc/trpc.h index d1adfb7494..b159155e9d 100644 --- a/src/inc/trpc.h +++ b/src/inc/trpc.h @@ -35,6 +35,11 @@ typedef struct SRpcIpSet { char fqdn[TSDB_MAX_REPLICA][TSDB_FQDN_LEN]; } SRpcIpSet; +typedef struct SRpcCorIpSet { + int32_t version; + SRpcIpSet ipSet; +} SRpcCorIpSet; + typedef struct SRpcConnInfo { uint32_t clientIp; uint16_t clientPort; From 3d10b6f81d7d0f301bc3dd42405bf4f3869ab723 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 17 Jul 2020 04:25:37 +0000 Subject: [PATCH 002/100] support multi thread update dnode --- src/client/src/tscSchemaUtil.c | 3 ++ src/client/src/tscServer.c | 67 +++++++++++++++++++++------------- src/inc/taosmsg.h | 1 + 3 files changed, 45 insertions(+), 26 deletions(-) diff --git a/src/client/src/tscSchemaUtil.c b/src/client/src/tscSchemaUtil.c index 934a562387..b391a5760e 100644 --- a/src/client/src/tscSchemaUtil.c +++ b/src/client/src/tscSchemaUtil.c @@ -157,6 +157,9 @@ STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg, size_t* size pTableMeta->sid = pTableMetaMsg->sid; pTableMeta->uid = pTableMetaMsg->uid; pTableMeta->vgroupInfo = pTableMetaMsg->vgroup; + //init version here + pTableMeta->vgroupInfo.version = 0; + pTableMeta->sversion = pTableMetaMsg->sversion; pTableMeta->tversion = pTableMetaMsg->tversion; diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index e8a03a6482..4a11288a85 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -51,8 +51,7 @@ static void tscDumpMgmtIpSet(SRpcIpSet *ipSet) { } bool tscIpSetIsEqual(SRpcIpSet *s1, SRpcIpSet *s2) { - if (s1->numOfIps != s2->numOfIps - || s1->inUse != s1->inUse) { + if (s1->numOfIps != s2->numOfIps /*|| s1->inUse != s1->inUse*/) { return false; } for (int32_t i = 0; i < s1->numOfIps; i++) { @@ -80,21 +79,38 @@ void tscSetMgmtIpList(SRpcIpSet *pIpSet) { } taosCorEndWrite(&tscMgmtIpSet.version); } -static void tscSetDnodeIpList(SSqlObj* pSql, SCMVgroupInfo* pVgroupInfo) { - SRpcIpSet* pIpList = &pSql->ipList; - pIpList->inUse = 0; - if (pVgroupInfo == NULL) { - pIpList->numOfIps = 0; +static void tscDumpIpSetFromVgroupInfo(SCMVgroupInfo *pVgroupInfo, SRpcIpSet *pIpSet, int32_t *vgId) { + if (pVgroupInfo == NULL) { return;} + taosCorBeginRead(&pVgroupInfo->version); + if (vgId) { + *vgId = pVgroupInfo->vgId; + } + pIpSet->inUse = 0; + pIpSet->numOfIps = pVgroupInfo->numOfIps; + for (int32_t i = 0; i < pVgroupInfo->numOfIps; ++i) { + strncpy(pIpSet->fqdn[i], pVgroupInfo->ipAddr[i].fqdn, TSDB_FQDN_LEN); + pIpSet->port[i] = pVgroupInfo->ipAddr[i].port; + } + taosCorEndRead(&pVgroupInfo->version); +} +static void tscSetVgroupInfoWithIpSet(SCMVgroupInfo *pVgroupInfo, SRpcIpSet *pIpSet) { + taosCorBeginWrite(&pVgroupInfo->version); + //TODO(dengyihao), dont care vgid + pVgroupInfo->numOfIps = pIpSet->numOfIps; + for (int32_t i = 0; pVgroupInfo->numOfIps; i++) { + strncpy(pVgroupInfo->ipAddr[i].fqdn, pIpSet->fqdn[i], TSDB_FQDN_LEN); + pVgroupInfo->ipAddr[i].port = pIpSet->port[i]; + } + taosCorEndWrite(&pVgroupInfo->version); +} +static void tscSetVgroupInfo(SSqlObj *pObj, SRpcIpSet *pIpSet) { + if (tscIpSetIsEqual(&pObj->ipList, pIpSet)) { return; } - - pIpList->numOfIps = pVgroupInfo->numOfIps; - for(int32_t i = 0; i < pVgroupInfo->numOfIps; ++i) { - strcpy(pIpList->fqdn[i], pVgroupInfo->ipAddr[i].fqdn); - pIpList->port[i] = pVgroupInfo->ipAddr[i].port; - } + SSqlCmd *pCmd = &pObj->cmd; + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); + tscSetVgroupInfoWithIpSet(&pTableMetaInfo->pTableMeta->vgroupInfo, pIpSet); } - void tscPrintMgmtIp() { SRpcIpSet dump; tscDumpMgmtIpSet(&dump); @@ -263,7 +279,7 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcIpSet *pIpSet) { } if (pCmd->command < TSDB_SQL_MGMT) { - if (pIpSet) pSql->ipList = *pIpSet; + if (pIpSet) tscSetVgroupInfo(pSql, pIpSet); } else { if (pIpSet) tscSetMgmtIpList(pIpSet); } @@ -553,7 +569,7 @@ int tscBuildSubmitMsg(SSqlObj *pSql, SSqlInfo *pInfo) { // pSql->cmd.payloadLen is set during copying data into payload pSql->cmd.msgType = TSDB_MSG_TYPE_SUBMIT; - tscSetDnodeIpList(pSql, &pTableMeta->vgroupInfo); + tscDumpIpSetFromVgroupInfo(&pTableMeta->vgroupInfo, &pSql->ipList, NULL); tscDebug("%p build submit msg, vgId:%d numOfTables:%d numberOfIP:%d", pSql, vgId, pSql->cmd.numOfTablesInSubmit, pSql->ipList.numOfIps); @@ -595,11 +611,9 @@ static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char } else { pVgroupInfo = &pTableMeta->vgroupInfo; } - - tscSetDnodeIpList(pSql, pVgroupInfo); - if (pVgroupInfo != NULL) { - pQueryMsg->head.vgId = htonl(pVgroupInfo->vgId); - } + int32_t vgId = 0; + tscDumpIpSetFromVgroupInfo(pVgroupInfo, &pSql->ipList, &vgId); + pQueryMsg->head.vgId = htonl(vgId); STableIdInfo *pTableIdInfo = (STableIdInfo *)pMsg; pTableIdInfo->tid = htonl(pTableMeta->sid); @@ -618,8 +632,9 @@ static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char SVgroupTableInfo* pTableIdList = taosArrayGet(pTableMetaInfo->pVgroupTables, index); // set the vgroup info - tscSetDnodeIpList(pSql, &pTableIdList->vgInfo); - pQueryMsg->head.vgId = htonl(pTableIdList->vgInfo.vgId); + int32_t vgId = 0; + tscDumpIpSetFromVgroupInfo(&pTableIdList->vgInfo, &pSql->ipList, &vgId); + pQueryMsg->head.vgId = htonl(vgId); int32_t numOfTables = taosArrayGetSize(pTableIdList->itemList); pQueryMsg->numOfTables = htonl(numOfTables); // set the number of tables @@ -1351,7 +1366,7 @@ int tscBuildUpdateTagMsg(SSqlObj* pSql, SSqlInfo *pInfo) { SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - tscSetDnodeIpList(pSql, &pTableMetaInfo->pTableMeta->vgroupInfo); + tscDumpIpSetFromVgroupInfo(&pTableMetaInfo->pTableMeta->vgroupInfo, &pSql->ipList, NULL); return TSDB_CODE_SUCCESS; } @@ -1875,11 +1890,11 @@ int tscProcessSTableVgroupRsp(SSqlObj *pSql) { memcpy(pInfo->vgroupList, pVgroupInfo, size); for (int32_t j = 0; j < pInfo->vgroupList->numOfVgroups; ++j) { + //just init, no need to lock SCMVgroupInfo *pVgroups = &pInfo->vgroupList->vgroups[j]; - + pVgroups->version = 0; pVgroups->vgId = htonl(pVgroups->vgId); assert(pVgroups->numOfIps >= 1); - for (int32_t k = 0; k < pVgroups->numOfIps; ++k) { pVgroups->ipAddr[k].port = htons(pVgroups->ipAddr[k].port); } diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index b7afaf1e06..c12aed2bcc 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -647,6 +647,7 @@ typedef struct SCMSTableVgroupMsg { } SCMSTableVgroupMsg, SCMSTableVgroupRspMsg; typedef struct { + int32_t version; int32_t vgId; int8_t numOfIps; SIpAddr ipAddr[TSDB_MAX_REPLICA]; From dd45e955dc5ed8c2a9862abbcda319ced4531f68 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 17 Jul 2020 06:53:56 +0000 Subject: [PATCH 003/100] fix bug --- src/client/src/tscServer.c | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 4a11288a85..045aab61d6 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -44,9 +44,18 @@ void tscSaveSubscriptionProgress(void* sub); static int32_t minMsgSize() { return tsRpcHeadSize + 100; } +void tscIpSetCopy(SRpcIpSet *dst, SRpcIpSet *src) { + dst->numOfIps = src->numOfIps; + dst->inUse = src->inUse; + for (int32_t i = 0; i < src->numOfIps; ++i) { + dst->port[i] = htons(dst->port[i]); + strncpy(dst->fqdn[i], src->fqdn[i], TSDB_FQDN_LEN); + } +} static void tscDumpMgmtIpSet(SRpcIpSet *ipSet) { taosCorBeginRead(&tscMgmtIpSet.version); - *ipSet = tscMgmtIpSet.ipSet; + SRpcIpSet* src = &tscMgmtIpSet.ipSet; + tscIpSetCopy(ipSet, src); taosCorEndRead(&tscMgmtIpSet.version); } @@ -71,12 +80,7 @@ void tscSetMgmtIpList(SRpcIpSet *pIpSet) { taosCorBeginWrite(&tscMgmtIpSet.version); SRpcIpSet *mgmtIpSet = &tscMgmtIpSet.ipSet; - mgmtIpSet->numOfIps = pIpSet->numOfIps; - mgmtIpSet->inUse = pIpSet->inUse; - for (int32_t i = 0; i < mgmtIpSet->numOfIps; ++i) { - mgmtIpSet->port[i] = htons(pIpSet->port[i]); - strncpy(mgmtIpSet->fqdn[i], pIpSet->fqdn[i], TSDB_FQDN_LEN); - } + tscIpSetCopy(mgmtIpSet, pIpSet); taosCorEndWrite(&tscMgmtIpSet.version); } static void tscDumpIpSetFromVgroupInfo(SCMVgroupInfo *pVgroupInfo, SRpcIpSet *pIpSet, int32_t *vgId) { @@ -227,7 +231,7 @@ int tscSendMsgToServer(SSqlObj *pSql) { if (pSql->cmd.command >= TSDB_SQL_MGMT) { SRpcIpSet dump; tscDumpMgmtIpSet(&dump); - pSql->ipList = dump; + tscIpSetCopy(&pSql->ipList, &dump); } memcpy(pMsg, pSql->cmd.payload, pSql->cmd.payloadLen); From 6163e2401f5d18822c206475543a14019991c500 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 17 Jul 2020 07:42:04 +0000 Subject: [PATCH 004/100] refor code --- src/client/src/tscServer.c | 41 ++++++++++++++++++-------------------- 1 file changed, 19 insertions(+), 22 deletions(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 045aab61d6..c82773a125 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -70,15 +70,10 @@ bool tscIpSetIsEqual(SRpcIpSet *s1, SRpcIpSet *s2) { } return true; } -void tscSetMgmtIpList(SRpcIpSet *pIpSet) { +void tscUpdateMgmtIpList(SRpcIpSet *pIpSet) { // no need to update if equal - SRpcIpSet dump; - tscDumpMgmtIpSet(&dump); - if (tscIpSetIsEqual(&dump, pIpSet)) { - return; - } - taosCorBeginWrite(&tscMgmtIpSet.version); + // or copy directly, tscMgmtIpSet.ipSet = *pIpSet SRpcIpSet *mgmtIpSet = &tscMgmtIpSet.ipSet; tscIpSetCopy(mgmtIpSet, pIpSet); taosCorEndWrite(&tscMgmtIpSet.version); @@ -97,7 +92,12 @@ static void tscDumpIpSetFromVgroupInfo(SCMVgroupInfo *pVgroupInfo, SRpcIpSet *pI } taosCorEndRead(&pVgroupInfo->version); } -static void tscSetVgroupInfoWithIpSet(SCMVgroupInfo *pVgroupInfo, SRpcIpSet *pIpSet) { + +static void tscUpdateVgroupInfo(SSqlObj *pObj, SRpcIpSet *pIpSet) { + SSqlCmd *pCmd = &pObj->cmd; + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); + SCMVgroupInfo *pVgroupInfo = &pTableMetaInfo->pTableMeta->vgroupInfo; + taosCorBeginWrite(&pVgroupInfo->version); //TODO(dengyihao), dont care vgid pVgroupInfo->numOfIps = pIpSet->numOfIps; @@ -107,14 +107,6 @@ static void tscSetVgroupInfoWithIpSet(SCMVgroupInfo *pVgroupInfo, SRpcIpSet *pIp } taosCorEndWrite(&pVgroupInfo->version); } -static void tscSetVgroupInfo(SSqlObj *pObj, SRpcIpSet *pIpSet) { - if (tscIpSetIsEqual(&pObj->ipList, pIpSet)) { - return; - } - SSqlCmd *pCmd = &pObj->cmd; - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); - tscSetVgroupInfoWithIpSet(&pTableMetaInfo->pTableMeta->vgroupInfo, pIpSet); -} void tscPrintMgmtIp() { SRpcIpSet dump; tscDumpMgmtIpSet(&dump); @@ -157,7 +149,7 @@ void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) { SCMHeartBeatRsp *pRsp = (SCMHeartBeatRsp *)pRes->pRsp; SRpcIpSet * pIpList = &pRsp->ipList; if (pIpList->numOfIps > 0) - tscSetMgmtIpList(pIpList); + tscUpdateMgmtIpList(pIpList); pSql->pTscObj->connId = htonl(pRsp->connId); @@ -231,6 +223,7 @@ int tscSendMsgToServer(SSqlObj *pSql) { if (pSql->cmd.command >= TSDB_SQL_MGMT) { SRpcIpSet dump; tscDumpMgmtIpSet(&dump); + // no need to update pSql->ipList tscIpSetCopy(&pSql->ipList, &dump); } @@ -282,10 +275,14 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcIpSet *pIpSet) { return; } - if (pCmd->command < TSDB_SQL_MGMT) { - if (pIpSet) tscSetVgroupInfo(pSql, pIpSet); - } else { - if (pIpSet) tscSetMgmtIpList(pIpSet); + if (pIpSet) { + if (!tscIpSetIsEqual(&pSql->ipList, pIpSet)) { + if (pCmd->command < TSDB_SQL_MGMT) { + tscUpdateVgroupInfo(pSql, pIpSet); + } else { + tscUpdateMgmtIpList(pIpSet); + } + } } if (rpcMsg->pCont == NULL) { @@ -1994,7 +1991,7 @@ int tscProcessConnectRsp(SSqlObj *pSql) { tstrncpy(pObj->db, temp, sizeof(pObj->db)); if (pConnect->ipList.numOfIps > 0) - tscSetMgmtIpList(&pConnect->ipList); + tscUpdateMgmtIpList(&pConnect->ipList); strcpy(pObj->sversion, pConnect->serverVersion); pObj->writeAuth = pConnect->writeAuth; From aae9b3c1bc5db5d2007eb467c35a5a7eafdea3a6 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 17 Jul 2020 17:32:56 +0800 Subject: [PATCH 005/100] add a idx file for query speed --- src/tsdb/inc/tsdbMain.h | 34 +++-- src/tsdb/src/tsdbFile.c | 18 +-- src/tsdb/src/tsdbMain.c | 12 +- src/tsdb/src/tsdbMemTable.c | 7 +- src/tsdb/src/tsdbRWHelper.c | 261 +++++++++++++++++++----------------- 5 files changed, 178 insertions(+), 154 deletions(-) diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h index 40f2dac660..da5f03a4fb 100644 --- a/src/tsdb/inc/tsdbMain.h +++ b/src/tsdb/inc/tsdbMain.h @@ -132,21 +132,23 @@ typedef struct { // ------------------ tsdbFile.c extern const char* tsdbFileSuffix[]; typedef enum { - TSDB_FILE_TYPE_HEAD = 0, + TSDB_FILE_TYPE_IDX = 0, + TSDB_FILE_TYPE_HEAD, TSDB_FILE_TYPE_DATA, TSDB_FILE_TYPE_LAST, TSDB_FILE_TYPE_MAX, + TSDB_FILE_TYPE_NIDX, TSDB_FILE_TYPE_NHEAD, TSDB_FILE_TYPE_NLAST } TSDB_FILE_TYPE; typedef struct { - uint32_t offset; + uint32_t magic; uint32_t len; - uint64_t size; // total size of the file - uint64_t tombSize; // unused file size uint32_t totalBlocks; uint32_t totalSubBlocks; + uint64_t size; // total size of the file + uint64_t tombSize; // unused file size } STsdbFileInfo; typedef struct { @@ -249,16 +251,12 @@ typedef struct { typedef enum { TSDB_WRITE_HELPER, TSDB_READ_HELPER } tsdb_rw_helper_t; typedef struct { - int fid; - TSKEY minKey; - TSKEY maxKey; - // For read/write purpose - SFile headF; - SFile dataF; - SFile lastF; - // For write purpose only - SFile nHeadF; - SFile nLastF; + TSKEY minKey; + TSKEY maxKey; + SFileGroup fGroup; + SFile nIdxF; + SFile nHeadF; + SFile nLastF; } SHelperFile; typedef struct { @@ -444,6 +442,14 @@ void tsdbRemoveFileGroup(STsdbRepo* pRepo, SFileGroup* pFGroup); #define helperRepo(h) (h)->pRepo #define helperState(h) (h)->state #define TSDB_NLAST_FILE_OPENED(h) ((h)->files.nLastF.fd > 0) +#define helperFileId(h) ((h)->files.fGroup.fileId) +#define helperIdxF(h) (&((h)->files.fGroup.files[TSDB_FILE_TYPE_IDX])) +#define helperHeadF(h) (&((h)->files.fGroup.files[TSDB_FILE_TYPE_HEAD])) +#define helperDataF(h) (&((h)->files.fGroup.files[TSDB_FILE_TYPE_DATA])) +#define helperLastF(h) (&((h)->files.fGroup.files[TSDB_FILE_TYPE_LAST])) +#define helperNewIdxF(h) (&((h)->files.nIdxF)) +#define helperNewHeadF(h) (&((h)->files.nHeadF)) +#define helperNewLastF(h) (&((h)->files.nLastF)) int tsdbInitReadHelper(SRWHelper* pHelper, STsdbRepo* pRepo); int tsdbInitWriteHelper(SRWHelper* pHelper, STsdbRepo* pRepo); diff --git a/src/tsdb/src/tsdbFile.c b/src/tsdb/src/tsdbFile.c index 95cc47292b..7b7788c4d9 100644 --- a/src/tsdb/src/tsdbFile.c +++ b/src/tsdb/src/tsdbFile.c @@ -30,7 +30,7 @@ #include "ttime.h" #include "tfile.h" -const char *tsdbFileSuffix[] = {".head", ".data", ".last", "", ".h", ".l"}; +const char *tsdbFileSuffix[] = {".idx", ".head", ".data", ".last", "", ".i", ".h", ".l"}; static int tsdbInitFile(SFile *pFile, STsdbRepo *pRepo, int fid, int type); static void tsdbDestroyFile(SFile *pFile); @@ -108,7 +108,7 @@ int tsdbOpenFileH(STsdbRepo *pRepo) { memset((void *)(&fileGroup), 0, sizeof(SFileGroup)); fileGroup.fileId = fid; - for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) { + for (int type = TSDB_FILE_TYPE_IDX; type < TSDB_FILE_TYPE_MAX; type++) { if (tsdbInitFile(&fileGroup.files[type], pRepo, fid, type) < 0) { tsdbError("vgId:%d failed to init file fid %d type %d", REPO_ID(pRepo), fid, type); goto _err; @@ -126,7 +126,7 @@ int tsdbOpenFileH(STsdbRepo *pRepo) { return 0; _err: - for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) tsdbDestroyFile(&fileGroup.files[type]); + for (int type = TSDB_FILE_TYPE_IDX; type < TSDB_FILE_TYPE_MAX; type++) tsdbDestroyFile(&fileGroup.files[type]); tfree(tDataDir); if (dir != NULL) closedir(dir); @@ -139,7 +139,7 @@ void tsdbCloseFileH(STsdbRepo *pRepo) { for (int i = 0; i < pFileH->nFGroups; i++) { SFileGroup *pFGroup = pFileH->pFGroup + i; - for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) { + for (int type = TSDB_FILE_TYPE_IDX; type < TSDB_FILE_TYPE_MAX; type++) { tsdbDestroyFile(&pFGroup->files[type]); } } @@ -156,7 +156,7 @@ SFileGroup *tsdbCreateFGroupIfNeed(STsdbRepo *pRepo, char *dataDir, int fid, int SFileGroup *pGroup = tsdbSearchFGroup(pFileH, fid, TD_EQ); if (pGroup == NULL) { // if not exists, create one pFGroup->fileId = fid; - for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) { + for (int type = TSDB_FILE_TYPE_IDX; type < TSDB_FILE_TYPE_MAX; type++) { if (tsdbCreateFile(&pFGroup->files[type], pRepo, fid, type) < 0) goto _err; } @@ -169,7 +169,7 @@ SFileGroup *tsdbCreateFGroupIfNeed(STsdbRepo *pRepo, char *dataDir, int fid, int return pGroup; _err: - for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) tsdbDestroyFile(&pGroup->files[type]); + for (int type = TSDB_FILE_TYPE_IDX; type < TSDB_FILE_TYPE_MAX; type++) tsdbDestroyFile(&pGroup->files[type]); return NULL; } @@ -323,7 +323,7 @@ int tsdbUpdateFileHeader(SFile *pFile, uint32_t version) { int tsdbEncodeSFileInfo(void **buf, const STsdbFileInfo *pInfo) { int tlen = 0; - tlen += taosEncodeFixedU32(buf, pInfo->offset); + tlen += taosEncodeFixedU32(buf, pInfo->magic); tlen += taosEncodeFixedU32(buf, pInfo->len); tlen += taosEncodeFixedU64(buf, pInfo->size); tlen += taosEncodeFixedU64(buf, pInfo->tombSize); @@ -334,7 +334,7 @@ int tsdbEncodeSFileInfo(void **buf, const STsdbFileInfo *pInfo) { } void *tsdbDecodeSFileInfo(void *buf, STsdbFileInfo *pInfo) { - buf = taosDecodeFixedU32(buf, &(pInfo->offset)); + buf = taosDecodeFixedU32(buf, &(pInfo->magic)); buf = taosDecodeFixedU32(buf, &(pInfo->len)); buf = taosDecodeFixedU64(buf, &(pInfo->size)); buf = taosDecodeFixedU64(buf, &(pInfo->tombSize)); @@ -358,7 +358,7 @@ void tsdbRemoveFileGroup(STsdbRepo *pRepo, SFileGroup *pFGroup) { pFileH->nFGroups--; ASSERT(pFileH->nFGroups >= 0); - for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) { + for (int type = TSDB_FILE_TYPE_IDX; type < TSDB_FILE_TYPE_MAX; type++) { if (remove(fileGroup.files[type].fname) < 0) { tsdbError("vgId:%d failed to remove file %s", REPO_ID(pRepo), fileGroup.files[type].fname); } diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index ffaab375a3..848cd4e85a 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -214,7 +214,7 @@ uint32_t tsdbGetFileInfo(TSDB_REPO_T *repo, char *name, uint32_t *index, uint32_ char *prefix = dirname(sdup); if (name[0] == 0) { // get the file from index or after, but not larger than eindex - int fid = (*index) / 3; + int fid = (*index) / TSDB_FILE_TYPE_MAX; if (pFileH->nFGroups == 0 || fid > pFileH->pFGroup[pFileH->nFGroups - 1].fileId) { if (*index <= TSDB_META_FILE_INDEX && TSDB_META_FILE_INDEX <= eindex) { @@ -228,11 +228,11 @@ uint32_t tsdbGetFileInfo(TSDB_REPO_T *repo, char *name, uint32_t *index, uint32_ SFileGroup *pFGroup = taosbsearch(&fid, pFileH->pFGroup, pFileH->nFGroups, sizeof(SFileGroup), keyFGroupCompFunc, TD_GE); if (pFGroup->fileId == fid) { - fname = strdup(pFGroup->files[(*index) % 3].fname); + fname = strdup(pFGroup->files[(*index) % TSDB_FILE_TYPE_MAX].fname); } else { - if (pFGroup->fileId * 3 + 2 < eindex) { + if ((pFGroup->fileId + 1) * TSDB_FILE_TYPE_MAX - 1 < eindex) { fname = strdup(pFGroup->files[0].fname); - *index = pFGroup->fileId * 3; + *index = pFGroup->fileId * TSDB_FILE_TYPE_MAX; } else { tfree(sdup); return 0; @@ -244,14 +244,14 @@ uint32_t tsdbGetFileInfo(TSDB_REPO_T *repo, char *name, uint32_t *index, uint32_ if (*index == TSDB_META_FILE_INDEX) { // get meta file fname = tsdbGetMetaFileName(pRepo->rootDir); } else { - int fid = (*index) / 3; + int fid = (*index) / TSDB_FILE_TYPE_MAX; SFileGroup *pFGroup = tsdbSearchFGroup(pFileH, fid, TD_EQ); if (pFGroup == NULL) { // not found tfree(sdup); return 0; } - SFile *pFile = &pFGroup->files[(*index) % 3]; + SFile *pFile = &pFGroup->files[(*index) % TSDB_FILE_TYPE_MAX]; fname = strdup(pFile->fname); } } diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index b29cec3cf9..69e6ab3c0f 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -628,9 +628,10 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHe tsdbCloseHelperFile(pHelper, 0); pthread_rwlock_wrlock(&(pFileH->fhlock)); - pGroup->files[TSDB_FILE_TYPE_HEAD] = pHelper->files.headF; - pGroup->files[TSDB_FILE_TYPE_DATA] = pHelper->files.dataF; - pGroup->files[TSDB_FILE_TYPE_LAST] = pHelper->files.lastF; + pGroup->files[TSDB_FILE_TYPE_IDX] = *(helperIdxF(pHelper)); + pGroup->files[TSDB_FILE_TYPE_HEAD] = *(helperHeadF(pHelper)); + pGroup->files[TSDB_FILE_TYPE_DATA] = *(helperDataF(pHelper)); + pGroup->files[TSDB_FILE_TYPE_LAST] = *(helperLastF(pHelper)); pthread_rwlock_unlock(&(pFileH->fhlock)); return 0; diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index 0d52b7ae33..79b4a8ef64 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -106,44 +106,36 @@ int tsdbSetAndOpenHelperFile(SRWHelper *pHelper, SFileGroup *pGroup) { ASSERT(pHelper->state == TSDB_HELPER_CLEAR_STATE); // Set the files - pHelper->files.fid = pGroup->fileId; - pHelper->files.headF = pGroup->files[TSDB_FILE_TYPE_HEAD]; - pHelper->files.dataF = pGroup->files[TSDB_FILE_TYPE_DATA]; - pHelper->files.lastF = pGroup->files[TSDB_FILE_TYPE_LAST]; + pHelper->files.fGroup = *pGroup; if (helperType(pHelper) == TSDB_WRITE_HELPER) { - tsdbGetDataFileName(pHelper->pRepo, pGroup->fileId, TSDB_FILE_TYPE_NHEAD, pHelper->files.nHeadF.fname); - tsdbGetDataFileName(pHelper->pRepo, pGroup->fileId, TSDB_FILE_TYPE_NLAST, pHelper->files.nLastF.fname); + tsdbGetDataFileName(pHelper->pRepo, pGroup->fileId, TSDB_FILE_TYPE_NIDX, helperNewIdxF(pHelper)->fname); + tsdbGetDataFileName(pHelper->pRepo, pGroup->fileId, TSDB_FILE_TYPE_NHEAD, helperNewHeadF(pHelper)->fname); + tsdbGetDataFileName(pHelper->pRepo, pGroup->fileId, TSDB_FILE_TYPE_NLAST, helperNewLastF(pHelper)->fname); } // Open the files - if (tsdbOpenFile(&(pHelper->files.headF), O_RDONLY) < 0) goto _err; + if (tsdbOpenFile(helperIdxF(pHelper), O_RDONLY) < 0) goto _err; + if (tsdbOpenFile(helperHeadF(pHelper), O_RDONLY) < 0) goto _err; if (helperType(pHelper) == TSDB_WRITE_HELPER) { - if (tsdbOpenFile(&(pHelper->files.dataF), O_RDWR) < 0) goto _err; - if (tsdbOpenFile(&(pHelper->files.lastF), O_RDWR) < 0) goto _err; + if (tsdbOpenFile(helperDataF(pHelper), O_RDWR) < 0) goto _err; + if (tsdbOpenFile(helperLastF(pHelper), O_RDWR) < 0) goto _err; + + // Create and open .i file + if (tsdbOpenFile(helperNewIdxF(pHelper), O_WRONLY | O_CREAT) < 0) return -1; + if (tsdbUpdateFileHeader(helperNewIdxF(pHelper), 0) < 0) return -1; // Create and open .h - if (tsdbOpenFile(&(pHelper->files.nHeadF), O_WRONLY | O_CREAT) < 0) return -1; - // size_t tsize = TSDB_FILE_HEAD_SIZE + sizeof(SCompIdx) * pCfg->maxTables + sizeof(TSCKSUM); - if (tsendfile(pHelper->files.nHeadF.fd, pHelper->files.headF.fd, NULL, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) { - tsdbError("vgId:%d failed to sendfile %d bytes from file %s to %s since %s", REPO_ID(pHelper->pRepo), - TSDB_FILE_HEAD_SIZE, pHelper->files.headF.fname, pHelper->files.nHeadF.fname, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - goto _err; - } + if (tsdbOpenFile(helperNewHeadF(pHelper), O_WRONLY | O_CREAT) < 0) return -1; + if (tsdbUpdateFileHeader(helperNewHeadF(pHelper), 0) < 0) return -1; // Create and open .l file if should if (tsdbShouldCreateNewLast(pHelper)) { - if (tsdbOpenFile(&(pHelper->files.nLastF), O_WRONLY | O_CREAT) < 0) goto _err; - if (tsendfile(pHelper->files.nLastF.fd, pHelper->files.lastF.fd, NULL, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) { - tsdbError("vgId:%d failed to sendfile %d bytes from file %s to %s since %s", REPO_ID(pHelper->pRepo), - TSDB_FILE_HEAD_SIZE, pHelper->files.lastF.fname, pHelper->files.nLastF.fname, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - goto _err; - } + if (tsdbOpenFile(helperNewLastF(pHelper), O_WRONLY | O_CREAT) < 0) goto _err; + if (tsdbUpdateFileHeader(helperNewLastF(pHelper), 0) < 0) return -1; } } else { - if (tsdbOpenFile(&(pHelper->files.dataF), O_RDONLY) < 0) goto _err; - if (tsdbOpenFile(&(pHelper->files.lastF), O_RDONLY) < 0) goto _err; + if (tsdbOpenFile(helperDataF(pHelper), O_RDONLY) < 0) goto _err; + if (tsdbOpenFile(helperLastF(pHelper), O_RDONLY) < 0) goto _err; } helperSetState(pHelper, TSDB_HELPER_FILE_SET_AND_OPEN); @@ -155,59 +147,94 @@ _err: } int tsdbCloseHelperFile(SRWHelper *pHelper, bool hasError) { - if (pHelper->files.headF.fd > 0) { - close(pHelper->files.headF.fd); - pHelper->files.headF.fd = -1; + SFile *pFile = NULL; + + pFile = helperIdxF(pHelper); + if (pFile->fd > 0) { + close(pFile->fd); + pFile->fd = -1; } - if (pHelper->files.dataF.fd > 0) { + + pFile = helperHeadF(pHelper); + if (pFile->fd > 0) { + close(pFile->fd); + pFile->fd = -1; + } + + pFile = helperDataF(pHelper); + if (pFile->fd > 0) { if (helperType(pHelper) == TSDB_WRITE_HELPER) { - tsdbUpdateFileHeader(&(pHelper->files.dataF), 0); - fsync(pHelper->files.dataF.fd); + tsdbUpdateFileHeader(pFile, 0); + fsync(pFile->fd); } - close(pHelper->files.dataF.fd); - pHelper->files.dataF.fd = -1; + close(pFile->fd); + pFile->fd = -1; } - if (pHelper->files.lastF.fd > 0) { - if (helperType(pHelper) == TSDB_WRITE_HELPER) { - fsync(pHelper->files.lastF.fd); + + pFile = helperLastF(pHelper); + if (pFile->fd > 0) { + if (helperType(pHelper) == TSDB_WRITE_HELPER && !TSDB_NLAST_FILE_OPENED(pHelper)) { + fsync(pFile->fd); } - close(pHelper->files.lastF.fd); - pHelper->files.lastF.fd = -1; + close(pFile->fd); + pFile->fd = -1; } + if (helperType(pHelper) == TSDB_WRITE_HELPER) { - if (pHelper->files.nHeadF.fd > 0) { - if (!hasError) tsdbUpdateFileHeader(&(pHelper->files.nHeadF), 0); - fsync(pHelper->files.nHeadF.fd); - close(pHelper->files.nHeadF.fd); - pHelper->files.nHeadF.fd = -1; + pFile = helperNewIdxF(pHelper); + if (pFile->fd > 0) { + if (!hasError) tsdbUpdateFileHeader(pFile, 0); + fsync(pFile->fd); + close(pFile->fd); + pFile->fd = -1; if (hasError) { - (void)remove(pHelper->files.nHeadF.fname); + (void)remove(pFile->fname); } else { - if (rename(pHelper->files.nHeadF.fname, pHelper->files.headF.fname) < 0) { - tsdbError("failed to rename file from %s to %s since %s", pHelper->files.nHeadF.fname, - pHelper->files.headF.fname, strerror(errno)); + if (rename(pFile->fname, helperIdxF(pHelper)->fname) < 0) { + tsdbError("failed to rename file from %s to %s since %s", pFile->fname, helperIdxF(pHelper)->fname, + strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); return -1; } - pHelper->files.headF.info = pHelper->files.nHeadF.info; + helperIdxF(pHelper)->info = pFile->info; } } - if (pHelper->files.nLastF.fd > 0) { - if (!hasError) tsdbUpdateFileHeader(&(pHelper->files.nLastF), 0); - fsync(pHelper->files.nLastF.fd); - close(pHelper->files.nLastF.fd); - pHelper->files.nLastF.fd = -1; + pFile = helperNewHeadF(pHelper); + if (pFile->fd > 0) { + if (!hasError) tsdbUpdateFileHeader(pFile, 0); + fsync(pFile->fd); + close(pFile->fd); + pFile->fd = -1; if (hasError) { - (void)remove(pHelper->files.nLastF.fname); + (void)remove(pFile->fname); } else { - if (rename(pHelper->files.nLastF.fname, pHelper->files.lastF.fname) < 0) { - tsdbError("failed to rename file from %s to %s since %s", pHelper->files.nLastF.fname, - pHelper->files.lastF.fname, strerror(errno)); + if (rename(pFile->fname, helperHeadF(pHelper)->fname) < 0) { + tsdbError("failed to rename file from %s to %s since %s", pFile->fname, helperHeadF(pHelper)->fname, + strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); return -1; } - pHelper->files.lastF.info = pHelper->files.nLastF.info; + helperHeadF(pHelper)->info = pFile->info; + } + } + + pFile = helperNewLastF(pHelper); + if (pFile->fd > 0) { + if (!hasError) tsdbUpdateFileHeader(pFile, 0); + fsync(pFile->fd); + close(pFile->fd); + pFile->fd = -1; + if (hasError) { + (void)remove(pFile->fname); + } else { + if (rename(pFile->fname, helperLastF(pHelper)->fname) < 0) { + tsdbError("failed to rename file from %s to %s since %s", pFile->fname, helperLastF(pHelper)->fname, + strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + helperLastF(pHelper)->info = helperNewLastF(pHelper)->info; } } } @@ -283,28 +310,28 @@ int tsdbMoveLastBlockIfNeccessary(SRWHelper *pHelper) { if (tsdbLoadBlockData(pHelper, pCompBlock, NULL) < 0) return -1; ASSERT(pHelper->pDataCols[0]->numOfRows == pCompBlock->numOfRows && pHelper->pDataCols[0]->numOfRows < pCfg->minRowsPerFileBlock); - if (tsdbWriteBlockToFile(pHelper, &(pHelper->files.nLastF), pHelper->pDataCols[0], &compBlock, true, true) < 0) + if (tsdbWriteBlockToFile(pHelper, helperNewLastF(pHelper), pHelper->pDataCols[0], &compBlock, true, true) < 0) return -1; if (tsdbUpdateSuperBlock(pHelper, &compBlock, pIdx->numOfBlocks - 1) < 0) return -1; } else { - if (lseek(pHelper->files.lastF.fd, pCompBlock->offset, SEEK_SET) < 0) { - tsdbError("vgId:%d failed to lseek file %s since %s", REPO_ID(pHelper->pRepo), pHelper->files.lastF.fname, + if (lseek(helperLastF(pHelper)->fd, pCompBlock->offset, SEEK_SET) < 0) { + tsdbError("vgId:%d failed to lseek file %s since %s", REPO_ID(pHelper->pRepo), helperLastF(pHelper)->fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); return -1; } - pCompBlock->offset = lseek(pHelper->files.nLastF.fd, 0, SEEK_END); + pCompBlock->offset = lseek(helperNewLastF(pHelper)->fd, 0, SEEK_END); if (pCompBlock->offset < 0) { - tsdbError("vgId:%d failed to lseek file %s since %s", REPO_ID(pHelper->pRepo), pHelper->files.nLastF.fname, + tsdbError("vgId:%d failed to lseek file %s since %s", REPO_ID(pHelper->pRepo), helperNewLastF(pHelper)->fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); return -1; } - if (tsendfile(pHelper->files.nLastF.fd, pHelper->files.lastF.fd, NULL, pCompBlock->len) < pCompBlock->len) { + if (tsendfile(helperNewLastF(pHelper)->fd, helperLastF(pHelper)->fd, NULL, pCompBlock->len) < pCompBlock->len) { tsdbError("vgId:%d failed to sendfile from file %s to file %s since %s", REPO_ID(pHelper->pRepo), - pHelper->files.lastF.fname, pHelper->files.nLastF.fname, strerror(errno)); + helperLastF(pHelper)->fname, helperNewLastF(pHelper)->fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -321,9 +348,9 @@ int tsdbWriteCompInfo(SRWHelper *pHelper) { SCompIdx *pIdx = pHelper->pCompIdx + pHelper->tableInfo.tid; if (!helperHasState(pHelper, TSDB_HELPER_INFO_LOAD)) { if (pIdx->offset > 0) { - offset = lseek(pHelper->files.nHeadF.fd, 0, SEEK_END); + offset = lseek(helperNewHeadF(pHelper)->fd, 0, SEEK_END); if (offset < 0) { - tsdbError("vgId:%d failed to lseed file %s since %s", REPO_ID(pHelper->pRepo), pHelper->files.nHeadF.fname, + tsdbError("vgId:%d failed to lseed file %s since %s", REPO_ID(pHelper->pRepo), helperNewHeadF(pHelper)->fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -332,9 +359,9 @@ int tsdbWriteCompInfo(SRWHelper *pHelper) { pIdx->offset = offset; ASSERT(pIdx->offset >= TSDB_FILE_HEAD_SIZE); - if (tsendfile(pHelper->files.nHeadF.fd, pHelper->files.headF.fd, NULL, pIdx->len) < pIdx->len) { + if (tsendfile(helperNewHeadF(pHelper)->fd, helperHeadF(pHelper)->fd, NULL, pIdx->len) < pIdx->len) { tsdbError("vgId:%d failed to send %d bytes from file %s to %s since %s", REPO_ID(pHelper->pRepo), pIdx->len, - pHelper->files.headF.fname, pHelper->files.nHeadF.fname, strerror(errno)); + helperHeadF(pHelper)->fname, helperNewHeadF(pHelper)->fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -347,9 +374,9 @@ int tsdbWriteCompInfo(SRWHelper *pHelper) { ASSERT(pIdx->len > sizeof(SCompInfo) + sizeof(TSCKSUM) && (pIdx->len - sizeof(SCompInfo) - sizeof(TSCKSUM)) % sizeof(SCompBlock) == 0); taosCalcChecksumAppend(0, (uint8_t *)pHelper->pCompInfo, pIdx->len); - offset = lseek(pHelper->files.nHeadF.fd, 0, SEEK_END); + offset = lseek(helperNewHeadF(pHelper)->fd, 0, SEEK_END); if (offset < 0) { - tsdbError("vgId:%d failed to lseek file %s since %s", REPO_ID(pHelper->pRepo), pHelper->files.nHeadF.fname, + tsdbError("vgId:%d failed to lseek file %s since %s", REPO_ID(pHelper->pRepo), helperNewHeadF(pHelper)->fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -358,9 +385,9 @@ int tsdbWriteCompInfo(SRWHelper *pHelper) { pIdx->uid = pHelper->tableInfo.uid; ASSERT(pIdx->offset >= TSDB_FILE_HEAD_SIZE); - if (twrite(pHelper->files.nHeadF.fd, (void *)(pHelper->pCompInfo), pIdx->len) < pIdx->len) { + if (twrite(helperNewHeadF(pHelper)->fd, (void *)(pHelper->pCompInfo), pIdx->len) < pIdx->len) { tsdbError("vgId:%d failed to write %d bytes to file %s since %s", REPO_ID(pHelper->pRepo), pIdx->len, - pHelper->files.nHeadF.fname, strerror(errno)); + helperNewHeadF(pHelper)->fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -371,19 +398,10 @@ int tsdbWriteCompInfo(SRWHelper *pHelper) { } int tsdbWriteCompIdx(SRWHelper *pHelper) { + ASSERT(helperType(pHelper) == TSDB_WRITE_HELPER); STsdbCfg *pCfg = &pHelper->pRepo->config; - ASSERT(helperType(pHelper) == TSDB_WRITE_HELPER); - off_t offset = lseek(pHelper->files.nHeadF.fd, 0, SEEK_END); - if (offset < 0) { - tsdbError("vgId:%d failed to lseek file %s to end since %s", REPO_ID(pHelper->pRepo), pHelper->files.nHeadF.fname, - strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - SFile *pFile = &(pHelper->files.nHeadF); - pFile->info.offset = offset; + SFile *pFile = helperNewIdxF(pHelper); void *buf = pHelper->pBuffer; for (uint32_t i = 0; i < pCfg->maxTables; i++) { @@ -406,9 +424,9 @@ int tsdbWriteCompIdx(SRWHelper *pHelper) { int tsize = (char *)buf - (char *)pHelper->pBuffer + sizeof(TSCKSUM); taosCalcChecksumAppend(0, (uint8_t *)pHelper->pBuffer, tsize); - if (twrite(pHelper->files.nHeadF.fd, (void *)pHelper->pBuffer, tsize) < tsize) { - tsdbError("vgId:%d failed to write %d bytes to file %s since %s", REPO_ID(pHelper->pRepo), tsize, - pHelper->files.nHeadF.fname, strerror(errno)); + if (twrite(pFile->fd, (void *)pHelper->pBuffer, tsize) < tsize) { + tsdbError("vgId:%d failed to write %d bytes to file %s since %s", REPO_ID(pHelper->pRepo), tsize, pFile->fname, + strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -423,23 +441,21 @@ int tsdbLoadCompIdx(SRWHelper *pHelper, void *target) { if (!helperHasState(pHelper, TSDB_HELPER_IDX_LOAD)) { // If not load from file, just load it in object - SFile *pFile = &(pHelper->files.headF); + SFile *pFile = helperIdxF(pHelper); int fd = pFile->fd; memset(pHelper->pCompIdx, 0, tsizeof(pHelper->pCompIdx)); - if (pFile->info.offset > 0) { - ASSERT(pFile->info.offset > TSDB_FILE_HEAD_SIZE); - - if (lseek(fd, pFile->info.offset, SEEK_SET) < 0) { - tsdbError("vgId:%d failed to lseek file %s to %u since %s", REPO_ID(pHelper->pRepo), pFile->fname, - pFile->info.offset, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } + if (pFile->info.len > 0) { if ((pHelper->pBuffer = trealloc(pHelper->pBuffer, pFile->info.len)) == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; return -1; } + if (lseek(fd, TSDB_FILE_HEAD_SIZE, SEEK_SET) < 0) { + tsdbError("vgId:%d failed to lseek file %s since %s", REPO_ID(pHelper->pRepo), pFile->fname, strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + if (tread(fd, (void *)(pHelper->pBuffer), pFile->info.len) < pFile->info.len) { tsdbError("vgId:%d failed to read %d bytes from file %s since %s", REPO_ID(pHelper->pRepo), pFile->info.len, pFile->fname, strerror(errno)); @@ -447,8 +463,8 @@ int tsdbLoadCompIdx(SRWHelper *pHelper, void *target) { return -1; } if (!taosCheckChecksumWhole((uint8_t *)(pHelper->pBuffer), pFile->info.len)) { - tsdbError("vgId:%d file %s SCompIdx part is corrupted. offset %u len %u", REPO_ID(pHelper->pRepo), pFile->fname, - pFile->info.offset, pFile->info.len); + tsdbError("vgId:%d file %s SCompIdx part is corrupted. len %u", REPO_ID(pHelper->pRepo), pFile->fname, + pFile->info.len); terrno = TSDB_CODE_TDB_FILE_CORRUPTED; return -1; } @@ -484,13 +500,13 @@ int tsdbLoadCompInfo(SRWHelper *pHelper, void *target) { SCompIdx *pIdx = pHelper->pCompIdx + pHelper->tableInfo.tid; - int fd = pHelper->files.headF.fd; + int fd = helperHeadF(pHelper)->fd; if (!helperHasState(pHelper, TSDB_HELPER_INFO_LOAD)) { if (pIdx->offset > 0) { ASSERT(pIdx->uid == pHelper->tableInfo.uid); if (lseek(fd, pIdx->offset, SEEK_SET) < 0) { - tsdbError("vgId:%d failed to lseek file %s since %s", REPO_ID(pHelper->pRepo), pHelper->files.headF.fname, + tsdbError("vgId:%d failed to lseek file %s since %s", REPO_ID(pHelper->pRepo), helperHeadF(pHelper)->fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -499,13 +515,13 @@ int tsdbLoadCompInfo(SRWHelper *pHelper, void *target) { pHelper->pCompInfo = trealloc((void *)pHelper->pCompInfo, pIdx->len); if (tread(fd, (void *)(pHelper->pCompInfo), pIdx->len) < pIdx->len) { tsdbError("vgId:%d failed to read %d bytes from file %s since %s", REPO_ID(pHelper->pRepo), pIdx->len, - pHelper->files.headF.fname, strerror(errno)); + helperHeadF(pHelper)->fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); return -1; } if (!taosCheckChecksumWhole((uint8_t *)pHelper->pCompInfo, pIdx->len)) { tsdbError("vgId:%d file %s SCompInfo part is corrupted, tid %d uid %" PRIu64, REPO_ID(pHelper->pRepo), - pHelper->files.headF.fname, pHelper->tableInfo.tid, pHelper->tableInfo.uid); + helperHeadF(pHelper)->fname, pHelper->tableInfo.tid, pHelper->tableInfo.uid); terrno = TSDB_CODE_TDB_FILE_CORRUPTED; return -1; } @@ -523,7 +539,7 @@ int tsdbLoadCompInfo(SRWHelper *pHelper, void *target) { int tsdbLoadCompData(SRWHelper *pHelper, SCompBlock *pCompBlock, void *target) { ASSERT(pCompBlock->numOfSubBlocks <= 1); - SFile *pFile = (pCompBlock->last) ? &(pHelper->files.lastF) : &(pHelper->files.dataF); + SFile *pFile = (pCompBlock->last) ? helperLastF(pHelper) : helperDataF(pHelper); if (lseek(pFile->fd, pCompBlock->offset, SEEK_SET) < 0) { tsdbError("vgId:%d failed to lseek file %s since %s", REPO_ID(pHelper->pRepo), pFile->fname, strerror(errno)); @@ -642,9 +658,9 @@ _err: // ---------------------- INTERNAL FUNCTIONS ---------------------- static bool tsdbShouldCreateNewLast(SRWHelper *pHelper) { - ASSERT(pHelper->files.lastF.fd > 0); + ASSERT(helperLastF(pHelper)->fd > 0); struct stat st; - if (fstat(pHelper->files.lastF.fd, &st) < 0) return true; + if (fstat(helperLastF(pHelper)->fd, &st) < 0) return true; if (st.st_size > 32 * 1024 + TSDB_FILE_HEAD_SIZE) return true; return false; } @@ -972,12 +988,13 @@ static int tsdbUpdateSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int static void tsdbResetHelperFileImpl(SRWHelper *pHelper) { memset((void *)&pHelper->files, 0, sizeof(pHelper->files)); - pHelper->files.fid = -1; - pHelper->files.headF.fd = -1; - pHelper->files.dataF.fd = -1; - pHelper->files.lastF.fd = -1; - pHelper->files.nHeadF.fd = -1; - pHelper->files.nLastF.fd = -1; + helperIdxF(pHelper)->fd = -1; + helperHeadF(pHelper)->fd = -1; + helperDataF(pHelper)->fd = -1; + helperLastF(pHelper)->fd = -1; + helperNewIdxF(pHelper)->fd = -1; + helperNewHeadF(pHelper)->fd = -1; + helperNewLastF(pHelper)->fd = -1; } static int tsdbInitHelperFile(SRWHelper *pHelper) { @@ -1154,7 +1171,7 @@ static int tsdbLoadBlockDataColsImpl(SRWHelper *pHelper, SCompBlock *pCompBlock, ASSERT(pCompBlock->numOfSubBlocks <= 1); ASSERT(colIds[0] == 0); - SFile * pFile = (pCompBlock->last) ? &(pHelper->files.lastF) : &(pHelper->files.dataF); + SFile * pFile = (pCompBlock->last) ? helperLastF(pHelper) : helperDataF(pHelper); SCompCol compCol = {0}; // If only load timestamp column, no need to load SCompData part @@ -1215,7 +1232,7 @@ _err: static int tsdbLoadBlockDataImpl(SRWHelper *pHelper, SCompBlock *pCompBlock, SDataCols *pDataCols) { ASSERT(pCompBlock->numOfSubBlocks <= 1); - SFile *pFile = (pCompBlock->last) ? &(pHelper->files.lastF) : &(pHelper->files.dataF); + SFile *pFile = (pCompBlock->last) ? helperLastF(pHelper) : helperDataF(pHelper); pHelper->pBuffer = trealloc(pHelper->pBuffer, pCompBlock->len); if (pHelper->pBuffer == NULL) { @@ -1362,7 +1379,7 @@ static int tsdbProcessAppendCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, ASSERT(rowsRead > 0 && rowsRead == pDataCols->numOfRows); if (rowsRead + pCompBlock->numOfRows < pCfg->minRowsPerFileBlock && pCompBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS && !TSDB_NLAST_FILE_OPENED(pHelper)) { - if (tsdbWriteBlockToFile(pHelper, &(pHelper->files.lastF), pDataCols, &compBlock, true, false) < 0) return -1; + if (tsdbWriteBlockToFile(pHelper, helperLastF(pHelper), pDataCols, &compBlock, true, false) < 0) return -1; if (tsdbAddSubBlock(pHelper, &compBlock, pIdx->numOfBlocks - 1, rowsRead) < 0) return -1; } else { if (tsdbLoadBlockData(pHelper, pCompBlock, NULL) < 0) return -1; @@ -1427,7 +1444,7 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, int rowsRead = tsdbLoadDataFromCache(pTable, pCommitIter->pIter, maxKey, rows1, pDataCols, pDataCols0->cols[0].pData, pDataCols0->numOfRows); ASSERT(rowsRead == rows2 && rowsRead == pDataCols->numOfRows); - if (tsdbWriteBlockToFile(pHelper, &(pHelper->files.lastF), pDataCols, &compBlock, true, false) < 0) return -1; + if (tsdbWriteBlockToFile(pHelper, helperLastF(pHelper), pDataCols, &compBlock, true, false) < 0) return -1; if (tsdbAddSubBlock(pHelper, &compBlock, tblkIdx, rowsRead) < 0) return -1; tblkIdx++; } else { @@ -1466,7 +1483,7 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, if (rowsRead == 0) break; ASSERT(rowsRead == pDataCols->numOfRows); - if (tsdbWriteBlockToFile(pHelper, &(pHelper->files.dataF), pDataCols, &compBlock, false, true) < 0) return -1; + if (tsdbWriteBlockToFile(pHelper, helperDataF(pHelper), pDataCols, &compBlock, false, true) < 0) return -1; if (tsdbInsertSuperBlock(pHelper, &compBlock, tblkIdx) < 0) return -1; tblkIdx++; } @@ -1493,7 +1510,7 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, int rowsRead = tsdbLoadDataFromCache(pTable, pCommitIter->pIter, keyLimit, rows, pDataCols, pDataCols0->cols[0].pData, pDataCols0->numOfRows); ASSERT(rowsRead == rows && rowsRead == pDataCols->numOfRows); - if (tsdbWriteBlockToFile(pHelper, &(pHelper->files.dataF), pDataCols, &compBlock, false, false) < 0) + if (tsdbWriteBlockToFile(pHelper, helperDataF(pHelper), pDataCols, &compBlock, false, false) < 0) return -1; if (tsdbAddSubBlock(pHelper, &compBlock, tblkIdx, rowsRead) < 0) return -1; tblkIdx++; @@ -1506,7 +1523,7 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, tsdbLoadAndMergeFromCache(pDataCols0, &dIter, pCommitIter, pDataCols, keyLimit, defaultRowsInBlock); if (rowsRead == 0) break; - if (tsdbWriteBlockToFile(pHelper, &(pHelper->files.dataF), pDataCols, &compBlock, false, true) < 0) + if (tsdbWriteBlockToFile(pHelper, helperDataF(pHelper), pDataCols, &compBlock, false, true) < 0) return -1; if (round == 0) { if (tsdbUpdateSuperBlock(pHelper, &compBlock, tblkIdx) < 0) return -1; @@ -1577,10 +1594,10 @@ static int tsdbWriteBlockToProperFile(SRWHelper *pHelper, SDataCols *pDataCols, ASSERT(pDataCols->numOfRows > 0); if (pDataCols->numOfRows >= pCfg->minRowsPerFileBlock) { - pFile = &(pHelper->files.dataF); + pFile = helperDataF(pHelper); } else { isLast = true; - pFile = TSDB_NLAST_FILE_OPENED(pHelper) ? &(pHelper->files.nLastF) : &(pHelper->files.lastF); + pFile = TSDB_NLAST_FILE_OPENED(pHelper) ? helperNewLastF(pHelper) : helperLastF(pHelper); } ASSERT(pFile->fd > 0); From 5f55393464f57939a17043d3a4a23d3f81ad8353 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 17 Jul 2020 21:54:16 +0000 Subject: [PATCH 006/100] fixbug --- src/client/src/tscServer.c | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index c82773a125..ced6914df2 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -48,7 +48,7 @@ void tscIpSetCopy(SRpcIpSet *dst, SRpcIpSet *src) { dst->numOfIps = src->numOfIps; dst->inUse = src->inUse; for (int32_t i = 0; i < src->numOfIps; ++i) { - dst->port[i] = htons(dst->port[i]); + dst->port[i] = src->port[i]; strncpy(dst->fqdn[i], src->fqdn[i], TSDB_FQDN_LEN); } } @@ -58,7 +58,11 @@ static void tscDumpMgmtIpSet(SRpcIpSet *ipSet) { tscIpSetCopy(ipSet, src); taosCorEndRead(&tscMgmtIpSet.version); } - +static void tscIpSetHtons(SRpcIpSet *s) { + for (int32_t i = 0; i < s->numOfIps; i++) { + s->port[i] = htons(s->port[i]); + } +} bool tscIpSetIsEqual(SRpcIpSet *s1, SRpcIpSet *s2) { if (s1->numOfIps != s2->numOfIps /*|| s1->inUse != s1->inUse*/) { return false; @@ -149,6 +153,7 @@ void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) { SCMHeartBeatRsp *pRsp = (SCMHeartBeatRsp *)pRes->pRsp; SRpcIpSet * pIpList = &pRsp->ipList; if (pIpList->numOfIps > 0) + tscIpSetHtons(pIpList); tscUpdateMgmtIpList(pIpList); pSql->pTscObj->connId = htonl(pRsp->connId); @@ -275,13 +280,15 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcIpSet *pIpSet) { return; } - if (pIpSet) { - if (!tscIpSetIsEqual(&pSql->ipList, pIpSet)) { - if (pCmd->command < TSDB_SQL_MGMT) { + if (pIpSet) { + //SRpcIpSet dump; + tscIpSetHtons(pIpSet); + if (tscIpSetIsEqual(&pSql->ipList, pIpSet)) { + if(pCmd->command < TSDB_SQL_MGMT) { tscUpdateVgroupInfo(pSql, pIpSet); } else { tscUpdateMgmtIpList(pIpSet); - } + } } } @@ -464,10 +471,8 @@ int tscProcessSql(SSqlObj *pSql) { return pSql->res.code; } } else if (pCmd->command < TSDB_SQL_LOCAL) { - SRpcIpSet dump; - tscDumpMgmtIpSet(&dump); - pSql->ipList = dump; - } else { // local handler + tscDumpMgmtIpSet(&pSql->ipList); + } else { return (*tscProcessMsgRsp[pCmd->command])(pSql); } @@ -1991,6 +1996,7 @@ int tscProcessConnectRsp(SSqlObj *pSql) { tstrncpy(pObj->db, temp, sizeof(pObj->db)); if (pConnect->ipList.numOfIps > 0) + tscIpSetHtons(&pConnect->ipList); tscUpdateMgmtIpList(&pConnect->ipList); strcpy(pObj->sversion, pConnect->serverVersion); From d2823d60e2bc30dffa04970bf970de53761f076a Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 17 Jul 2020 22:17:23 +0000 Subject: [PATCH 007/100] modify code style --- src/client/src/tscServer.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index ced6914df2..b162387685 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -226,10 +226,7 @@ int tscSendMsgToServer(SSqlObj *pSql) { // set the mgmt ip list if (pSql->cmd.command >= TSDB_SQL_MGMT) { - SRpcIpSet dump; - tscDumpMgmtIpSet(&dump); - // no need to update pSql->ipList - tscIpSetCopy(&pSql->ipList, &dump); + tscDumpMgmtIpSet(&pSql->ipList); } memcpy(pMsg, pSql->cmd.payload, pSql->cmd.payloadLen); @@ -471,7 +468,7 @@ int tscProcessSql(SSqlObj *pSql) { return pSql->res.code; } } else if (pCmd->command < TSDB_SQL_LOCAL) { - tscDumpMgmtIpSet(&pSql->ipList); + //tscDumpMgmtIpSet(&pSql->ipList); } else { return (*tscProcessMsgRsp[pCmd->command])(pSql); } From 65b32c3855ecfd7caf2aef977ada081e5e3b1b66 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 17 Jul 2020 22:35:57 +0000 Subject: [PATCH 008/100] modify code style --- src/client/src/tscServer.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index b162387685..3b3884eb07 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -100,6 +100,7 @@ static void tscDumpIpSetFromVgroupInfo(SCMVgroupInfo *pVgroupInfo, SRpcIpSet *pI static void tscUpdateVgroupInfo(SSqlObj *pObj, SRpcIpSet *pIpSet) { SSqlCmd *pCmd = &pObj->cmd; STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); + if (pTableMetaInfo == NULL || pTableMetaInfo->pTableMeta == NULL) { return;} SCMVgroupInfo *pVgroupInfo = &pTableMetaInfo->pTableMeta->vgroupInfo; taosCorBeginWrite(&pVgroupInfo->version); From 0ac592fbfd44686a875629a93760d6e48e24b8bf Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sat, 18 Jul 2020 01:04:02 +0000 Subject: [PATCH 009/100] add inuse to vgroupinfo --- src/client/src/tscServer.c | 9 ++++++--- src/inc/taosmsg.h | 1 + 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 3b3884eb07..d0c9d3636b 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -64,7 +64,7 @@ static void tscIpSetHtons(SRpcIpSet *s) { } } bool tscIpSetIsEqual(SRpcIpSet *s1, SRpcIpSet *s2) { - if (s1->numOfIps != s2->numOfIps /*|| s1->inUse != s1->inUse*/) { + if (s1->numOfIps != s2->numOfIps || s1->inUse != s2->inUse) { return false; } for (int32_t i = 0; i < s1->numOfIps; i++) { @@ -88,7 +88,8 @@ static void tscDumpIpSetFromVgroupInfo(SCMVgroupInfo *pVgroupInfo, SRpcIpSet *pI if (vgId) { *vgId = pVgroupInfo->vgId; } - pIpSet->inUse = 0; + int8_t inUse = pVgroupInfo->inUse; + pIpSet->inUse = (inUse >= 0 && inUse < TSDB_MAX_REPLICA) ? inUse: 0; pIpSet->numOfIps = pVgroupInfo->numOfIps; for (int32_t i = 0; i < pVgroupInfo->numOfIps; ++i) { strncpy(pIpSet->fqdn[i], pVgroupInfo->ipAddr[i].fqdn, TSDB_FQDN_LEN); @@ -104,7 +105,8 @@ static void tscUpdateVgroupInfo(SSqlObj *pObj, SRpcIpSet *pIpSet) { SCMVgroupInfo *pVgroupInfo = &pTableMetaInfo->pTableMeta->vgroupInfo; taosCorBeginWrite(&pVgroupInfo->version); - //TODO(dengyihao), dont care vgid + //TODO(dengyihao), dont care vgid + pVgroupInfo->inUse = pIpSet->inUse; pVgroupInfo->numOfIps = pIpSet->numOfIps; for (int32_t i = 0; pVgroupInfo->numOfIps; i++) { strncpy(pVgroupInfo->ipAddr[i].fqdn, pIpSet->fqdn[i], TSDB_FQDN_LEN); @@ -1897,6 +1899,7 @@ int tscProcessSTableVgroupRsp(SSqlObj *pSql) { //just init, no need to lock SCMVgroupInfo *pVgroups = &pInfo->vgroupList->vgroups[j]; pVgroups->version = 0; + pVgroups->inUse = 0; pVgroups->vgId = htonl(pVgroups->vgId); assert(pVgroups->numOfIps >= 1); for (int32_t k = 0; k < pVgroups->numOfIps; ++k) { diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index c12aed2bcc..fec3d77d8e 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -649,6 +649,7 @@ typedef struct SCMSTableVgroupMsg { typedef struct { int32_t version; int32_t vgId; + int8_t inUse; int8_t numOfIps; SIpAddr ipAddr[TSDB_MAX_REPLICA]; } SCMVgroupInfo; From f569bb4babb1341c4128e24cfe2c20005cca011e Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Fri, 17 Jul 2020 14:27:02 +0800 Subject: [PATCH 010/100] td-909: fix subscription bugs --- src/client/src/tscSub.c | 164 ++++++++++++++++++++++---------------- src/query/src/qExecutor.c | 2 +- 2 files changed, 97 insertions(+), 69 deletions(-) diff --git a/src/client/src/tscSub.c b/src/client/src/tscSub.c index 2c5035c2ef..73614fa0a5 100644 --- a/src/client/src/tscSub.c +++ b/src/client/src/tscSub.c @@ -34,6 +34,7 @@ typedef struct SSubscriptionProgress { typedef struct SSub { void * signature; char topic[32]; + sem_t sem; int64_t lastSyncTime; int64_t lastConsumeTime; TAOS * taos; @@ -83,84 +84,107 @@ void tscUpdateSubscriptionProgress(void* sub, int64_t uid, TSKEY ts) { static void asyncCallback(void *param, TAOS_RES *tres, int code) { assert(param != NULL); - SSqlObj *pSql = ((SSqlObj *)param); - - pSql->res.code = code; - sem_post(&pSql->rspSem); + SSub *pSub = ((SSub *)param); + pSub->pSql->res.code = code; + sem_post(&pSub->sem); } static SSub* tscCreateSubscription(STscObj* pObj, const char* topic, const char* sql) { - SSub* pSub = NULL; + int code = TSDB_CODE_SUCCESS, line = __LINE__; - TRY( 8 ) { - SSqlObj* pSql = calloc_throw(1, sizeof(SSqlObj)); - CLEANUP_PUSH_FREE(true, pSql); - SSqlCmd *pCmd = &pSql->cmd; - SSqlRes *pRes = &pSql->res; + SSub* pSub = calloc(1, sizeof(SSub)); + if (pSub == NULL) { + line = __LINE__; + code = TSDB_CODE_TSC_OUT_OF_MEMORY; + goto fail; + } + pSub->signature = pSub; + if (tsem_init(&pSub->sem, 0, 0) == -1) { + line = __LINE__; + code = TAOS_SYSTEM_ERROR(errno); + goto fail; + } + tstrncpy(pSub->topic, topic, sizeof(pSub->topic)); + pSub->progress = taosArrayInit(32, sizeof(SSubscriptionProgress)); + if (pSub->progress == NULL) { + line = __LINE__; + code = TSDB_CODE_TSC_OUT_OF_MEMORY; + goto fail; + } - if (tsem_init(&pSql->rspSem, 0, 0) == -1) { - THROW(TAOS_SYSTEM_ERROR(errno)); - } - CLEANUP_PUSH_INT_PTR(true, tsem_destroy, &pSql->rspSem); + SSqlObj* pSql = calloc(1, sizeof(SSqlObj)); + if (pSql == NULL) { + line = __LINE__; + code = TSDB_CODE_TSC_OUT_OF_MEMORY; + goto fail; + } + pSql->signature = pSql; + pSql->pTscObj = pObj; + pSql->pSubscription = pSub; + pSub->pSql = pSql; - pSql->signature = pSql; - pSql->param = pSql; - pSql->pTscObj = pObj; - pSql->maxRetry = TSDB_MAX_REPLICA; - pSql->fp = asyncCallback; + SSqlCmd* pCmd = &pSql->cmd; + SSqlRes* pRes = &pSql->res; + if (tsem_init(&pSql->rspSem, 0, 0) == -1) { + line = __LINE__; + code = TAOS_SYSTEM_ERROR(errno); + goto fail; + } - int code = tscAllocPayload(pCmd, TSDB_DEFAULT_PAYLOAD_SIZE); - if (code != TSDB_CODE_SUCCESS) { - THROW(code); - } - CLEANUP_PUSH_FREE(true, pCmd->payload); + pSql->param = pSub; + pSql->maxRetry = TSDB_MAX_REPLICA; + pSql->fp = asyncCallback; + pSql->fetchFp = asyncCallback; + pSql->sqlstr = strdup(sql); + if (pSql->sqlstr == NULL) { + line = __LINE__; + code = TSDB_CODE_TSC_OUT_OF_MEMORY; + goto fail; + } + strtolower(pSql->sqlstr, pSql->sqlstr); + pRes->qhandle = 0; + pRes->numOfRows = 1; - pRes->qhandle = 0; - pRes->numOfRows = 1; + code = tscAllocPayload(pCmd, TSDB_DEFAULT_PAYLOAD_SIZE); + if (code != TSDB_CODE_SUCCESS) { + line = __LINE__; + goto fail; + } - pSql->sqlstr = strdup_throw(sql); - CLEANUP_PUSH_FREE(true, pSql->sqlstr); - strtolower(pSql->sqlstr, pSql->sqlstr); + code = tsParseSql(pSql, false); + if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { + sem_wait(&pSub->sem); + code = pSql->res.code; + } + if (code != TSDB_CODE_SUCCESS) { + line = __LINE__; + goto fail; + } - code = tsParseSql(pSql, false); - if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { - // wait for the callback function to post the semaphore - sem_wait(&pSql->rspSem); - code = pSql->res.code; - } - if (code != TSDB_CODE_SUCCESS) { - tscError("failed to parse sql statement: %s, error: %s", pSub->topic, tstrerror(code)); - THROW( code ); - } - - if (pSql->cmd.command != TSDB_SQL_SELECT) { - tscError("only 'select' statement is allowed in subscription: %s", pSub->topic); - THROW( -1 ); // TODO - } - - pSub = calloc_throw(1, sizeof(SSub)); - CLEANUP_PUSH_FREE(true, pSub); - pSql->pSubscription = pSub; - pSub->pSql = pSql; - pSub->signature = pSub; - strncpy(pSub->topic, topic, sizeof(pSub->topic)); - pSub->topic[sizeof(pSub->topic) - 1] = 0; - pSub->progress = taosArrayInit(32, sizeof(SSubscriptionProgress)); - if (pSub->progress == NULL) { - THROW(TSDB_CODE_TSC_OUT_OF_MEMORY); - } - - CLEANUP_EXECUTE(); - - } CATCH( code ) { - tscError("failed to create subscription object: %s", tstrerror(code)); - CLEANUP_EXECUTE(); - pSub = NULL; - - } END_TRY + if (pSql->cmd.command != TSDB_SQL_SELECT) { + line = __LINE__; + code = TSDB_CODE_TSC_INVALID_SQL; + goto fail; + } return pSub; + +fail: + tscError("tscCreateSubscription failed at line %d, reason: %s", line, tstrerror(code)); + if (pSql != NULL) { + tscFreeSqlObj(pSql); + pSql = NULL; + } + if (pSub != NULL) { + taosArrayDestroy(pSub->progress); + tsem_destroy(&pSub->sem); + free(pSub); + pSub = NULL; + } + + terrno = code; + return NULL; } @@ -405,9 +429,10 @@ TAOS_RES *taos_consume(TAOS_SUB *tsub) { tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0)->vgroupIndex = 0; pSql->fp = asyncCallback; - pSql->param = pSql; + pSql->fetchFp = asyncCallback; + pSql->param = pSub; tscDoQuery(pSql); - sem_wait(&pSql->rspSem); + sem_wait(&pSub->sem); if (pRes->code != TSDB_CODE_SUCCESS) { continue; @@ -437,7 +462,9 @@ void taos_unsubscribe(TAOS_SUB *tsub, int keepProgress) { } if (keepProgress) { - tscSaveSubscriptionProgress(pSub); + if (pSub->progress != NULL) { + tscSaveSubscriptionProgress(pSub); + } } else { char path[256]; sprintf(path, "%s/subscribe/%s", tsDataDir, pSub->topic); @@ -448,6 +475,7 @@ void taos_unsubscribe(TAOS_SUB *tsub, int keepProgress) { tscFreeSqlObj(pSub->pSql); taosArrayDestroy(pSub->progress); + tsem_destroy(&pSub->sem); memset(pSub, 0, sizeof(*pSub)); free(pSub); } diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index cf13ace11f..3b10967e33 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -4918,7 +4918,7 @@ static void tableMultiOutputProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) pQuery->current->lastKey, pQuery->window.ekey); } else if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { STableIdInfo tidInfo; - STableId* id = TSDB_TABLEID(pQuery->current); + STableId* id = TSDB_TABLEID(pQuery->current->pTable); tidInfo.uid = id->uid; tidInfo.tid = id->tid; From f9cb99e6f79e0cab9a40ad4840c49f348f2392b8 Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Fri, 17 Jul 2020 15:47:20 +0800 Subject: [PATCH 011/100] td-909: make taos_validate_sql work --- src/client/src/tscSql.c | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index d5160cd3c6..49e4e027a0 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -724,6 +724,13 @@ int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields) return len; } +static void asyncCallback(void *param, TAOS_RES *tres, int code) { + assert(param != NULL); + SSqlObj *pSql = ((SSqlObj *)param); + pSql->res.code = code; + sem_post(&pSql->rspSem); +} + int taos_validate_sql(TAOS *taos, const char *sql) { STscObj *pObj = (STscObj *)taos; if (pObj == NULL || pObj->signature != pObj) { @@ -732,7 +739,8 @@ int taos_validate_sql(TAOS *taos, const char *sql) { } SSqlObj* pSql = calloc(1, sizeof(SSqlObj)); - + pSql->pTscObj = taos; + pSql->signature = pSql; SSqlRes *pRes = &pSql->res; SSqlCmd *pCmd = &pSql->cmd; @@ -766,10 +774,17 @@ int taos_validate_sql(TAOS *taos, const char *sql) { pCmd->pTableList = NULL; } - pRes->code = (uint8_t)tsParseSql(pSql, false); - int code = pRes->code; - - tscDebug("%p Valid SQL result:%d, %s pObj:%p", pSql, pRes->code, taos_errstr(taos), pObj); + pSql->fp = asyncCallback; + pSql->fetchFp = asyncCallback; + pSql->param = pSql; + int code = tsParseSql(pSql, true); + if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { + sem_wait(&pSql->rspSem); + code = pSql->res.code; + } + if (code != TSDB_CODE_SUCCESS) { + tscDebug("%p Valid SQL result:%d, %s pObj:%p", pSql, code, taos_errstr(taos), pObj); + } taos_free_result(pSql); return code; From efe558731b14053a04cbfb84ea1e1000fce0881b Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Fri, 17 Jul 2020 15:56:49 +0800 Subject: [PATCH 012/100] td-909: add verification for some API --- tests/examples/c/apitest.c | 394 +++++++++++++++++++++++++++++++++++++ 1 file changed, 394 insertions(+) create mode 100644 tests/examples/c/apitest.c diff --git a/tests/examples/c/apitest.c b/tests/examples/c/apitest.c new file mode 100644 index 0000000000..0be1f01bbe --- /dev/null +++ b/tests/examples/c/apitest.c @@ -0,0 +1,394 @@ +// sample code to verify all TDengine API +// to compile: gcc -o apitest apitest.c -ltaos + +#include +#include +#include +#include +#include + + +static void prepare_data(TAOS* taos) { + taos_query(taos, "drop database if exists test;"); + usleep(100000); + taos_query(taos, "create database test;"); + usleep(100000); + taos_select_db(taos, "test"); + + taos_query(taos, "create table meters(ts timestamp, a int) tags(area int);"); + + taos_query(taos, "create table t0 using meters tags(0);"); + taos_query(taos, "create table t1 using meters tags(1);"); + taos_query(taos, "create table t2 using meters tags(2);"); + taos_query(taos, "create table t3 using meters tags(3);"); + taos_query(taos, "create table t4 using meters tags(4);"); + taos_query(taos, "create table t5 using meters tags(5);"); + taos_query(taos, "create table t6 using meters tags(6);"); + taos_query(taos, "create table t7 using meters tags(7);"); + taos_query(taos, "create table t8 using meters tags(8);"); + taos_query(taos, "create table t9 using meters tags(9);"); + + TAOS_RES* res = taos_query(taos, "insert into t0 values('2020-01-01 00:00:00.000', 0)" + " ('2020-01-01 00:01:00.000', 0)" + " ('2020-01-01 00:02:00.000', 0)" + " t1 values('2020-01-01 00:00:00.000', 0)" + " ('2020-01-01 00:01:00.000', 0)" + " ('2020-01-01 00:02:00.000', 0)" + " ('2020-01-01 00:03:00.000', 0)" + " t2 values('2020-01-01 00:00:00.000', 0)" + " ('2020-01-01 00:01:00.000', 0)" + " ('2020-01-01 00:01:01.000', 0)" + " ('2020-01-01 00:01:02.000', 0)" + " t3 values('2020-01-01 00:01:02.000', 0)" + " t4 values('2020-01-01 00:01:02.000', 0)" + " t5 values('2020-01-01 00:01:02.000', 0)" + " t6 values('2020-01-01 00:01:02.000', 0)" + " t7 values('2020-01-01 00:01:02.000', 0)" + " t8 values('2020-01-01 00:01:02.000', 0)" + " t9 values('2020-01-01 00:01:02.000', 0)"); + int affected = taos_affected_rows(res); + if (affected != 18) { + printf("\033[31m%d rows affected by last insert statement, but it should be 18\033[0m\n", affected); + } + // super tables subscription + usleep(1000000); +} + + +static int print_result(TAOS_RES* res, int blockFetch) { + TAOS_ROW row = NULL; + int num_fields = taos_num_fields(res); + TAOS_FIELD* fields = taos_fetch_fields(res); + int nRows = 0; + + if (blockFetch) { + int rows = 0; + while ((rows = taos_fetch_block(res, &row))) { + for (int i = 0; i < rows; i++) { + char temp[256]; + taos_print_row(temp, row + i, fields, num_fields); + puts(temp); + } + nRows += rows; + } + } else { + while ((row = taos_fetch_row(res))) { + char temp[256]; + taos_print_row(temp, row, fields, num_fields); + puts(temp); + nRows++; + } + } + + printf("%d rows consumed.\n", nRows); + return nRows; +} + + +static void check_row_count(int line, TAOS_RES* res, int expected) { + int actual = print_result(res, expected % 2); + if (actual != expected) { + printf("\033[31mline %d: row count mismatch, expected: %d, actual: %d\033[0m\n", line, expected, actual); + } else { + printf("line %d: %d rows consumed as expected\n", line, actual); + } +} + + +static void verify_query(TAOS* taos) { + prepare_data(taos); + + int code = taos_validate_sql(taos, "select * from nonexisttable"); + if (code == 0) { + printf("\033[31mimpossible, the table does not exists\033[0m\n"); + } + + code = taos_validate_sql(taos, "select * from meters"); + if (code != 0) { + printf("\033[31mimpossible, the table does exists: 0x%08x\033[0m\n", code); + } + + TAOS_RES* res = taos_query(taos, "select * from meters"); + check_row_count(__LINE__, res, 18); + printf("result precision is: %d\n", taos_result_precision(res)); + int c = taos_field_count(res); + printf("field count is: %d\n", c); + int* lengths = taos_fetch_lengths(res); + for (int i = 0; i < c; i++) { + printf("length of column %d is %d\n", i, lengths[i]); + } + taos_free_result(res); + + res = taos_query(taos, "select * from t0"); + check_row_count(__LINE__, res, 3); + taos_free_result(res); + + res = taos_query(taos, "select * from nonexisttable"); + code = taos_errno(res); + printf("code=%d, error msg=%s\n", code, taos_errstr(res)); + taos_free_result(res); +} + + +static void verify_subscribe(TAOS* taos) { + prepare_data(taos); + + TAOS_SUB* tsub = taos_subscribe(taos, 0, "test", "select * from meters;", NULL, NULL, 0); + TAOS_RES* res = taos_consume(tsub); + check_row_count(__LINE__, res, 18); + + res = taos_consume(tsub); + check_row_count(__LINE__, res, 0); + + taos_query(taos, "insert into t0 values('2020-01-01 00:02:00.001', 0);"); + taos_query(taos, "insert into t8 values('2020-01-01 00:01:03.000', 0);"); + res = taos_consume(tsub); + check_row_count(__LINE__, res, 2); + + taos_query(taos, "insert into t2 values('2020-01-01 00:01:02.001', 0);"); + taos_query(taos, "insert into t1 values('2020-01-01 00:03:00.001', 0);"); + res = taos_consume(tsub); + check_row_count(__LINE__, res, 2); + + taos_query(taos, "insert into t1 values('2020-01-01 00:03:00.002', 0);"); + res = taos_consume(tsub); + check_row_count(__LINE__, res, 1); + + // keep progress information and restart subscription + taos_unsubscribe(tsub, 1); + taos_query(taos, "insert into t0 values('2020-01-01 00:04:00.000', 0);"); + tsub = taos_subscribe(taos, 1, "test", "select * from meters;", NULL, NULL, 0); + res = taos_consume(tsub); + check_row_count(__LINE__, res, 24); + + // keep progress information and continue previous subscription + taos_unsubscribe(tsub, 1); + tsub = taos_subscribe(taos, 0, "test", "select * from meters;", NULL, NULL, 0); + res = taos_consume(tsub); + check_row_count(__LINE__, res, 0); + + // don't keep progress information and continue previous subscription + taos_unsubscribe(tsub, 0); + tsub = taos_subscribe(taos, 0, "test", "select * from meters;", NULL, NULL, 0); + res = taos_consume(tsub); + check_row_count(__LINE__, res, 24); + + // single meter subscription + + taos_unsubscribe(tsub, 0); + tsub = taos_subscribe(taos, 0, "test", "select * from t0;", NULL, NULL, 0); + res = taos_consume(tsub); + check_row_count(__LINE__, res, 5); + + res = taos_consume(tsub); + check_row_count(__LINE__, res, 0); + + taos_query(taos, "insert into t0 values('2020-01-01 00:04:00.001', 0);"); + res = taos_consume(tsub); + check_row_count(__LINE__, res, 1); + + taos_unsubscribe(tsub, 0); +} + + +void verify_prepare(TAOS* taos) { + TAOS_RES* result = taos_query(taos, "drop database if exists test;"); + usleep(100000); + taos_query(taos, "create database test;"); + + int code = taos_errno(result); + if (code != 0) { + printf("failed to create database, reason:%s\n", taos_errstr(result)); + taos_free_result(result); + exit(1); + } + taos_free_result(result); + + usleep(100000); + taos_select_db(taos, "test"); + + // create table + const char* sql = "create table m1 (ts timestamp, b bool, v1 tinyint, v2 smallint, v4 int, v8 bigint, f4 float, f8 double, bin binary(40), blob nchar(10))"; + result = taos_query(taos, sql); + code = taos_errno(result); + if (code != 0) { + printf("failed to create table, reason:%s\n", taos_errstr(result)); + taos_free_result(result); + exit(1); + } + taos_free_result(result); + + // insert 10 records + struct { + int64_t ts; + int8_t b; + int8_t v1; + int16_t v2; + int32_t v4; + int64_t v8; + float f4; + double f8; + char bin[40]; + char blob[80]; + } v = {0}; + + TAOS_STMT* stmt = taos_stmt_init(taos); + TAOS_BIND params[10]; + params[0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; + params[0].buffer_length = sizeof(v.ts); + params[0].buffer = &v.ts; + params[0].length = ¶ms[0].buffer_length; + params[0].is_null = NULL; + + params[1].buffer_type = TSDB_DATA_TYPE_BOOL; + params[1].buffer_length = sizeof(v.b); + params[1].buffer = &v.b; + params[1].length = ¶ms[1].buffer_length; + params[1].is_null = NULL; + + params[2].buffer_type = TSDB_DATA_TYPE_TINYINT; + params[2].buffer_length = sizeof(v.v1); + params[2].buffer = &v.v1; + params[2].length = ¶ms[2].buffer_length; + params[2].is_null = NULL; + + params[3].buffer_type = TSDB_DATA_TYPE_SMALLINT; + params[3].buffer_length = sizeof(v.v2); + params[3].buffer = &v.v2; + params[3].length = ¶ms[3].buffer_length; + params[3].is_null = NULL; + + params[4].buffer_type = TSDB_DATA_TYPE_INT; + params[4].buffer_length = sizeof(v.v4); + params[4].buffer = &v.v4; + params[4].length = ¶ms[4].buffer_length; + params[4].is_null = NULL; + + params[5].buffer_type = TSDB_DATA_TYPE_BIGINT; + params[5].buffer_length = sizeof(v.v8); + params[5].buffer = &v.v8; + params[5].length = ¶ms[5].buffer_length; + params[5].is_null = NULL; + + params[6].buffer_type = TSDB_DATA_TYPE_FLOAT; + params[6].buffer_length = sizeof(v.f4); + params[6].buffer = &v.f4; + params[6].length = ¶ms[6].buffer_length; + params[6].is_null = NULL; + + params[7].buffer_type = TSDB_DATA_TYPE_DOUBLE; + params[7].buffer_length = sizeof(v.f8); + params[7].buffer = &v.f8; + params[7].length = ¶ms[7].buffer_length; + params[7].is_null = NULL; + + params[8].buffer_type = TSDB_DATA_TYPE_BINARY; + params[8].buffer_length = sizeof(v.bin); + params[8].buffer = v.bin; + params[8].length = ¶ms[8].buffer_length; + params[8].is_null = NULL; + + strcpy(v.blob, "一二三四五六七八九十"); + params[9].buffer_type = TSDB_DATA_TYPE_NCHAR; + params[9].buffer_length = strlen(v.blob); + params[9].buffer = v.blob; + params[9].length = ¶ms[9].buffer_length; + params[9].is_null = NULL; + + int is_null = 1; + + sql = "insert into m1 values(?,?,?,?,?,?,?,?,?,?)"; + code = taos_stmt_prepare(stmt, sql, 0); + if (code != 0){ + printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); + } + v.ts = 1591060628000; + for (int i = 0; i < 10; ++i) { + v.ts += 1; + for (int j = 1; j < 10; ++j) { + params[j].is_null = ((i == j) ? &is_null : 0); + } + v.b = (int8_t)i % 2; + v.v1 = (int8_t)i; + v.v2 = (int16_t)(i * 2); + v.v4 = (int32_t)(i * 4); + v.v8 = (int64_t)(i * 8); + v.f4 = (float)(i * 40); + v.f8 = (double)(i * 80); + for (int j = 0; j < sizeof(v.bin) - 1; ++j) { + v.bin[j] = (char)(i + '0'); + } + + taos_stmt_bind_param(stmt, params); + taos_stmt_add_batch(stmt); + } + if (taos_stmt_execute(stmt) != 0) { + printf("failed to execute insert statement.\n"); + exit(1); + } + taos_stmt_close(stmt); + + // query the records + stmt = taos_stmt_init(taos); + taos_stmt_prepare(stmt, "SELECT * FROM m1 WHERE v1 > ? AND v2 < ?", 0); + v.v1 = 5; + v.v2 = 15; + taos_stmt_bind_param(stmt, params + 2); + if (taos_stmt_execute(stmt) != 0) { + printf("failed to execute select statement.\n"); + exit(1); + } + + result = taos_stmt_use_result(stmt); + + TAOS_ROW row; + int rows = 0; + int num_fields = taos_num_fields(result); + TAOS_FIELD *fields = taos_fetch_fields(result); + char temp[256]; + + // fetch the records row by row + while ((row = taos_fetch_row(result))) { + rows++; + taos_print_row(temp, row, fields, num_fields); + printf("%s\n", temp); + } + + taos_free_result(result); + taos_stmt_close(stmt); +} + +void verify_stream(TAOS* taos) { +} + +int main(int argc, char *argv[]) { + const char* host = "127.0.0.1"; + const char* user = "root"; + const char* passwd = "taosdata"; + + taos_options(TSDB_OPTION_TIMEZONE, "GMT-8"); + taos_init(); + + TAOS* taos = taos_connect(host, user, passwd, "", 0); + if (taos == NULL) { + printf("failed to connect to db, reason:%s\n", taos_errstr(taos)); + exit(1); + } + + char* info = taos_get_server_info(taos); + printf("server info: %s\n", info); + info = taos_get_client_info(taos); + printf("client info: %s\n", info); + + printf("************ verify query *************\n"); + verify_query(taos); + printf("*********** verify subscribe ************\n"); + verify_subscribe(taos); + printf("************ verify prepare *************\n"); + verify_prepare(taos); + printf("************ verify stream *************\n"); + verify_stream(taos); + + taos_close(taos); + taos_cleanup(); +} \ No newline at end of file From dbd22da833284b240dda7fe32935c1fb9a5009d0 Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Fri, 17 Jul 2020 18:28:33 +0800 Subject: [PATCH 013/100] td-909: fix stream start time issue fix crash in taos_load_table_info --- src/client/src/tscSql.c | 2 + src/client/src/tscStream.c | 6 ++- tests/examples/c/apitest.c | 82 +++++++++++++++++++++++++++++++++++++- 3 files changed, 88 insertions(+), 2 deletions(-) diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index 49e4e027a0..b1c85dd662 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -880,6 +880,8 @@ int taos_load_table_info(TAOS *taos, const char *tableNameList) { } SSqlObj* pSql = calloc(1, sizeof(SSqlObj)); + pSql->pTscObj = taos; + pSql->signature = pSql; SSqlRes *pRes = &pSql->res; pRes->numOfTotal = 0; // the number of getting table meta from server diff --git a/src/client/src/tscStream.c b/src/client/src/tscStream.c index 7c188ec969..b07627c87b 100644 --- a/src/client/src/tscStream.c +++ b/src/client/src/tscStream.c @@ -122,7 +122,7 @@ static void tscProcessStreamTimer(void *handle, void *tmrId) { pQueryInfo->window.ekey = pStream->etime; } } else { - pQueryInfo->window.skey = pStream->stime - pStream->interval; + pQueryInfo->window.skey = pStream->stime; int64_t etime = taosGetTimestamp(pStream->precision); // delay to wait all data in last time window if (pStream->precision == TSDB_TIME_PRECISION_MICRO) { @@ -232,6 +232,9 @@ static void tscProcessStreamRetrieveResult(void *param, TAOS_RES *res, int numOf (*pStream->fp)(pStream->param, res, row); } + if (!pStream->isProject) { + pStream->stime += pStream->slidingTime; + } // actually only one row is returned. this following is not necessary taos_fetch_rows_a(res, tscProcessStreamRetrieveResult, pStream); } else { // numOfRows == 0, all data has been retrieved @@ -432,6 +435,7 @@ static int64_t tscGetStreamStartTimestamp(SSqlObj *pSql, SSqlStream *pStream, in } else { // timewindow based aggregation stream if (stime == 0) { // no data in meter till now stime = ((int64_t)taosGetTimestamp(pStream->precision) / pStream->interval) * pStream->interval; + stime -= pStream->interval; tscWarn("%p stream:%p, last timestamp:0, reset to:%" PRId64, pSql, pStream, stime); } else { int64_t newStime = (stime / pStream->interval) * pStream->interval; diff --git a/tests/examples/c/apitest.c b/tests/examples/c/apitest.c index 0be1f01bbe..bc628fb58f 100644 --- a/tests/examples/c/apitest.c +++ b/tests/examples/c/apitest.c @@ -98,7 +98,12 @@ static void check_row_count(int line, TAOS_RES* res, int expected) { static void verify_query(TAOS* taos) { prepare_data(taos); - int code = taos_validate_sql(taos, "select * from nonexisttable"); + int code = taos_load_table_info(taos, "t0,t1,t2,t3,t4,t5,t6,t7,t8,t9"); + if (code != 0) { + printf("\033[31mfailed to load table info: 0x%08x\033[0m\n", code); + } + + code = taos_validate_sql(taos, "select * from nonexisttable"); if (code == 0) { printf("\033[31mimpossible, the table does not exists\033[0m\n"); } @@ -127,9 +132,17 @@ static void verify_query(TAOS* taos) { code = taos_errno(res); printf("code=%d, error msg=%s\n", code, taos_errstr(res)); taos_free_result(res); + + res = taos_query(taos, "select * from meters"); + taos_stop_query(res); } +void subscribe_callback(TAOS_SUB* tsub, TAOS_RES *res, void* param, int code) { + int rows = print_result(res, *(int*)param); + printf("%d rows consumed in subscribe_callback\n", rows); +} + static void verify_subscribe(TAOS* taos) { prepare_data(taos); @@ -188,6 +201,13 @@ static void verify_subscribe(TAOS* taos) { check_row_count(__LINE__, res, 1); taos_unsubscribe(tsub, 0); + + int blockFetch = 0; + tsub = taos_subscribe(taos, 1, "test", "select * from meters;", subscribe_callback, &blockFetch, 1000); + usleep(2000000); + taos_query(taos, "insert into t0 values('2020-01-01 00:05:00.001', 0);"); + usleep(2000000); + taos_unsubscribe(tsub, 0); } @@ -358,7 +378,60 @@ void verify_prepare(TAOS* taos) { taos_stmt_close(stmt); } +void retrieve_callback(void *param, TAOS_RES *tres, int numOfRows) +{ + if (numOfRows > 0) { + printf("%d rows async retrieved\n", numOfRows); + taos_fetch_rows_a(tres, retrieve_callback, param); + } else { + if (numOfRows < 0) { + printf("\033[31masync retrieve failed, code: %d\033[0m\n", numOfRows); + } else { + printf("async retrieve completed\n"); + } + taos_free_result(tres); + } +} + +void select_callback(void *param, TAOS_RES *tres, int code) +{ + if (code == 0 && tres) { + taos_fetch_rows_a(tres, retrieve_callback, param); + } else { + printf("\033[31masync select failed, code: %d\033[0m\n", code); + } +} + +void verify_async(TAOS* taos) { + prepare_data(taos); + taos_query_a(taos, "select * from meters", select_callback, NULL); + usleep(1000000); +} + +void stream_callback(void *param, TAOS_RES *res, TAOS_ROW row) { + int num_fields = taos_num_fields(res); + TAOS_FIELD* fields = taos_fetch_fields(res); + + printf("got one row from stream_callback\n"); + char temp[256]; + taos_print_row(temp, row, fields, num_fields); + puts(temp); +} + void verify_stream(TAOS* taos) { + prepare_data(taos); + TAOS_STREAM* strm = taos_open_stream( + taos, + "select count(*) from meters interval(1m)", + stream_callback, + 0, + NULL, + NULL); + printf("waiting for stream data\n"); + usleep(100000); + taos_query(taos, "insert into t0 values(now, 0)(now+5s,1)(now+10s, 2);"); + usleep(200000000); + taos_close_stream(strm); } int main(int argc, char *argv[]) { @@ -382,12 +455,19 @@ int main(int argc, char *argv[]) { printf("************ verify query *************\n"); verify_query(taos); + + printf("********* verify async query **********\n"); + verify_async(taos); + printf("*********** verify subscribe ************\n"); verify_subscribe(taos); + printf("************ verify prepare *************\n"); verify_prepare(taos); + printf("************ verify stream *************\n"); verify_stream(taos); + printf("done\n"); taos_close(taos); taos_cleanup(); From 700504900fc6419f6e87f26d2449ab2168144017 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sat, 18 Jul 2020 01:32:00 +0000 Subject: [PATCH 014/100] compile error --- src/client/src/tscServer.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index d0c9d3636b..cafdcfbf25 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -155,9 +155,10 @@ void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) { if (code == 0) { SCMHeartBeatRsp *pRsp = (SCMHeartBeatRsp *)pRes->pRsp; SRpcIpSet * pIpList = &pRsp->ipList; - if (pIpList->numOfIps > 0) + if (pIpList->numOfIps > 0) { tscIpSetHtons(pIpList); tscUpdateMgmtIpList(pIpList); + } pSql->pTscObj->connId = htonl(pRsp->connId); @@ -1996,9 +1997,10 @@ int tscProcessConnectRsp(SSqlObj *pSql) { assert(len <= sizeof(pObj->db)); tstrncpy(pObj->db, temp, sizeof(pObj->db)); - if (pConnect->ipList.numOfIps > 0) + if (pConnect->ipList.numOfIps > 0) { tscIpSetHtons(&pConnect->ipList); tscUpdateMgmtIpList(&pConnect->ipList); + } strcpy(pObj->sversion, pConnect->serverVersion); pObj->writeAuth = pConnect->writeAuth; From 54ab10737a7dfc0b433680a16751858077fb6e8b Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sat, 18 Jul 2020 01:43:22 +0000 Subject: [PATCH 015/100] compile error --- src/client/src/tscServer.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index cafdcfbf25..9425b32aab 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -26,6 +26,7 @@ #include "ttime.h" #include "ttimer.h" #include "tutil.h" +#include "tlockfree.h" #define TSC_MGMT_VNODE 999 From 7aa062c9ad5679891f0d949160146c02eaa6b71a Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sat, 18 Jul 2020 03:03:04 +0000 Subject: [PATCH 016/100] modify function --- src/util/inc/tlockfree.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/util/inc/tlockfree.h b/src/util/inc/tlockfree.h index e425d71d27..a81f597832 100644 --- a/src/util/inc/tlockfree.h +++ b/src/util/inc/tlockfree.h @@ -75,7 +75,7 @@ void taosRUnLockLatch(SRWLatch *pLatch); // copy on read #define taosCorBeginRead(x) for (uint32_t i_ = 1; 1; ++i_) { \ - int32_t old_ = atomic_load_32(x); \ + int32_t old_ = atomic_add_fetch_32((x), 0); \ if (old_ & 0x00000001) { \ if (i_ % 1000 == 0) { \ sched_yield(); \ @@ -84,7 +84,7 @@ void taosRUnLockLatch(SRWLatch *pLatch); } #define taosCorEndRead(x) \ - if (atomic_load_32(x) == old_) { \ + if (atomic_add_fetch_32((x), 0) == old_) { \ break; \ } \ } From a28e9a9402fcb802fa7a301d76385fb083d6db3d Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sat, 18 Jul 2020 03:21:16 +0000 Subject: [PATCH 017/100] test --- src/client/src/tscServer.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 9425b32aab..7269173c8c 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -85,7 +85,7 @@ void tscUpdateMgmtIpList(SRpcIpSet *pIpSet) { } static void tscDumpIpSetFromVgroupInfo(SCMVgroupInfo *pVgroupInfo, SRpcIpSet *pIpSet, int32_t *vgId) { if (pVgroupInfo == NULL) { return;} - taosCorBeginRead(&pVgroupInfo->version); + //taosCorBeginRead(&pVgroupInfo->version); if (vgId) { *vgId = pVgroupInfo->vgId; } @@ -96,7 +96,7 @@ static void tscDumpIpSetFromVgroupInfo(SCMVgroupInfo *pVgroupInfo, SRpcIpSet *pI strncpy(pIpSet->fqdn[i], pVgroupInfo->ipAddr[i].fqdn, TSDB_FQDN_LEN); pIpSet->port[i] = pVgroupInfo->ipAddr[i].port; } - taosCorEndRead(&pVgroupInfo->version); + //taosCorEndRead(&pVgroupInfo->version); } static void tscUpdateVgroupInfo(SSqlObj *pObj, SRpcIpSet *pIpSet) { @@ -105,7 +105,7 @@ static void tscUpdateVgroupInfo(SSqlObj *pObj, SRpcIpSet *pIpSet) { if (pTableMetaInfo == NULL || pTableMetaInfo->pTableMeta == NULL) { return;} SCMVgroupInfo *pVgroupInfo = &pTableMetaInfo->pTableMeta->vgroupInfo; - taosCorBeginWrite(&pVgroupInfo->version); + //taosCorBeginWrite(&pVgroupInfo->version); //TODO(dengyihao), dont care vgid pVgroupInfo->inUse = pIpSet->inUse; pVgroupInfo->numOfIps = pIpSet->numOfIps; @@ -113,7 +113,7 @@ static void tscUpdateVgroupInfo(SSqlObj *pObj, SRpcIpSet *pIpSet) { strncpy(pVgroupInfo->ipAddr[i].fqdn, pIpSet->fqdn[i], TSDB_FQDN_LEN); pVgroupInfo->ipAddr[i].port = pIpSet->port[i]; } - taosCorEndWrite(&pVgroupInfo->version); + //taosCorEndWrite(&pVgroupInfo->version); } void tscPrintMgmtIp() { SRpcIpSet dump; From 7fd0fc2bf03a40b9ff8edad0a7ea7e4fbe34ace4 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 18 Jul 2020 12:03:50 +0800 Subject: [PATCH 018/100] [td-225] fix bugs in first/last query --- src/query/src/qExecutor.c | 64 ++++++++++++------- .../general/parser/first_last_query.sim | 11 ++++ 2 files changed, 51 insertions(+), 24 deletions(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index cf13ace11f..daca6fab25 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -2236,6 +2236,27 @@ static void ensureOutputBuffer(SQueryRuntimeEnv* pRuntimeEnv, SDataBlockInfo* pB } } +static void doSetInitialTimewindow(SQueryRuntimeEnv* pRuntimeEnv, SDataBlockInfo* pBlockInfo) { + SQuery* pQuery = pRuntimeEnv->pQuery; + + if (QUERY_IS_INTERVAL_QUERY(pQuery) && pRuntimeEnv->windowResInfo.prevSKey == TSKEY_INITIAL_VAL) { + STimeWindow w = TSWINDOW_INITIALIZER; + SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; + + if (QUERY_IS_ASC_QUERY(pQuery)) { + getAlignQueryTimeWindow(pQuery, pBlockInfo->window.skey, pBlockInfo->window.skey, pQuery->window.ekey, &w); + pWindowResInfo->startTime = w.skey; + pWindowResInfo->prevSKey = w.skey; + } else { + // the start position of the first time window in the endpoint that spreads beyond the queried last timestamp + getAlignQueryTimeWindow(pQuery, pBlockInfo->window.ekey, pQuery->window.ekey, pBlockInfo->window.ekey, &w); + + pWindowResInfo->startTime = pQuery->window.skey; + pWindowResInfo->prevSKey = w.skey; + } + } +} + static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { SQuery *pQuery = pRuntimeEnv->pQuery; STableQueryInfo* pTableQueryInfo = pQuery->current; @@ -2263,24 +2284,7 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { } tsdbRetrieveDataBlockInfo(pQueryHandle, &blockInfo); - - // todo extract methods - if (QUERY_IS_INTERVAL_QUERY(pQuery) && pRuntimeEnv->windowResInfo.prevSKey == TSKEY_INITIAL_VAL) { - STimeWindow w = TSWINDOW_INITIALIZER; - SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; - - if (QUERY_IS_ASC_QUERY(pQuery)) { - getAlignQueryTimeWindow(pQuery, blockInfo.window.skey, blockInfo.window.skey, pQuery->window.ekey, &w); - pWindowResInfo->startTime = w.skey; - pWindowResInfo->prevSKey = w.skey; - } else { - // the start position of the first time window in the endpoint that spreads beyond the queried last timestamp - getAlignQueryTimeWindow(pQuery, blockInfo.window.ekey, pQuery->window.ekey, blockInfo.window.ekey, &w); - - pWindowResInfo->startTime = pQuery->window.skey; - pWindowResInfo->prevSKey = w.skey; - } - } + doSetInitialTimewindow(pRuntimeEnv, &blockInfo); // in case of prj/diff query, ensure the output buffer is sufficient to accommodate the results of current block ensureOutputBuffer(pRuntimeEnv, &blockInfo); @@ -2314,7 +2318,6 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) { if (QUERY_IS_INTERVAL_QUERY(pQuery) && IS_MASTER_SCAN(pRuntimeEnv)) { if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { closeAllTimeWindow(&pRuntimeEnv->windowResInfo); -// removeRedundantWindow(&pRuntimeEnv->windowResInfo, pTableQueryInfo->lastKey - step, step); pRuntimeEnv->windowResInfo.curIndex = pRuntimeEnv->windowResInfo.size - 1; // point to the last time window } else { assert(Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL)); @@ -3223,6 +3226,13 @@ static void setEnvBeforeReverseScan(SQueryRuntimeEnv *pRuntimeEnv, SQueryStatusI SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); SWITCH_ORDER(pQuery->order.order); + + if (QUERY_IS_ASC_QUERY(pQuery)) { + assert(pQuery->window.skey <= pQuery->window.ekey); + } else { + assert(pQuery->window.skey >= pQuery->window.ekey); + } + SET_REVERSE_SCAN_FLAG(pRuntimeEnv); STsdbQueryCond cond = { @@ -3262,8 +3272,7 @@ static void clearEnvAfterReverseScan(SQueryRuntimeEnv *pRuntimeEnv, SQueryStatus SET_MASTER_SCAN_FLAG(pRuntimeEnv); - // update the pQuery->window.skey and pQuery->window.ekey to limit the scan scope of sliding query - // during reverse scan + // update the pQuery->window.skey and pQuery->window.ekey to limit the scan scope of sliding query during reverse scan pTableQueryInfo->lastKey = pStatus->lastKey; pQuery->status = pStatus->status; @@ -3289,7 +3298,12 @@ void scanOneTableDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start) { if (pRuntimeEnv->scanFlag == MASTER_SCAN) { qstatus.status = pQuery->status; - qstatus.curWindow.ekey = pTableQueryInfo->lastKey - step; + + // do nothing if no data blocks are found qualified during scan + if (qstatus.lastKey != pTableQueryInfo->lastKey) { + qstatus.curWindow.ekey = pTableQueryInfo->lastKey - step; + } + qstatus.lastKey = pTableQueryInfo->lastKey; } @@ -6282,7 +6296,7 @@ void qTableQuery(qinfo_t qinfo) { SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; if (onlyQueryTags(pQInfo->runtimeEnv.pQuery)) { assert(pQInfo->runtimeEnv.pQueryHandle == NULL); - buildTagQueryResult(pQInfo); // todo support the limit/offset + buildTagQueryResult(pQInfo); } else if (pQInfo->runtimeEnv.stableQuery) { stableQueryImpl(pQInfo); } else { @@ -6494,7 +6508,9 @@ static void buildTagQueryResult(SQInfo* pQInfo) { } else { // return only the tags|table name etc. count = 0; SSchema tbnameSchema = tGetTableNameColumnSchema(); - while(pQInfo->tableIndex < num && count < pQuery->rec.capacity) { + + int32_t maxNumOfTables = (pQuery->limit.limit < pQuery->rec.capacity)? pQuery->limit.limit:pQuery->rec.capacity; + while(pQInfo->tableIndex < num && count < maxNumOfTables) { int32_t i = pQInfo->tableIndex++; SExprInfo* pExprInfo = pQuery->pSelectExpr; diff --git a/tests/script/general/parser/first_last_query.sim b/tests/script/general/parser/first_last_query.sim index fa5ed8b4ad..d11bdccb12 100644 --- a/tests/script/general/parser/first_last_query.sim +++ b/tests/script/general/parser/first_last_query.sim @@ -126,3 +126,14 @@ endi if $data01 != 0 then return -1 endi + +#add check for out of range first/last query +sql select first(ts),last(ts) from first_tb4 where ts>'2018-9-18 1:40:01'; +if $row != 0 then + return -1 +endi + +sql select first(ts),last(ts) from first_tb4 where ts<'2018-9-17 8:50:0'; +if $row != 0 then + return -1 +endi \ No newline at end of file From 3d16404e19ad8cfb399f7f69b9934aed7af6da84 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 18 Jul 2020 13:25:53 +0800 Subject: [PATCH 019/100] [td-225] fix bugs in tags retrieval --- src/query/src/qExecutor.c | 88 ++++++++++++----------- tests/script/general/parser/null_char.sim | 15 ++++ 2 files changed, 62 insertions(+), 41 deletions(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index daca6fab25..6dba5cbd2a 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -6417,6 +6417,22 @@ int32_t qKillQuery(qinfo_t qinfo) { return TSDB_CODE_SUCCESS; } +static void doSetTagValueToResultBuf(char* output, const char* val, int16_t type, int16_t bytes) { + if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { + if (val == NULL) { + setVardataNull(output, type); + } else { + memcpy(output, val, varDataTLen(val)); + } + } else { + if (val == NULL) { + setNull(output, type, bytes); + } else { // todo here stop will cause client crash + memcpy(output, val, bytes); + } + } +} + static void buildTagQueryResult(SQInfo* pQInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery * pQuery = pRuntimeEnv->pQuery; @@ -6473,25 +6489,11 @@ static void buildTagQueryResult(SQInfo* pQInfo) { output += sizeof(pQInfo->vgId); if (pExprInfo->base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX) { - char *data = tsdbGetTableName(item->pTable); + char* data = tsdbGetTableName(item->pTable); memcpy(output, data, varDataTLen(data)); } else { - char *val = tsdbGetTableTagVal(item->pTable, pExprInfo->base.colInfo.colId, type, bytes); - - // todo refactor - if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { - if (val == NULL) { - setVardataNull(output, type); - } else { - memcpy(output, val, varDataTLen(val)); - } - } else { - if (val == NULL) { - setNull(output, type, bytes); - } else { // todo here stop will cause client crash - memcpy(output, val, bytes); - } - } + char* data = tsdbGetTableTagVal(item->pTable, pExprInfo->base.colInfo.colId, type, bytes); + doSetTagValueToResultBuf(output, data, type, bytes); } count += 1; @@ -6509,39 +6511,43 @@ static void buildTagQueryResult(SQInfo* pQInfo) { count = 0; SSchema tbnameSchema = tGetTableNameColumnSchema(); - int32_t maxNumOfTables = (pQuery->limit.limit < pQuery->rec.capacity)? pQuery->limit.limit:pQuery->rec.capacity; + int32_t maxNumOfTables = pQuery->rec.capacity; + if (pQuery->limit.limit >= 0 && pQuery->limit.limit < pQuery->rec.capacity) { + maxNumOfTables = pQuery->limit.limit; + } + while(pQInfo->tableIndex < num && count < maxNumOfTables) { int32_t i = pQInfo->tableIndex++; + // discard current result due to offset + if (pQuery->limit.offset > 0) { + pQuery->limit.offset -= 1; + continue; + } + SExprInfo* pExprInfo = pQuery->pSelectExpr; STableQueryInfo* item = taosArrayGetP(pa, i); + char *data = NULL, *dst = NULL; + int16_t type = 0, bytes = 0; for(int32_t j = 0; j < pQuery->numOfOutput; ++j) { - if (pExprInfo[j].base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX) { - char* data = tsdbGetTableName(item->pTable); - char* dst = pQuery->sdata[j]->data + count * tbnameSchema.bytes; - memcpy(dst, data, varDataTLen(data)); - } else {// todo refactor - int16_t type = pExprInfo[j].type; - int16_t bytes = pExprInfo[j].bytes; - - char* data = tsdbGetTableTagVal(item->pTable, pExprInfo[j].base.colInfo.colId, type, bytes); - char* dst = pQuery->sdata[j]->data + count * pExprInfo[j].bytes; - if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { - if (data == NULL) { - setVardataNull(dst, type); - } else { - memcpy(dst, data, varDataTLen(data)); - } - } else { - if (data == NULL) { - setNull(dst, type, bytes); - } else { - memcpy(dst, data, pExprInfo[j].bytes); - } - } + if (pExprInfo[j].base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX) { + bytes = tbnameSchema.bytes; + type = tbnameSchema.type; + + data = tsdbGetTableName(item->pTable); + dst = pQuery->sdata[j]->data + count * tbnameSchema.bytes; + } else { + type = pExprInfo[j].type; + bytes = pExprInfo[j].bytes; + + data = tsdbGetTableTagVal(item->pTable, pExprInfo[j].base.colInfo.colId, type, bytes); + dst = pQuery->sdata[j]->data + count * pExprInfo[j].bytes; + } + + doSetTagValueToResultBuf(dst, data, type, bytes); } count += 1; } diff --git a/tests/script/general/parser/null_char.sim b/tests/script/general/parser/null_char.sim index 09e761c85e..2e39fc7db2 100644 --- a/tests/script/general/parser/null_char.sim +++ b/tests/script/general/parser/null_char.sim @@ -105,6 +105,21 @@ if $data03 != 1 then return -1 endi +sql select tag1 from st2 limit 20 offset 1 +if $rows != 0 then + return -1 +endi + +sql select tag1 from st2 limit 10 offset 2 +if $rows != 0 then + return -1 +endi + +sql select tag1 from st2 limit 0 offset 0 +if $rows != 0 then + return -1 +endi + sql create table st3 using mt2 tags (NULL, 'ABC', 103, 'FALSE') sql select tag1, tag2, tag3, tag5 from st3 if $rows != 1 then From 3c4e620f31257219d3871857f8c95102f87efecc Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sat, 18 Jul 2020 05:41:19 +0000 Subject: [PATCH 020/100] bugfix --- src/client/inc/tsclient.h | 10 ++++++- src/client/src/tscSchemaUtil.c | 16 ++++++++--- src/client/src/tscServer.c | 49 ++++++++++++++++++++-------------- src/inc/taosmsg.h | 2 -- 4 files changed, 50 insertions(+), 27 deletions(-) diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index e1180c4143..3a420567a9 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -52,12 +52,20 @@ typedef struct STableComInfo { int32_t rowSize; } STableComInfo; +typedef struct SCMCorVgroupInfo { + int32_t version; + int8_t inUse; + int8_t numOfIps; + SIpAddr ipAddr[TSDB_MAX_REPLICA]; +} SCMCorVgroupInfo; + typedef struct STableMeta { STableComInfo tableInfo; uint8_t tableType; int16_t sversion; int16_t tversion; - SCMVgroupInfo vgroupInfo; + SCMVgroupInfo vgroupInfo; + SCMCorVgroupInfo corVgroupInfo; int32_t sid; // the index of one table in a virtual node uint64_t uid; // unique id of a table SSchema schema[]; // if the table is TSDB_CHILD_TABLE, schema is acquired by super table meta info diff --git a/src/client/src/tscSchemaUtil.c b/src/client/src/tscSchemaUtil.c index b391a5760e..52342b3650 100644 --- a/src/client/src/tscSchemaUtil.c +++ b/src/client/src/tscSchemaUtil.c @@ -140,7 +140,15 @@ struct SSchema tscGetTbnameColumnSchema() { strcpy(s.name, TSQL_TBNAME_L); return s; } - +static void tscInitCorVgroupInfo(SCMCorVgroupInfo *corVgroupInfo, SCMVgroupInfo *vgroupInfo) { + corVgroupInfo->version = 0; + corVgroupInfo->inUse = 0; + corVgroupInfo->numOfIps = vgroupInfo->numOfIps; + for (int32_t i = 0; i < corVgroupInfo->numOfIps; i++) { + strncpy(corVgroupInfo->ipAddr[i].fqdn, vgroupInfo->ipAddr[i].fqdn, TSDB_FQDN_LEN); + corVgroupInfo->ipAddr[i].port = vgroupInfo->ipAddr[i].port; + } +} STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg, size_t* size) { assert(pTableMetaMsg != NULL); @@ -157,9 +165,9 @@ STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg, size_t* size pTableMeta->sid = pTableMetaMsg->sid; pTableMeta->uid = pTableMetaMsg->uid; pTableMeta->vgroupInfo = pTableMetaMsg->vgroup; - //init version here - pTableMeta->vgroupInfo.version = 0; - + + tscInitCorVgroupInfo(&pTableMeta->corVgroupInfo, &pTableMeta->vgroupInfo); + pTableMeta->sversion = pTableMetaMsg->sversion; pTableMeta->tversion = pTableMetaMsg->tversion; diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 7269173c8c..cb97307b04 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -45,6 +45,20 @@ void tscSaveSubscriptionProgress(void* sub); static int32_t minMsgSize() { return tsRpcHeadSize + 100; } +static void tscSetDnodeIpList(SSqlObj* pSql, SCMVgroupInfo* pVgroupInfo) { + SRpcIpSet* pIpList = &pSql->ipList; + pIpList->inUse = 0; + if (pVgroupInfo == NULL) { + pIpList->numOfIps = 0; + return; + } + + pIpList->numOfIps = pVgroupInfo->numOfIps; + for(int32_t i = 0; i < pVgroupInfo->numOfIps; ++i) { + strcpy(pIpList->fqdn[i], pVgroupInfo->ipAddr[i].fqdn); + pIpList->port[i] = pVgroupInfo->ipAddr[i].port; + } +} void tscIpSetCopy(SRpcIpSet *dst, SRpcIpSet *src) { dst->numOfIps = src->numOfIps; dst->inUse = src->inUse; @@ -83,12 +97,9 @@ void tscUpdateMgmtIpList(SRpcIpSet *pIpSet) { tscIpSetCopy(mgmtIpSet, pIpSet); taosCorEndWrite(&tscMgmtIpSet.version); } -static void tscDumpIpSetFromVgroupInfo(SCMVgroupInfo *pVgroupInfo, SRpcIpSet *pIpSet, int32_t *vgId) { +static void tscDumpIpSetFromVgroupInfo(SCMCorVgroupInfo *pVgroupInfo, SRpcIpSet *pIpSet) { if (pVgroupInfo == NULL) { return;} - //taosCorBeginRead(&pVgroupInfo->version); - if (vgId) { - *vgId = pVgroupInfo->vgId; - } + taosCorBeginRead(&pVgroupInfo->version); int8_t inUse = pVgroupInfo->inUse; pIpSet->inUse = (inUse >= 0 && inUse < TSDB_MAX_REPLICA) ? inUse: 0; pIpSet->numOfIps = pVgroupInfo->numOfIps; @@ -96,16 +107,16 @@ static void tscDumpIpSetFromVgroupInfo(SCMVgroupInfo *pVgroupInfo, SRpcIpSet *pI strncpy(pIpSet->fqdn[i], pVgroupInfo->ipAddr[i].fqdn, TSDB_FQDN_LEN); pIpSet->port[i] = pVgroupInfo->ipAddr[i].port; } - //taosCorEndRead(&pVgroupInfo->version); + taosCorEndRead(&pVgroupInfo->version); } static void tscUpdateVgroupInfo(SSqlObj *pObj, SRpcIpSet *pIpSet) { SSqlCmd *pCmd = &pObj->cmd; STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); if (pTableMetaInfo == NULL || pTableMetaInfo->pTableMeta == NULL) { return;} - SCMVgroupInfo *pVgroupInfo = &pTableMetaInfo->pTableMeta->vgroupInfo; + SCMCorVgroupInfo *pVgroupInfo = &pTableMetaInfo->pTableMeta->corVgroupInfo; - //taosCorBeginWrite(&pVgroupInfo->version); + taosCorBeginWrite(&pVgroupInfo->version); //TODO(dengyihao), dont care vgid pVgroupInfo->inUse = pIpSet->inUse; pVgroupInfo->numOfIps = pIpSet->numOfIps; @@ -113,7 +124,7 @@ static void tscUpdateVgroupInfo(SSqlObj *pObj, SRpcIpSet *pIpSet) { strncpy(pVgroupInfo->ipAddr[i].fqdn, pIpSet->fqdn[i], TSDB_FQDN_LEN); pVgroupInfo->ipAddr[i].port = pIpSet->port[i]; } - //taosCorEndWrite(&pVgroupInfo->version); + taosCorEndWrite(&pVgroupInfo->version); } void tscPrintMgmtIp() { SRpcIpSet dump; @@ -577,7 +588,7 @@ int tscBuildSubmitMsg(SSqlObj *pSql, SSqlInfo *pInfo) { // pSql->cmd.payloadLen is set during copying data into payload pSql->cmd.msgType = TSDB_MSG_TYPE_SUBMIT; - tscDumpIpSetFromVgroupInfo(&pTableMeta->vgroupInfo, &pSql->ipList, NULL); + tscDumpIpSetFromVgroupInfo(&pTableMeta->corVgroupInfo, &pSql->ipList); tscDebug("%p build submit msg, vgId:%d numOfTables:%d numberOfIP:%d", pSql, vgId, pSql->cmd.numOfTablesInSubmit, pSql->ipList.numOfIps); @@ -619,9 +630,10 @@ static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char } else { pVgroupInfo = &pTableMeta->vgroupInfo; } - int32_t vgId = 0; - tscDumpIpSetFromVgroupInfo(pVgroupInfo, &pSql->ipList, &vgId); - pQueryMsg->head.vgId = htonl(vgId); + tscSetDnodeIpList(pSql, pVgroupInfo); + if (pVgroupInfo != NULL) { + pQueryMsg->head.vgId = htonl(pVgroupInfo->vgId); + } STableIdInfo *pTableIdInfo = (STableIdInfo *)pMsg; pTableIdInfo->tid = htonl(pTableMeta->sid); @@ -639,10 +651,9 @@ static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char SVgroupTableInfo* pTableIdList = taosArrayGet(pTableMetaInfo->pVgroupTables, index); - // set the vgroup info - int32_t vgId = 0; - tscDumpIpSetFromVgroupInfo(&pTableIdList->vgInfo, &pSql->ipList, &vgId); - pQueryMsg->head.vgId = htonl(vgId); + // set the vgroup info + tscSetDnodeIpList(pSql, &pTableIdList->vgInfo); + pQueryMsg->head.vgId = htonl(pTableIdList->vgInfo.vgId); int32_t numOfTables = taosArrayGetSize(pTableIdList->itemList); pQueryMsg->numOfTables = htonl(numOfTables); // set the number of tables @@ -1374,7 +1385,7 @@ int tscBuildUpdateTagMsg(SSqlObj* pSql, SSqlInfo *pInfo) { SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - tscDumpIpSetFromVgroupInfo(&pTableMetaInfo->pTableMeta->vgroupInfo, &pSql->ipList, NULL); + tscDumpIpSetFromVgroupInfo(&pTableMetaInfo->pTableMeta->corVgroupInfo, &pSql->ipList); return TSDB_CODE_SUCCESS; } @@ -1900,8 +1911,6 @@ int tscProcessSTableVgroupRsp(SSqlObj *pSql) { for (int32_t j = 0; j < pInfo->vgroupList->numOfVgroups; ++j) { //just init, no need to lock SCMVgroupInfo *pVgroups = &pInfo->vgroupList->vgroups[j]; - pVgroups->version = 0; - pVgroups->inUse = 0; pVgroups->vgId = htonl(pVgroups->vgId); assert(pVgroups->numOfIps >= 1); for (int32_t k = 0; k < pVgroups->numOfIps; ++k) { diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index fec3d77d8e..b7afaf1e06 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -647,9 +647,7 @@ typedef struct SCMSTableVgroupMsg { } SCMSTableVgroupMsg, SCMSTableVgroupRspMsg; typedef struct { - int32_t version; int32_t vgId; - int8_t inUse; int8_t numOfIps; SIpAddr ipAddr[TSDB_MAX_REPLICA]; } SCMVgroupInfo; From 7ccba2bdf2bb85eb7e5ddafea78e1c602e59efd3 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 18 Jul 2020 15:12:08 +0800 Subject: [PATCH 021/100] [td-225] refactor codes and add some log. --- src/query/inc/qast.h | 1 - src/query/src/qExecutor.c | 106 +++++++++++++++++++------------------- src/query/src/qast.c | 4 +- src/tsdb/src/tsdbRead.c | 30 ++--------- 4 files changed, 59 insertions(+), 82 deletions(-) diff --git a/src/query/inc/qast.h b/src/query/inc/qast.h index 918604f8c9..00049b486d 100644 --- a/src/query/inc/qast.h +++ b/src/query/inc/qast.h @@ -45,7 +45,6 @@ typedef void (*__do_filter_suppl_fn_t)(void *, void *); * */ typedef struct tQueryInfo { - int32_t colIndex; // index of column in schema uint8_t optr; // expression operator SSchema sch; // schema of tags char* q; diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 6dba5cbd2a..42fc90f478 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -50,11 +50,6 @@ #define SDATA_BLOCK_INITIALIZER (SDataBlockInfo) {{0}, 0} -/* get the qinfo struct address from the query struct address */ -#define GET_COLUMN_BYTES(query, colidx) \ - ((query)->colList[(query)->pSelectExpr[colidx].base.colInfo.colIndex].bytes) -#define GET_COLUMN_TYPE(query, colidx) ((query)->colList[(query)->pSelectExpr[colidx].base.colInfo.colIndex].type) - enum { // when query starts to execute, this status will set QUERY_NOT_COMPLETED = 0x1u, @@ -394,15 +389,15 @@ static SWindowResult *doSetTimeWindowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SWin if (pWindowResInfo->size >= pWindowResInfo->capacity) { int64_t newCap = pWindowResInfo->capacity * 1.5; char *t = realloc(pWindowResInfo->pResult, newCap * sizeof(SWindowResult)); - if (t != NULL) { - pWindowResInfo->pResult = (SWindowResult *)t; - - int32_t inc = newCap - pWindowResInfo->capacity; - memset(&pWindowResInfo->pResult[pWindowResInfo->capacity], 0, sizeof(SWindowResult) * inc); - } else { - // todo + if (t == NULL) { + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } + pWindowResInfo->pResult = (SWindowResult *)t; + + int32_t inc = newCap - pWindowResInfo->capacity; + memset(&pWindowResInfo->pResult[pWindowResInfo->capacity], 0, sizeof(SWindowResult) * inc); + for (int32_t i = pWindowResInfo->capacity; i < newCap; ++i) { createQueryResultInfo(pQuery, &pWindowResInfo->pResult[i], pRuntimeEnv->stableQuery, pRuntimeEnv->interBufSize); } @@ -1053,9 +1048,9 @@ static char *getGroupbyColumnData(SQuery *pQuery, int16_t *type, int16_t *bytes, *type = pQuery->colList[colIndex].type; *bytes = pQuery->colList[colIndex].bytes; /* - * the colIndex is acquired from the first meter of all qualified meters in this vnode during query prepare - * stage, the remain meter may not have the required column in cache actually. So, the validation of required - * column in cache with the corresponding meter schema is reinforced. + * the colIndex is acquired from the first tables of all qualified tables in this vnode during query prepare + * stage, the remain tables may not have the required column in cache actually. So, the validation of required + * column in cache with the corresponding schema is reinforced. */ int32_t numOfCols = taosArrayGetSize(pDataBlock); @@ -2364,6 +2359,18 @@ static void doSetTagValueInParam(void *tsdb, void* pTable, int32_t tagColId, tVa } } +static SColumnInfo* doGetTagColumnInfoById(SColumnInfo* pTagColList, int32_t numOfTags, int16_t colId) { + assert(pTagColList != NULL && numOfTags > 0); + + for(int32_t i = 0; i < numOfTags; ++i) { + if (pTagColList[i].colId == colId) { + return &pTagColList[i]; + } + } + + return NULL; +} + void setTagVal(SQueryRuntimeEnv *pRuntimeEnv, void *pTable, void *tsdb) { SQuery *pQuery = pRuntimeEnv->pQuery; SQInfo* pQInfo = GET_QINFO_ADDR(pRuntimeEnv); @@ -2372,16 +2379,10 @@ void setTagVal(SQueryRuntimeEnv *pRuntimeEnv, void *pTable, void *tsdb) { if (pQuery->numOfOutput == 1 && pExprInfo->base.functionId == TSDB_FUNC_TS_COMP) { assert(pExprInfo->base.numOfParams == 1); - // todo refactor extract function. - int16_t type = -1, bytes = -1; - for(int32_t i = 0; i < pQuery->numOfTags; ++i) { - if (pQuery->tagColList[i].colId == pExprInfo->base.arg->argValue.i64) { - type = pQuery->tagColList[i].type; - bytes = pQuery->tagColList[i].bytes; - } - } + int16_t tagColId = pExprInfo->base.arg->argValue.i64; + SColumnInfo* pColInfo = doGetTagColumnInfoById(pQuery->tagColList, pQuery->numOfTags, tagColId); - doSetTagValueInParam(tsdb, pTable, pExprInfo->base.arg->argValue.i64, &pRuntimeEnv->pCtx[0].tag, type, bytes); + doSetTagValueInParam(tsdb, pTable, tagColId, &pRuntimeEnv->pCtx[0].tag, pColInfo->type, pColInfo->bytes); } else { // set tag value, by which the results are aggregated. for (int32_t idx = 0; idx < pQuery->numOfOutput; ++idx) { @@ -2399,20 +2400,14 @@ void setTagVal(SQueryRuntimeEnv *pRuntimeEnv, void *pTable, void *tsdb) { // set the join tag for first column SSqlFuncMsg *pFuncMsg = &pExprInfo->base; - if ((pFuncMsg->functionId == TSDB_FUNC_TS || pFuncMsg->functionId == TSDB_FUNC_PRJ) && pFuncMsg->colInfo.colIndex == PRIMARYKEY_TIMESTAMP_COL_INDEX && - pRuntimeEnv->pTSBuf != NULL) { + if ((pFuncMsg->functionId == TSDB_FUNC_TS || pFuncMsg->functionId == TSDB_FUNC_PRJ) && pRuntimeEnv->pTSBuf != NULL && + pFuncMsg->colInfo.colIndex == PRIMARYKEY_TIMESTAMP_COL_INDEX) { assert(pFuncMsg->numOfParams == 1); - // todo refactor - int16_t type = -1, bytes = -1; - for(int32_t i = 0; i < pQuery->numOfTags; ++i) { - if (pQuery->tagColList[i].colId == pExprInfo->base.arg->argValue.i64) { - type = pQuery->tagColList[i].type; - bytes = pQuery->tagColList[i].bytes; - } - } + int16_t tagColId = pExprInfo->base.arg->argValue.i64; + SColumnInfo* pColInfo = doGetTagColumnInfoById(pQuery->tagColList, pQuery->numOfTags, tagColId); - doSetTagValueInParam(tsdb, pTable, pExprInfo->base.arg->argValue.i64, &pRuntimeEnv->pCtx[0].tag, type, bytes); + doSetTagValueInParam(tsdb, pTable, tagColId, &pRuntimeEnv->pCtx[0].tag, pColInfo->type, pColInfo->bytes); qDebug("QInfo:%p set tag value for join comparison, colId:%" PRId64 ", val:%"PRId64, pQInfo, pExprInfo->base.arg->argValue.i64, pRuntimeEnv->pCtx[0].tag.i64Key) } @@ -4149,6 +4144,7 @@ static int32_t setupQueryHandle(void* tsdb, SQInfo* pQInfo, bool isSTableQuery) } else { pRuntimeEnv->pQueryHandle = tsdbQueryTables(tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo); } + return terrno; } @@ -4174,10 +4170,10 @@ static SFillColInfo* taosCreateFillColInfo(SQuery* pQuery) { } int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bool isSTableQuery) { - int32_t code = TSDB_CODE_SUCCESS; - + qDebug("QInfo:%p start to init qhandle", pQInfo); SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; + int32_t code = TSDB_CODE_SUCCESS; SQuery *pQuery = pQInfo->runtimeEnv.pQuery; pQuery->precision = tsdbGetCfg(tsdb)->precision; @@ -4186,6 +4182,7 @@ int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bo setScanLimitationByResultBuffer(pQuery); changeExecuteScanOrder(pQInfo, false); + code = setupQueryHandle(tsdb, pQInfo, isSTableQuery); if (code != TSDB_CODE_SUCCESS) { return code; @@ -5693,7 +5690,6 @@ static void doUpdateExprColumnIndex(SQuery *pQuery) { } } - static int compareTableIdInfo(const void* a, const void* b) { const STableIdInfo* x = (const STableIdInfo*)a; const STableIdInfo* y = (const STableIdInfo*)b; @@ -5926,6 +5922,7 @@ static int32_t initQInfo(SQueryTableMsg *pQueryMsg, void *tsdb, int32_t vgId, SQ pQuery->window.ekey, pQuery->order.order); setQueryStatus(pQuery, QUERY_COMPLETED); pQInfo->tableqinfoGroupInfo.numOfTables = 0; + sem_post(&pQInfo->dataReady); return TSDB_CODE_SUCCESS; } @@ -6136,16 +6133,17 @@ int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, vo int32_t code = TSDB_CODE_SUCCESS; - char * tagCond = NULL, *tbnameCond = NULL; - SArray * pTableIdList = NULL; - SSqlFuncMsg **pExprMsg = NULL; - SColIndex * pGroupColIndex = NULL; - SColumnInfo* pTagColumnInfo = NULL; - SExprInfo *pExprs = NULL; - SSqlGroupbyExpr *pGroupbyExpr = NULL; + char *tagCond = NULL; + char *tbnameCond = NULL; + SArray *pTableIdList = NULL; + SSqlFuncMsg **pExprMsg = NULL; + SExprInfo *pExprs = NULL; + SColIndex *pGroupColIndex = NULL; + SColumnInfo *pTagColumnInfo = NULL; + SSqlGroupbyExpr *pGroupbyExpr = NULL; - if ((code = convertQueryMsg(pQueryMsg, &pTableIdList, &pExprMsg, &tagCond, &tbnameCond, &pGroupColIndex, &pTagColumnInfo)) != - TSDB_CODE_SUCCESS) { + code = convertQueryMsg(pQueryMsg, &pTableIdList, &pExprMsg, &tagCond, &tbnameCond, &pGroupColIndex, &pTagColumnInfo); + if (code != TSDB_CODE_SUCCESS) { goto _over; } @@ -6172,7 +6170,8 @@ int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, vo bool isSTableQuery = false; STableGroupInfo tableGroupInfo = {0}; - + int64_t st = taosGetTimestampUs(); + if (TSDB_QUERY_HAS_TYPE(pQueryMsg->queryType, TSDB_QUERY_TYPE_TABLE_QUERY)) { STableIdInfo *id = taosArrayGet(pTableIdList, 0); @@ -6182,7 +6181,6 @@ int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, vo } } else if (TSDB_QUERY_HAS_TYPE(pQueryMsg->queryType, TSDB_QUERY_TYPE_MULTITABLE_QUERY|TSDB_QUERY_TYPE_STABLE_QUERY)) { isSTableQuery = true; - // TODO: need a macro from TSDB to check if table is super table // also note there's possibility that only one table in the super table if (!TSDB_QUERY_HAS_TYPE(pQueryMsg->queryType, TSDB_QUERY_TYPE_MULTITABLE_QUERY)) { @@ -6193,11 +6191,12 @@ int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, vo if (pQueryMsg->numOfGroupCols == 1 && !TSDB_COL_IS_TAG(pGroupColIndex->flag)) { numOfGroupByCols = 0; } - + + qDebug("qmsg:%p query stable, uid:%"PRId64", tid:%d", pQueryMsg, id->uid, id->tid); code = tsdbQuerySTableByTagCond(tsdb, id->uid, tagCond, pQueryMsg->tagCondLen, pQueryMsg->tagNameRelType, tbnameCond, &tableGroupInfo, pGroupColIndex, numOfGroupByCols); if (code != TSDB_CODE_SUCCESS) { - qError("qmsg:%p failed to QueryStable, reason: %s", pQueryMsg, tstrerror(code)); + qError("qmsg:%p failed to query stable, reason: %s", pQueryMsg, tstrerror(code)); goto _over; } } else { @@ -6208,6 +6207,9 @@ int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, vo qDebug("qmsg:%p query on %zu tables in one group from client", pQueryMsg, tableGroupInfo.numOfTables); } + + int64_t el = taosGetTimestampUs() - st; + qDebug("qmsg:%p tag filter completed, elapsed time:%"PRId64"us", pQueryMsg, el); } else { assert(0); } @@ -6247,7 +6249,7 @@ _over: *pQInfo = NULL; } - // if failed to add ref for all meters in this query, abort current query + // if failed to add ref for all tables in this query, abort current query return code; } diff --git a/src/query/src/qast.c b/src/query/src/qast.c index da4eb8f3ba..42f9f214f5 100644 --- a/src/query/src/qast.c +++ b/src/query/src/qast.c @@ -678,7 +678,7 @@ static void tQueryIndexlessColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, tstr *name = (tstr*) tsdbGetTableName(*(void**) pData); // todo speed up by using hash - if (pQueryInfo->colIndex == TSDB_TBNAME_COLUMN_INDEX) { + if (pQueryInfo->sch.colId == TSDB_TBNAME_COLUMN_INDEX) { if (pQueryInfo->optr == TSDB_RELATION_IN) { addToResult = pQueryInfo->compare(name, pQueryInfo->q); } else if (pQueryInfo->optr == TSDB_RELATION_LIKE) { @@ -716,7 +716,7 @@ void tExprTreeTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, S } tQueryInfo *pQueryInfo = pExpr->_node.info; - if (pQueryInfo->colIndex == 0 && pQueryInfo->optr != TSDB_RELATION_LIKE) { + if (pQueryInfo->sch.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX && pQueryInfo->optr != TSDB_RELATION_LIKE) { tQueryIndexColumn(pSkipList, pQueryInfo, result); } else { tQueryIndexlessColumn(pSkipList, pQueryInfo, result, param->nodeFilterFn); diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index eb34805de4..8463e791db 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -240,7 +240,7 @@ TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STab pQueryHandle->defaultLoadColumn = getDefaultLoadColumns(pQueryHandle, true); - tsdbDebug("%p total numOfTable:%zu in query", pQueryHandle, taosArrayGetSize(pQueryHandle->pTableCheckInfo)); + tsdbDebug("%p total numOfTable:%zu in query, %p", pQueryHandle, taosArrayGetSize(pQueryHandle->pTableCheckInfo), pQueryHandle->qinfo); tsdbInitDataBlockLoadInfo(&pQueryHandle->dataBlockLoadInfo); tsdbInitCompBlockLoadInfo(&pQueryHandle->compBlockLoadInfo); @@ -1869,7 +1869,6 @@ int32_t tsdbRetrieveDataBlockStatisInfo(TsdbQueryHandleT* pQueryHandle, SDataSta pHandle->statis[i].numOfNull = pBlockInfo->compBlock->numOfRows; } - // todo opt perf SColumnInfo* pColInfo = taosArrayGet(pHandle->pColumns, i); if (pColInfo->type == TSDB_DATA_TYPE_TIMESTAMP) { pHandle->statis[i].min = pBlockInfo->compBlock->keyFirst; @@ -1961,43 +1960,20 @@ static void destroyHelper(void* param) { free(param); } -#define TAG_INVALID_COLUMN_INDEX -2 -static int32_t getTagColumnIndex(STSchema* pTSchema, SSchema* pSchema) { - // filter on table name(TBNAME) - if (strcasecmp(pSchema->name, TSQL_TBNAME_L) == 0) { - return TSDB_TBNAME_COLUMN_INDEX; - } - - for(int32_t i = 0; i < schemaNCols(pTSchema); ++i) { - STColumn* pColumn = &pTSchema->columns[i]; - if (pColumn->bytes == pSchema->bytes && pColumn->type == pSchema->type && pColumn->colId == pSchema->colId) { - return i; - } - } - - return -2; -} - void filterPrepare(void* expr, void* param) { tExprNode* pExpr = (tExprNode*)expr; if (pExpr->_node.info != NULL) { return; } - int32_t i = 0; pExpr->_node.info = calloc(1, sizeof(tQueryInfo)); - STSchema* pTSSchema = (STSchema*) param; - + STSchema* pTSSchema = (STSchema*) param; tQueryInfo* pInfo = pExpr->_node.info; tVariant* pCond = pExpr->_node.pRight->pVal; SSchema* pSchema = pExpr->_node.pLeft->pSchema; - int32_t index = getTagColumnIndex(pTSSchema, pSchema); - assert((index >= 0 && i < TSDB_MAX_TAGS) || (index == TSDB_TBNAME_COLUMN_INDEX) || index == TAG_INVALID_COLUMN_INDEX); - pInfo->sch = *pSchema; - pInfo->colIndex = index; pInfo->optr = pExpr->_node.optr; pInfo->compare = getComparFunc(pSchema->type, pInfo->optr); pInfo->param = pTSSchema; @@ -2143,7 +2119,7 @@ bool indexedNodeFilterFp(const void* pNode, void* param) { char* val = NULL; - if (pInfo->colIndex == TSDB_TBNAME_COLUMN_INDEX) { + if (pInfo->sch.colId == TSDB_TBNAME_COLUMN_INDEX) { val = (char*) TABLE_NAME(pTable); } else { val = tdGetKVRowValOfCol(pTable->tagVal, pInfo->sch.colId); From 9bd890775f9186db34a8675da6e5712842965c18 Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Sat, 18 Jul 2020 15:24:33 +0800 Subject: [PATCH 022/100] fix td-956 --- src/cq/src/cqMain.c | 6 +++--- src/tsdb/src/tsdbMain.c | 3 ++- src/vnode/src/vnodeMain.c | 14 +++++++------- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/src/cq/src/cqMain.c b/src/cq/src/cqMain.c index 51cd471a6b..e0f30166c4 100644 --- a/src/cq/src/cqMain.c +++ b/src/cq/src/cqMain.c @@ -103,9 +103,6 @@ void cqClose(void *handle) { SCqContext *pContext = handle; if (handle == NULL) return; - taosTmrCleanUp(pContext->tmrCtrl); - pContext->tmrCtrl = NULL; - // stop all CQs cqStop(pContext); @@ -125,6 +122,9 @@ void cqClose(void *handle) { pthread_mutex_destroy(&pContext->mutex); + taosTmrCleanUp(pContext->tmrCtrl); + pContext->tmrCtrl = NULL; + cTrace("vgId:%d, CQ is closed", pContext->vgId); free(pContext); } diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index ffaab375a3..e30164592d 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -148,7 +148,7 @@ void tsdbCloseRepo(TSDB_REPO_T *repo, int toCommit) { STsdbRepo *pRepo = (STsdbRepo *)repo; int vgId = REPO_ID(pRepo); - tsdbStopStream(repo); + tsdbStopStream(pRepo); if (toCommit) { tsdbAsyncCommit(pRepo); @@ -1126,6 +1126,7 @@ static void tsdbStartStream(STsdbRepo *pRepo) { } } + static void tsdbStopStream(STsdbRepo *pRepo) { STsdbMeta *pMeta = pRepo->tsdbMeta; diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index 98882e4c3c..249fb428e7 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -340,6 +340,13 @@ void vnodeRelease(void *pVnodeRaw) { tsdbCloseRepo(pVnode->tsdb, 1); pVnode->tsdb = NULL; + // stop continuous query + if (pVnode->cq) { + void *cq = pVnode->cq; + pVnode->cq = NULL; + cqClose(cq); + } + if (pVnode->wal) walClose(pVnode->wal); pVnode->wal = NULL; @@ -511,13 +518,6 @@ static void vnodeCleanUp(SVnodeObj *pVnode) { syncStop(sync); } - // stop continuous query - if (pVnode->cq) { - void *cq = pVnode->cq; - pVnode->cq = NULL; - cqClose(cq); - } - vTrace("vgId:%d, vnode will cleanup, refCount:%d", pVnode->vgId, pVnode->refCount); // release local resources only after cutting off outside connections From 246ea1df248e8721bad64f40fd57ed0ec0db263c Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sat, 18 Jul 2020 10:31:31 +0000 Subject: [PATCH 023/100] modify from ip to ep --- src/client/inc/tsclient.h | 4 +- src/client/src/tscSchemaUtil.c | 8 +-- src/client/src/tscServer.c | 95 +++++++++++++++------------------- src/client/src/tscSql.c | 2 +- src/client/src/tscUtil.c | 2 +- src/inc/trpc.h | 6 +-- 6 files changed, 53 insertions(+), 64 deletions(-) diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index 83f9165d35..b5455ed1fb 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -55,8 +55,8 @@ typedef struct STableComInfo { typedef struct SCMCorVgroupInfo { int32_t version; int8_t inUse; - int8_t numOfIps; - SIpAddr ipAddr[TSDB_MAX_REPLICA]; + int8_t numOfEps; + SEpAddr epAddr[TSDB_MAX_REPLICA]; } SCMCorVgroupInfo; typedef struct STableMeta { diff --git a/src/client/src/tscSchemaUtil.c b/src/client/src/tscSchemaUtil.c index 52342b3650..9b8f48b109 100644 --- a/src/client/src/tscSchemaUtil.c +++ b/src/client/src/tscSchemaUtil.c @@ -143,10 +143,10 @@ struct SSchema tscGetTbnameColumnSchema() { static void tscInitCorVgroupInfo(SCMCorVgroupInfo *corVgroupInfo, SCMVgroupInfo *vgroupInfo) { corVgroupInfo->version = 0; corVgroupInfo->inUse = 0; - corVgroupInfo->numOfIps = vgroupInfo->numOfIps; - for (int32_t i = 0; i < corVgroupInfo->numOfIps; i++) { - strncpy(corVgroupInfo->ipAddr[i].fqdn, vgroupInfo->ipAddr[i].fqdn, TSDB_FQDN_LEN); - corVgroupInfo->ipAddr[i].port = vgroupInfo->ipAddr[i].port; + corVgroupInfo->numOfEps = vgroupInfo->numOfEps; + for (int32_t i = 0; i < corVgroupInfo->numOfEps; i++) { + strncpy(corVgroupInfo->epAddr[i].fqdn, vgroupInfo->epAddr[i].fqdn, TSDB_FQDN_LEN); + corVgroupInfo->epAddr[i].port = vgroupInfo->epAddr[i].port; } } STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg, size_t* size) { diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 3d78063bed..d9922b8718 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -53,64 +53,53 @@ static void tscSetDnodeEpSet(SSqlObj* pSql, SCMVgroupInfo* pVgroupInfo) { return; } - pEpList->numOfEps = pVgroupInfo->numOfEps; + pEpSet->numOfEps = pVgroupInfo->numOfEps; for(int32_t i = 0; i < pVgroupInfo->numOfEps; ++i) { - strcpy(pEpList->fqdn[i], pVgroupInfo->epAddr[i].fqdn); - pEpList->port[i] = pVgroupInfo->epAddr[i].port; + strcpy(pEpSet->fqdn[i], pVgroupInfo->epAddr[i].fqdn); + pEpSet->port[i] = pVgroupInfo->epAddr[i].port; } } -void tscIpSetCopy(SRpcIpSet *dst, SRpcIpSet *src) { - dst->numOfIps = src->numOfIps; - dst->inUse = src->inUse; - for (int32_t i = 0; i < src->numOfIps; ++i) { - dst->port[i] = src->port[i]; - strncpy(dst->fqdn[i], src->fqdn[i], TSDB_FQDN_LEN); - } -} -static void tscDumpMgmtIpSet(SRpcIpSet *ipSet) { - taosCorBeginRead(&tscMgmtIpSet.version); - SRpcIpSet* src = &tscMgmtIpSet.ipSet; - tscIpSetCopy(ipSet, src); - taosCorEndRead(&tscMgmtIpSet.version); +static void tscDumpMgmtEpSet(SRpcEpSet *epSet) { + taosCorBeginRead(&tscMgmtEpSet.version); + *epSet = tscMgmtEpSet.epSet; + taosCorEndRead(&tscMgmtEpSet.version); } -static void tscIpSetHtons(SRpcIpSet *s) { - for (int32_t i = 0; i < s->numOfIps; i++) { +static void tscEpSetHtons(SRpcEpSet *s) { + for (int32_t i = 0; i < s->numOfEps; i++) { s->port[i] = htons(s->port[i]); } } -bool tscIpSetIsEqual(SRpcIpSet *s1, SRpcIpSet *s2) { - if (s1->numOfIps != s2->numOfIps || s1->inUse != s2->inUse) { +bool tscEpSetIsEqual(SRpcEpSet *s1, SRpcEpSet *s2) { + if (s1->numOfEps != s2->numOfEps || s1->inUse != s2->inUse) { return false; } - for (int32_t i = 0; i < s1->numOfIps; i++) { + for (int32_t i = 0; i < s1->numOfEps; i++) { if (s1->port[i] != s2->port[i] || strncmp(s1->fqdn[i], s2->fqdn[i], TSDB_FQDN_LEN) != 0) return false; } return true; } -void tscUpdateMgmtIpList(SRpcIpSet *pIpSet) { +void tscUpdateMgmtEpSet(SRpcEpSet *pEpSet) { // no need to update if equal - taosCorBeginWrite(&tscMgmtIpSet.version); - // or copy directly, tscMgmtIpSet.ipSet = *pIpSet - SRpcIpSet *mgmtIpSet = &tscMgmtIpSet.ipSet; - tscIpSetCopy(mgmtIpSet, pIpSet); - taosCorEndWrite(&tscMgmtIpSet.version); + taosCorBeginWrite(&tscMgmtEpSet.version); + tscMgmtEpSet.epSet = *pEpSet; + taosCorEndWrite(&tscMgmtEpSet.version); } -static void tscDumpIpSetFromVgroupInfo(SCMCorVgroupInfo *pVgroupInfo, SRpcIpSet *pIpSet) { +static void tscDumpEpSetFromVgroupInfo(SCMCorVgroupInfo *pVgroupInfo, SRpcEpSet *pEpSet) { if (pVgroupInfo == NULL) { return;} taosCorBeginRead(&pVgroupInfo->version); int8_t inUse = pVgroupInfo->inUse; - pIpSet->inUse = (inUse >= 0 && inUse < TSDB_MAX_REPLICA) ? inUse: 0; - pIpSet->numOfIps = pVgroupInfo->numOfIps; - for (int32_t i = 0; i < pVgroupInfo->numOfIps; ++i) { - strncpy(pIpSet->fqdn[i], pVgroupInfo->ipAddr[i].fqdn, TSDB_FQDN_LEN); - pIpSet->port[i] = pVgroupInfo->ipAddr[i].port; + pEpSet->inUse = (inUse >= 0 && inUse < TSDB_MAX_REPLICA) ? inUse: 0; + pEpSet->numOfEps = pVgroupInfo->numOfEps; + for (int32_t i = 0; i < pVgroupInfo->numOfEps; ++i) { + strncpy(pEpSet->fqdn[i], pVgroupInfo->epAddr[i].fqdn, TSDB_FQDN_LEN); + pEpSet->port[i] = pVgroupInfo->epAddr[i].port; } taosCorEndRead(&pVgroupInfo->version); } -static void tscUpdateVgroupInfo(SSqlObj *pObj, SRpcIpSet *pIpSet) { +static void tscUpdateVgroupInfo(SSqlObj *pObj, SRpcEpSet *pEpSet) { SSqlCmd *pCmd = &pObj->cmd; STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); if (pTableMetaInfo == NULL || pTableMetaInfo->pTableMeta == NULL) { return;} @@ -130,7 +119,7 @@ void tscPrintMgmtEp() { SRpcEpSet dump; tscDumpMgmtEpSet(&dump); if (dump.numOfEps <= 0) { - tscError("invalid mnode EP list:%d", dump.numOfEPs); + tscError("invalid mnode EP list:%d", dump.numOfEps); } else { for (int i = 0; i < dump.numOfEps; ++i) { tscDebug("mnode index:%d %s:%d", i, dump.fqdn[i], dump.port[i]); @@ -166,10 +155,10 @@ void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) { if (code == 0) { SCMHeartBeatRsp *pRsp = (SCMHeartBeatRsp *)pRes->pRsp; - SRpcEpSet * pEpList = &pRsp->epList; - if (pEpList->numOfEps > 0) { - tscEpSetHtons(pEpList); - tscUpdateMgmtEpList(pEpList); + SRpcEpSet * epSet = &pRsp->epSet; + if (epSet->numOfEps > 0) { + tscEpSetHtons(epSet); + tscUpdateMgmtEpSet(epSet); } pSql->pTscObj->connId = htonl(pRsp->connId); @@ -242,7 +231,7 @@ int tscSendMsgToServer(SSqlObj *pSql) { // set the mgmt ip list if (pSql->cmd.command >= TSDB_SQL_MGMT) { - tscDumpMgmtEpSet(&pSql->epList); + tscDumpMgmtEpSet(&pSql->epSet); } memcpy(pMsg, pSql->cmd.payload, pSql->cmd.payloadLen); @@ -296,11 +285,11 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) { if (pEpSet) { //SRpcEpSet dump; tscEpSetHtons(pEpSet); - if (tscEpSetIsEqual(&pSql->epList, pEpSet)) { + if (tscEpSetIsEqual(&pSql->epSet, pEpSet)) { if(pCmd->command < TSDB_SQL_MGMT) { tscUpdateVgroupInfo(pSql, pEpSet); } else { - tscUpdateMgmtEpList(pEpSet); + tscUpdateMgmtEpSet(pEpSet); } } } @@ -589,7 +578,7 @@ int tscBuildSubmitMsg(SSqlObj *pSql, SSqlInfo *pInfo) { // pSql->cmd.payloadLen is set during copying data into payload pSql->cmd.msgType = TSDB_MSG_TYPE_SUBMIT; - tscDumpEpSetFromVgroupInfo(&pTableMeta->corVgroupInfo, &pSql->epList); + tscDumpEpSetFromVgroupInfo(&pTableMeta->corVgroupInfo, &pSql->epSet); tscDebug("%p build submit msg, vgId:%d numOfTables:%d numberOfEP:%d", pSql, vgId, pSql->cmd.numOfTablesInSubmit, pSql->epSet.numOfEps); @@ -631,7 +620,7 @@ static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char } else { pVgroupInfo = &pTableMeta->vgroupInfo; } - tscSetDnodeEpList(pSql, pVgroupInfo); + tscSetDnodeEpSet(pSql, pVgroupInfo); if (pVgroupInfo != NULL) { pQueryMsg->head.vgId = htonl(pVgroupInfo->vgId); @@ -654,7 +643,7 @@ static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char SVgroupTableInfo* pTableIdList = taosArrayGet(pTableMetaInfo->pVgroupTables, index); // set the vgroup info - tscSetDnodeEpList(pSql, &pTableIdList->vgInfo); + tscSetDnodeEpSet(pSql, &pTableIdList->vgInfo); pQueryMsg->head.vgId = htonl(pTableIdList->vgInfo.vgId); int32_t numOfTables = taosArrayGetSize(pTableIdList->itemList); @@ -1200,11 +1189,11 @@ int32_t tscBuildShowMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pShowMsg->payloadLen = htons(pPattern->n); } } else { - SSQLToken *pIpAddr = &pShowInfo->prefix; - assert(pIpAddr->n > 0 && pIpAddr->type > 0); + SSQLToken *pEpAddr = &pShowInfo->prefix; + assert(pEpAddr->n > 0 && pEpAddr->type > 0); - strncpy(pShowMsg->payload, pIpAddr->z, pIpAddr->n); - pShowMsg->payloadLen = htons(pIpAddr->n); + strncpy(pShowMsg->payload, pEpAddr->z, pEpAddr->n); + pShowMsg->payloadLen = htons(pEpAddr->n); } pCmd->payloadLen = sizeof(SCMShowMsg) + pShowMsg->payloadLen; @@ -1387,7 +1376,7 @@ int tscBuildUpdateTagMsg(SSqlObj* pSql, SSqlInfo *pInfo) { SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - tscDumpEpSetFromVgroupInfo(&pTableMetaInfo->pTableMeta->corVgroupInfo, &pSql->epList); + tscDumpEpSetFromVgroupInfo(&pTableMetaInfo->pTableMeta->corVgroupInfo, &pSql->epSet); return TSDB_CODE_SUCCESS; } @@ -2011,9 +2000,9 @@ int tscProcessConnectRsp(SSqlObj *pSql) { assert(len <= sizeof(pObj->db)); tstrncpy(pObj->db, temp, sizeof(pObj->db)); - if (pConnect->epList.numOfEps > 0) { - tscEpSetHtons(&pConnect->epList); - tscUpdateMgmtEpList(&pConnect->epList); + if (pConnect->epSet.numOfEps > 0) { + tscEpSetHtons(&pConnect->epSet); + tscUpdateMgmtEpSet(&pConnect->epSet); } strcpy(pObj->sversion, pConnect->serverVersion); diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index 59b506a454..5848b7b82f 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -62,7 +62,7 @@ SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pass, con } if (ip) { - if (tscSetMgmtEpListFromCfg(ip, NULL) < 0) return NULL; + if (tscSetMgmtEpSetFromCfg(ip, NULL) < 0) return NULL; if (port) tscMgmtEpSet.epSet.port[0] = port; } diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index c3cae58d8a..957bdeeb7f 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -2145,7 +2145,7 @@ char* strdup_throw(const char* str) { return p; } -int tscSetMgmtEpListFromCfg(const char *first, const char *second) { +int tscSetMgmtEpSetFromCfg(const char *first, const char *second) { // init mgmt ip set tscMgmtEpSet.version = 0; SRpcEpSet *mgmtEpSet = &(tscMgmtEpSet.epSet); diff --git a/src/inc/trpc.h b/src/inc/trpc.h index 41ed98fd70..bdee917b5e 100644 --- a/src/inc/trpc.h +++ b/src/inc/trpc.h @@ -35,10 +35,10 @@ typedef struct SRpcEpSet { char fqdn[TSDB_MAX_REPLICA][TSDB_FQDN_LEN]; } SRpcEpSet; -typedef struct SRpcCorIpSet { +typedef struct SRpcCorEpSet { int32_t version; - SRpcIpSet ipSet; -} SRpcCorIpSet; + SRpcEpSet epSet; +} SRpcCorEpSet; typedef struct SRpcConnInfo { uint32_t clientIp; From fdfa789cbcffb9f621badec2d09e6c8ca3ebda10 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 18 Jul 2020 12:13:01 +0000 Subject: [PATCH 024/100] replace ipList and ipSet to epSet --- src/client/inc/tscUtil.h | 2 +- src/client/inc/tsclient.h | 6 +- src/client/src/tscSQLParser.c | 4 +- src/client/src/tscServer.c | 102 ++++++++++----------- src/client/src/tscSql.c | 4 +- src/client/src/tscSubquery.c | 14 +-- src/client/src/tscSystem.c | 6 +- src/client/src/tscUtil.c | 16 ++-- src/dnode/inc/dnodeMgmt.h | 4 +- src/dnode/src/dnodeMgmt.c | 104 +++++++++++----------- src/dnode/src/dnodePeer.c | 24 ++--- src/dnode/src/dnodeShell.c | 4 +- src/inc/dnode.h | 6 +- src/inc/taosmsg.h | 10 +-- src/inc/trpc.h | 14 +-- src/kit/taosmigrate/taosmigrate.c | 8 +- src/kit/taosmigrate/taosmigrate.h | 2 +- src/kit/taosmigrate/taosmigrateDnodeCfg.c | 36 ++++---- src/mnode/inc/mnodeMnode.h | 6 +- src/mnode/inc/mnodeVgroup.h | 6 +- src/mnode/src/mnodeDnode.c | 10 +-- src/mnode/src/mnodeMnode.c | 48 +++++----- src/mnode/src/mnodePeer.c | 14 +-- src/mnode/src/mnodeRead.c | 14 +-- src/mnode/src/mnodeSdb.c | 2 +- src/mnode/src/mnodeShow.c | 4 +- src/mnode/src/mnodeTable.c | 28 +++--- src/mnode/src/mnodeVgroup.c | 58 ++++++------ src/mnode/src/mnodeWrite.c | 16 ++-- src/rpc/src/rpcMain.c | 60 ++++++------- src/rpc/test/rclient.c | 30 +++---- src/rpc/test/rsclient.c | 26 +++--- src/rpc/test/rserver.c | 2 +- 33 files changed, 345 insertions(+), 345 deletions(-) diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index 6a5c2d0099..786133a8f3 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -268,7 +268,7 @@ bool hasMoreClauseToTry(SSqlObj* pSql); void tscTryQueryNextVnode(SSqlObj *pSql, __async_cb_func_t fp); void tscAsyncQuerySingleRowForNextVnode(void *param, TAOS_RES *tres, int numOfRows); void tscTryQueryNextClause(SSqlObj* pSql, __async_cb_func_t fp); -int tscSetMgmtIpListFromCfg(const char *first, const char *second); +int tscSetMgmtEpSetFromCfg(const char *first, const char *second); void* malloc_throw(size_t size); void* calloc_throw(size_t nmemb, size_t size); diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index 1b30c4ffca..ef8184990d 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -306,7 +306,7 @@ typedef struct SSqlObj { char * sqlstr; char retry; char maxRetry; - SRpcIpSet ipList; + SRpcEpSet epSet; char listed; tsem_t rspSem; SSqlCmd cmd; @@ -350,7 +350,7 @@ void tscInitMsgsFp(); int tsParseSql(SSqlObj *pSql, bool initial); -void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcIpSet *pIpSet); +void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet); int tscProcessSql(SSqlObj *pSql); int tscRenewTableMeta(SSqlObj *pSql, char *tableId); @@ -456,7 +456,7 @@ extern void * tscQhandle; extern int tscKeepConn[]; extern int tsInsertHeadSize; extern int tscNumOfThreads; -extern SRpcIpSet tscMgmtIpSet; +extern SRpcEpSet tscMgmtEpSet; extern int (*tscBuildMsg[TSDB_SQL_MAX])(SSqlObj *pSql, SSqlInfo *pInfo); diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index f9b12f3f6d..b97e486449 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -2355,9 +2355,9 @@ bool validateIpAddress(const char* ip, size_t size) { strncpy(tmp, ip, size); - in_addr_t ipAddr = inet_addr(tmp); + in_addr_t epAddr = inet_addr(tmp); - return ipAddr != INADDR_NONE; + return epAddr != INADDR_NONE; } int32_t tscTansformSQLFuncForSTableQuery(SQueryInfo* pQueryInfo) { diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 99ac2249e0..398dea9d09 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -29,8 +29,8 @@ #define TSC_MGMT_VNODE 999 -SRpcIpSet tscMgmtIpSet; -SRpcIpSet tscDnodeIpSet; +SRpcEpSet tscMgmtEpSet; +SRpcEpSet tscDnodeEpSet; int (*tscBuildMsg[TSDB_SQL_MAX])(SSqlObj *pSql, SSqlInfo *pInfo) = {0}; @@ -44,44 +44,44 @@ void tscSaveSubscriptionProgress(void* sub); static int32_t minMsgSize() { return tsRpcHeadSize + 100; } -static void tscSetDnodeIpList(SSqlObj* pSql, SCMVgroupInfo* pVgroupInfo) { - SRpcIpSet* pIpList = &pSql->ipList; - pIpList->inUse = 0; +static void tscSetDnodeEpSet(SSqlObj* pSql, SCMVgroupInfo* pVgroupInfo) { + SRpcEpSet* pEpSet = &pSql->epSet; + pEpSet->inUse = 0; if (pVgroupInfo == NULL) { - pIpList->numOfIps = 0; + pEpSet->numOfEps = 0; return; } - pIpList->numOfIps = pVgroupInfo->numOfIps; - for(int32_t i = 0; i < pVgroupInfo->numOfIps; ++i) { - strcpy(pIpList->fqdn[i], pVgroupInfo->ipAddr[i].fqdn); - pIpList->port[i] = pVgroupInfo->ipAddr[i].port; + pEpSet->numOfEps = pVgroupInfo->numOfEps; + for(int32_t i = 0; i < pVgroupInfo->numOfEps; ++i) { + strcpy(pEpSet->fqdn[i], pVgroupInfo->epAddr[i].fqdn); + pEpSet->port[i] = pVgroupInfo->epAddr[i].port; } } -void tscPrintMgmtIp() { - if (tscMgmtIpSet.numOfIps <= 0) { - tscError("invalid mnode IP list:%d", tscMgmtIpSet.numOfIps); +void tscPrintMgmtEp() { + if (tscMgmtEpSet.numOfEps <= 0) { + tscError("invalid mnode EP list:%d", tscMgmtEpSet.numOfEps); } else { - for (int i = 0; i < tscMgmtIpSet.numOfIps; ++i) { - tscDebug("mnode index:%d %s:%d", i, tscMgmtIpSet.fqdn[i], tscMgmtIpSet.port[i]); + for (int i = 0; i < tscMgmtEpSet.numOfEps; ++i) { + tscDebug("mnode index:%d %s:%d", i, tscMgmtEpSet.fqdn[i], tscMgmtEpSet.port[i]); } } } -void tscSetMgmtIpList(SRpcIpSet *pIpList) { - tscMgmtIpSet.numOfIps = pIpList->numOfIps; - tscMgmtIpSet.inUse = pIpList->inUse; - for (int32_t i = 0; i < tscMgmtIpSet.numOfIps; ++i) { - tscMgmtIpSet.port[i] = htons(pIpList->port[i]); +void tscSetMgmtEpSet(SRpcEpSet *pEpSet) { + tscMgmtEpSet.numOfEps = pEpSet->numOfEps; + tscMgmtEpSet.inUse = pEpSet->inUse; + for (int32_t i = 0; i < tscMgmtEpSet.numOfEps; ++i) { + tscMgmtEpSet.port[i] = htons(pEpSet->port[i]); } } -void tscUpdateIpSet(void *ahandle, SRpcIpSet *pIpSet) { - tscMgmtIpSet = *pIpSet; - tscDebug("mnode IP list is changed for ufp is called, numOfIps:%d inUse:%d", tscMgmtIpSet.numOfIps, tscMgmtIpSet.inUse); - for (int32_t i = 0; i < tscMgmtIpSet.numOfIps; ++i) { - tscDebug("index:%d fqdn:%s port:%d", i, tscMgmtIpSet.fqdn[i], tscMgmtIpSet.port[i]); +void tscUpdateEpSet(void *ahandle, SRpcEpSet *pEpSet) { + tscMgmtEpSet = *pEpSet; + tscDebug("mnode EP list is changed for ufp is called, numOfEps:%d inUse:%d", tscMgmtEpSet.numOfEps, tscMgmtEpSet.inUse); + for (int32_t i = 0; i < tscMgmtEpSet.numOfEps; ++i) { + tscDebug("index:%d fqdn:%s port:%d", i, tscMgmtEpSet.fqdn[i], tscMgmtEpSet.port[i]); } } @@ -95,7 +95,7 @@ void tscUpdateIpSet(void *ahandle, SRpcIpSet *pIpSet) { UNUSED_FUNC static int32_t tscGetMgmtConnMaxRetryTimes() { int32_t factor = 2; - return tscMgmtIpSet.numOfIps * factor; + return tscMgmtEpSet.numOfEps * factor; } void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) { @@ -111,9 +111,9 @@ void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) { if (code == 0) { SCMHeartBeatRsp *pRsp = (SCMHeartBeatRsp *)pRes->pRsp; - SRpcIpSet * pIpList = &pRsp->ipList; - if (pIpList->numOfIps > 0) - tscSetMgmtIpList(pIpList); + SRpcEpSet * pEpSet = &pRsp->epSet; + if (pEpSet->numOfEps > 0) + tscSetMgmtEpSet(pEpSet); pSql->pTscObj->connId = htonl(pRsp->connId); @@ -185,7 +185,7 @@ int tscSendMsgToServer(SSqlObj *pSql) { // set the mgmt ip list if (pSql->cmd.command >= TSDB_SQL_MGMT) { - pSql->ipList = tscMgmtIpSet; + pSql->epSet = tscMgmtEpSet; } memcpy(pMsg, pSql->cmd.payload, pSql->cmd.payloadLen); @@ -203,11 +203,11 @@ int tscSendMsgToServer(SSqlObj *pSql) { // Otherwise, the pSql object may have been released already during the response function, which is // processMsgFromServer function. In the meanwhile, the assignment of the rpc context to sql object will absolutely // cause crash. - rpcSendRequest(pObj->pDnodeConn, &pSql->ipList, &rpcMsg); + rpcSendRequest(pObj->pDnodeConn, &pSql->epSet, &rpcMsg); return TSDB_CODE_SUCCESS; } -void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcIpSet *pIpSet) { +void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) { SSqlObj *pSql = (SSqlObj *)rpcMsg->ahandle; if (pSql == NULL || pSql->signature != pSql) { tscError("%p sql is already released", pSql); @@ -237,9 +237,9 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcIpSet *pIpSet) { } if (pCmd->command < TSDB_SQL_MGMT) { - if (pIpSet) pSql->ipList = *pIpSet; + if (pEpSet) pSql->epSet = *pEpSet; } else { - if (pIpSet) tscMgmtIpSet = *pIpSet; + if (pEpSet) tscMgmtEpSet = *pEpSet; } if (rpcMsg->pCont == NULL) { @@ -421,7 +421,7 @@ int tscProcessSql(SSqlObj *pSql) { return pSql->res.code; } } else if (pCmd->command < TSDB_SQL_LOCAL) { - pSql->ipList = tscMgmtIpSet; + pSql->epSet = tscMgmtEpSet; } else { // local handler return (*tscProcessMsgRsp[pCmd->command])(pSql); } @@ -525,10 +525,10 @@ int tscBuildSubmitMsg(SSqlObj *pSql, SSqlInfo *pInfo) { // pSql->cmd.payloadLen is set during copying data into payload pSql->cmd.msgType = TSDB_MSG_TYPE_SUBMIT; - tscSetDnodeIpList(pSql, &pTableMeta->vgroupInfo); + tscSetDnodeEpSet(pSql, &pTableMeta->vgroupInfo); - tscDebug("%p build submit msg, vgId:%d numOfTables:%d numberOfIP:%d", pSql, vgId, pSql->cmd.numOfTablesInSubmit, - pSql->ipList.numOfIps); + tscDebug("%p build submit msg, vgId:%d numOfTables:%d numberOfEP:%d", pSql, vgId, pSql->cmd.numOfTablesInSubmit, + pSql->epSet.numOfEps); return TSDB_CODE_SUCCESS; } @@ -568,7 +568,7 @@ static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char pVgroupInfo = &pTableMeta->vgroupInfo; } - tscSetDnodeIpList(pSql, pVgroupInfo); + tscSetDnodeEpSet(pSql, pVgroupInfo); if (pVgroupInfo != NULL) { pQueryMsg->head.vgId = htonl(pVgroupInfo->vgId); } @@ -580,7 +580,7 @@ static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char pQueryMsg->numOfTables = htonl(1); // set the number of tables pMsg += sizeof(STableIdInfo); - } else { // it is a subquery of the super table query, this IP info is acquired from vgroupInfo + } else { // it is a subquery of the super table query, this EP info is acquired from vgroupInfo int32_t index = pTableMetaInfo->vgroupIndex; int32_t numOfVgroups = taosArrayGetSize(pTableMetaInfo->pVgroupTables); assert(index >= 0 && index < numOfVgroups); @@ -590,7 +590,7 @@ static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char SVgroupTableInfo* pTableIdList = taosArrayGet(pTableMetaInfo->pVgroupTables, index); // set the vgroup info - tscSetDnodeIpList(pSql, &pTableIdList->vgInfo); + tscSetDnodeEpSet(pSql, &pTableIdList->vgInfo); pQueryMsg->head.vgId = htonl(pTableIdList->vgInfo.vgId); int32_t numOfTables = taosArrayGetSize(pTableIdList->itemList); @@ -1323,7 +1323,7 @@ int tscBuildUpdateTagMsg(SSqlObj* pSql, SSqlInfo *pInfo) { SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - tscSetDnodeIpList(pSql, &pTableMetaInfo->pTableMeta->vgroupInfo); + tscSetDnodeEpSet(pSql, &pTableMetaInfo->pTableMeta->vgroupInfo); return TSDB_CODE_SUCCESS; } @@ -1658,8 +1658,8 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) { pMetaMsg->contLen = htons(pMetaMsg->contLen); pMetaMsg->numOfColumns = htons(pMetaMsg->numOfColumns); - if (pMetaMsg->sid < 0 || pMetaMsg->vgroup.numOfIps < 0) { - tscError("invalid meter vgId:%d, sid%d", pMetaMsg->vgroup.numOfIps, pMetaMsg->sid); + if (pMetaMsg->sid < 0 || pMetaMsg->vgroup.numOfEps < 0) { + tscError("invalid meter vgId:%d, sid%d", pMetaMsg->vgroup.numOfEps, pMetaMsg->sid); return TSDB_CODE_TSC_INVALID_VALUE; } @@ -1673,8 +1673,8 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) { return TSDB_CODE_TSC_INVALID_VALUE; } - for (int i = 0; i < pMetaMsg->vgroup.numOfIps; ++i) { - pMetaMsg->vgroup.ipAddr[i].port = htons(pMetaMsg->vgroup.ipAddr[i].port); + for (int i = 0; i < pMetaMsg->vgroup.numOfEps; ++i) { + pMetaMsg->vgroup.epAddr[i].port = htons(pMetaMsg->vgroup.epAddr[i].port); } SSchema* pSchema = pMetaMsg->schema; @@ -1850,10 +1850,10 @@ int tscProcessSTableVgroupRsp(SSqlObj *pSql) { SCMVgroupInfo *pVgroups = &pInfo->vgroupList->vgroups[j]; pVgroups->vgId = htonl(pVgroups->vgId); - assert(pVgroups->numOfIps >= 1); + assert(pVgroups->numOfEps >= 1); - for (int32_t k = 0; k < pVgroups->numOfIps; ++k) { - pVgroups->ipAddr[k].port = htons(pVgroups->ipAddr[k].port); + for (int32_t k = 0; k < pVgroups->numOfEps; ++k) { + pVgroups->epAddr[k].port = htons(pVgroups->epAddr[k].port); } pMsg += size; @@ -1946,8 +1946,8 @@ int tscProcessConnectRsp(SSqlObj *pSql) { assert(len <= sizeof(pObj->db)); tstrncpy(pObj->db, temp, sizeof(pObj->db)); - if (pConnect->ipList.numOfIps > 0) - tscSetMgmtIpList(&pConnect->ipList); + if (pConnect->epSet.numOfEps > 0) + tscSetMgmtEpSet(&pConnect->epSet); strcpy(pObj->sversion, pConnect->serverVersion); pObj->writeAuth = pConnect->writeAuth; diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index d5160cd3c6..6a14d3a65e 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -62,8 +62,8 @@ SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pass, con } if (ip) { - if (tscSetMgmtIpListFromCfg(ip, NULL) < 0) return NULL; - if (port) tscMgmtIpSet.port[0] = port; + if (tscSetMgmtEpSetFromCfg(ip, NULL) < 0) return NULL; + if (port) tscMgmtEpSet.port[0] = port; } void *pDnodeConn = NULL; diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index faa4a2488a..1dbc52efb0 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -458,7 +458,7 @@ void tscBuildVgroupTableInfo(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo, SArr break; } } - assert(info.vgInfo.numOfIps != 0); + assert(info.vgInfo.numOfEps != 0); vgTables = taosArrayInit(4, sizeof(STableIdInfo)); info.itemList = vgTables; @@ -1600,8 +1600,8 @@ static void tscAllDataRetrievedFromDnode(SRetrieveSupport *trsupport, SSqlObj* p // data in from current vnode is stored in cache and disk uint32_t numOfRowsFromSubquery = trsupport->pExtMemBuffer[idx]->numOfTotalElems + trsupport->localBuffer->num; - tscDebug("%p sub:%p all data retrieved from ip:%s, vgId:%d, numOfRows:%d, orderOfSub:%d", pParentSql, pSql, - pTableMetaInfo->vgroupList->vgroups[0].ipAddr[0].fqdn, pTableMetaInfo->vgroupList->vgroups[0].vgId, + tscDebug("%p sub:%p all data retrieved from ep:%s, vgId:%d, numOfRows:%d, orderOfSub:%d", pParentSql, pSql, + pTableMetaInfo->vgroupList->vgroups[0].epAddr[0].fqdn, pTableMetaInfo->vgroupList->vgroups[0].vgId, numOfRowsFromSubquery, idx); tColModelCompact(pDesc->pColumnModel, trsupport->localBuffer, pDesc->pColumnModel->capacity); @@ -1719,8 +1719,8 @@ static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfR assert(pRes->numOfRows == numOfRows); int64_t num = atomic_add_fetch_64(&pState->numOfRetrievedRows, numOfRows); - tscDebug("%p sub:%p retrieve numOfRows:%" PRId64 " totalNumOfRows:%" PRIu64 " from ip:%s, orderOfSub:%d", pParentSql, pSql, - pRes->numOfRows, pState->numOfRetrievedRows, pSql->ipList.fqdn[pSql->ipList.inUse], idx); + tscDebug("%p sub:%p retrieve numOfRows:%" PRId64 " totalNumOfRows:%" PRIu64 " from ep:%s, orderOfSub:%d", pParentSql, pSql, + pRes->numOfRows, pState->numOfRetrievedRows, pSql->epSet.fqdn[pSql->epSet.inUse], idx); if (num > tsMaxNumOfOrderedResults && tscIsProjectionQueryOnSTable(pQueryInfo, 0)) { tscError("%p sub:%p num of OrderedRes is too many, max allowed:%" PRId32 " , current:%" PRId64, @@ -1828,8 +1828,8 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) { return; } - tscTrace("%p sub:%p query complete, ip:%s, vgId:%d, orderOfSub:%d, retrieve data", trsupport->pParentSql, pSql, - pVgroup->ipAddr[0].fqdn, pVgroup->vgId, trsupport->subqueryIndex); + tscTrace("%p sub:%p query complete, ep:%s, vgId:%d, orderOfSub:%d, retrieve data", trsupport->pParentSql, pSql, + pVgroup->epAddr[0].fqdn, pVgroup->vgId, trsupport->subqueryIndex); if (pSql->res.qhandle == 0) { // qhandle is NULL, code is TSDB_CODE_SUCCESS means no results generated from this vnode tscRetrieveFromDnodeCallBack(param, pSql, 0); diff --git a/src/client/src/tscSystem.c b/src/client/src/tscSystem.c index 82cc8cc225..7722acf0d0 100644 --- a/src/client/src/tscSystem.c +++ b/src/client/src/tscSystem.c @@ -41,7 +41,7 @@ int tscNumOfThreads; static pthread_once_t tscinit = PTHREAD_ONCE_INIT; void taosInitNote(int numOfNoteLines, int maxNotes, char* lable); -void tscUpdateIpSet(void *ahandle, SRpcIpSet *pIpSet); +void tscUpdateEpSet(void *ahandle, SRpcEpSet *pEpSet); void tscCheckDiskUsage(void *UNUSED_PARAM(para), void* UNUSED_PARAM(param)) { taosGetDisk(); @@ -116,8 +116,8 @@ void taos_init_imp() { taosInitNote(tsNumOfLogLines / 10, 1, (char*)"tsc_note"); } - if (tscSetMgmtIpListFromCfg(tsFirst, tsSecond) < 0) { - tscError("failed to init mnode IP list"); + if (tscSetMgmtEpSetFromCfg(tsFirst, tsSecond) < 0) { + tscError("failed to init mnode EP list"); return; } diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 49d15f6499..4e6133663e 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -2145,17 +2145,17 @@ char* strdup_throw(const char* str) { return p; } -int tscSetMgmtIpListFromCfg(const char *first, const char *second) { - tscMgmtIpSet.numOfIps = 0; - tscMgmtIpSet.inUse = 0; +int tscSetMgmtEpSetFromCfg(const char *first, const char *second) { + tscMgmtEpSet.numOfEps = 0; + tscMgmtEpSet.inUse = 0; if (first && first[0] != 0) { if (strlen(first) >= TSDB_EP_LEN) { terrno = TSDB_CODE_TSC_INVALID_FQDN; return -1; } - taosGetFqdnPortFromEp(first, tscMgmtIpSet.fqdn[tscMgmtIpSet.numOfIps], &tscMgmtIpSet.port[tscMgmtIpSet.numOfIps]); - tscMgmtIpSet.numOfIps++; + taosGetFqdnPortFromEp(first, tscMgmtEpSet.fqdn[tscMgmtEpSet.numOfEps], &tscMgmtEpSet.port[tscMgmtEpSet.numOfEps]); + tscMgmtEpSet.numOfEps++; } if (second && second[0] != 0) { @@ -2163,11 +2163,11 @@ int tscSetMgmtIpListFromCfg(const char *first, const char *second) { terrno = TSDB_CODE_TSC_INVALID_FQDN; return -1; } - taosGetFqdnPortFromEp(second, tscMgmtIpSet.fqdn[tscMgmtIpSet.numOfIps], &tscMgmtIpSet.port[tscMgmtIpSet.numOfIps]); - tscMgmtIpSet.numOfIps++; + taosGetFqdnPortFromEp(second, tscMgmtEpSet.fqdn[tscMgmtEpSet.numOfEps], &tscMgmtEpSet.port[tscMgmtEpSet.numOfEps]); + tscMgmtEpSet.numOfEps++; } - if ( tscMgmtIpSet.numOfIps == 0) { + if ( tscMgmtEpSet.numOfEps == 0) { terrno = TSDB_CODE_TSC_INVALID_FQDN; return -1; } diff --git a/src/dnode/inc/dnodeMgmt.h b/src/dnode/inc/dnodeMgmt.h index 092c06d84b..e8f4a0823f 100644 --- a/src/dnode/inc/dnodeMgmt.h +++ b/src/dnode/inc/dnodeMgmt.h @@ -35,8 +35,8 @@ void* dnodeGetVnodeTsdb(void *pVnode); void dnodeReleaseVnode(void *pVnode); void dnodeSendRedirectMsg(SRpcMsg *rpcMsg, bool forShell); -void dnodeGetMnodeIpSetForPeer(void *ipSet); -void dnodeGetMnodeIpSetForShell(void *ipSet); +void dnodeGetMnodeEpSetForPeer(void *epSet); +void dnodeGetMnodeEpSetForShell(void *epSet); #ifdef __cplusplus } diff --git a/src/dnode/src/dnodeMgmt.c b/src/dnode/src/dnodeMgmt.c index 7d3ef9926d..9050b9c582 100644 --- a/src/dnode/src/dnodeMgmt.c +++ b/src/dnode/src/dnodeMgmt.c @@ -52,7 +52,7 @@ void * tsDnodeTmr = NULL; static void * tsStatusTimer = NULL; static uint32_t tsRebootTime; -static SRpcIpSet tsDMnodeIpSet = {0}; +static SRpcEpSet tsDMnodeEpSet = {0}; static SDMMnodeInfos tsDMnodeInfos = {0}; static SDMDnodeCfg tsDnodeCfg = {0}; static taos_qset tsMgmtQset = NULL; @@ -90,21 +90,21 @@ int32_t dnodeInitMgmt() { tsRebootTime = taosGetTimestampSec(); if (!dnodeReadMnodeInfos()) { - memset(&tsDMnodeIpSet, 0, sizeof(SRpcIpSet)); + memset(&tsDMnodeEpSet, 0, sizeof(SRpcEpSet)); memset(&tsDMnodeInfos, 0, sizeof(SDMMnodeInfos)); - tsDMnodeIpSet.numOfIps = 1; - taosGetFqdnPortFromEp(tsFirst, tsDMnodeIpSet.fqdn[0], &tsDMnodeIpSet.port[0]); + tsDMnodeEpSet.numOfEps = 1; + taosGetFqdnPortFromEp(tsFirst, tsDMnodeEpSet.fqdn[0], &tsDMnodeEpSet.port[0]); if (strcmp(tsSecond, tsFirst) != 0) { - tsDMnodeIpSet.numOfIps = 2; - taosGetFqdnPortFromEp(tsSecond, tsDMnodeIpSet.fqdn[1], &tsDMnodeIpSet.port[1]); + tsDMnodeEpSet.numOfEps = 2; + taosGetFqdnPortFromEp(tsSecond, tsDMnodeEpSet.fqdn[1], &tsDMnodeEpSet.port[1]); } } else { - tsDMnodeIpSet.inUse = tsDMnodeInfos.inUse; - tsDMnodeIpSet.numOfIps = tsDMnodeInfos.nodeNum; + tsDMnodeEpSet.inUse = tsDMnodeInfos.inUse; + tsDMnodeEpSet.numOfEps = tsDMnodeInfos.nodeNum; for (int32_t i = 0; i < tsDMnodeInfos.nodeNum; i++) { - taosGetFqdnPortFromEp(tsDMnodeInfos.nodeInfos[i].nodeEp, tsDMnodeIpSet.fqdn[i], &tsDMnodeIpSet.port[i]); + taosGetFqdnPortFromEp(tsDMnodeInfos.nodeInfos[i].nodeEp, tsDMnodeEpSet.fqdn[i], &tsDMnodeEpSet.port[i]); } } @@ -450,27 +450,27 @@ static int32_t dnodeProcessConfigDnodeMsg(SRpcMsg *pMsg) { return taosCfgDynamicOptions(pCfg->config); } -void dnodeUpdateMnodeIpSetForPeer(SRpcIpSet *pIpSet) { - dInfo("mnode IP list for is changed, numOfIps:%d inUse:%d", pIpSet->numOfIps, pIpSet->inUse); - for (int i = 0; i < pIpSet->numOfIps; ++i) { - pIpSet->port[i] -= TSDB_PORT_DNODEDNODE; - dInfo("mnode index:%d %s:%u", i, pIpSet->fqdn[i], pIpSet->port[i]) +void dnodeUpdateMnodeEpSetForPeer(SRpcEpSet *pEpSet) { + dInfo("mnode EP list for is changed, numOfEps:%d inUse:%d", pEpSet->numOfEps, pEpSet->inUse); + for (int i = 0; i < pEpSet->numOfEps; ++i) { + pEpSet->port[i] -= TSDB_PORT_DNODEDNODE; + dInfo("mnode index:%d %s:%u", i, pEpSet->fqdn[i], pEpSet->port[i]) } - tsDMnodeIpSet = *pIpSet; + tsDMnodeEpSet = *pEpSet; } -void dnodeGetMnodeIpSetForPeer(void *ipSetRaw) { - SRpcIpSet *ipSet = ipSetRaw; - *ipSet = tsDMnodeIpSet; +void dnodeGetMnodeEpSetForPeer(void *epSetRaw) { + SRpcEpSet *epSet = epSetRaw; + *epSet = tsDMnodeEpSet; - for (int i=0; inumOfIps; ++i) - ipSet->port[i] += TSDB_PORT_DNODEDNODE; + for (int i=0; inumOfEps; ++i) + epSet->port[i] += TSDB_PORT_DNODEDNODE; } -void dnodeGetMnodeIpSetForShell(void *ipSetRaw) { - SRpcIpSet *ipSet = ipSetRaw; - *ipSet = tsDMnodeIpSet; +void dnodeGetMnodeEpSetForShell(void *epSetRaw) { + SRpcEpSet *epSet = epSetRaw; + *epSet = tsDMnodeEpSet; } static void dnodeProcessStatusRsp(SRpcMsg *pMsg) { @@ -536,10 +536,10 @@ static void dnodeUpdateMnodeInfos(SDMMnodeInfos *pMnodes) { dInfo("mnode index:%d, %s", tsDMnodeInfos.nodeInfos[i].nodeId, tsDMnodeInfos.nodeInfos[i].nodeEp); } - tsDMnodeIpSet.inUse = tsDMnodeInfos.inUse; - tsDMnodeIpSet.numOfIps = tsDMnodeInfos.nodeNum; + tsDMnodeEpSet.inUse = tsDMnodeInfos.inUse; + tsDMnodeEpSet.numOfEps = tsDMnodeInfos.nodeNum; for (int32_t i = 0; i < tsDMnodeInfos.nodeNum; i++) { - taosGetFqdnPortFromEp(tsDMnodeInfos.nodeInfos[i].nodeEp, tsDMnodeIpSet.fqdn[i], &tsDMnodeIpSet.port[i]); + taosGetFqdnPortFromEp(tsDMnodeInfos.nodeInfos[i].nodeEp, tsDMnodeEpSet.fqdn[i], &tsDMnodeEpSet.port[i]); } dnodeSaveMnodeInfos(); @@ -549,10 +549,10 @@ static void dnodeUpdateMnodeInfos(SDMMnodeInfos *pMnodes) { static bool dnodeReadMnodeInfos() { char ipFile[TSDB_FILENAME_LEN*2] = {0}; - sprintf(ipFile, "%s/mnodeIpList.json", tsDnodeDir); + sprintf(ipFile, "%s/mnodeEpSet.json", tsDnodeDir); FILE *fp = fopen(ipFile, "r"); if (!fp) { - dDebug("failed to read mnodeIpList.json, file not exist"); + dDebug("failed to read mnodeEpSet.json, file not exist"); return false; } @@ -563,40 +563,40 @@ static bool dnodeReadMnodeInfos() { if (len <= 0) { free(content); fclose(fp); - dError("failed to read mnodeIpList.json, content is null"); + dError("failed to read mnodeEpSet.json, content is null"); return false; } content[len] = 0; cJSON* root = cJSON_Parse(content); if (root == NULL) { - dError("failed to read mnodeIpList.json, invalid json format"); + dError("failed to read mnodeEpSet.json, invalid json format"); goto PARSE_OVER; } cJSON* inUse = cJSON_GetObjectItem(root, "inUse"); if (!inUse || inUse->type != cJSON_Number) { - dError("failed to read mnodeIpList.json, inUse not found"); + dError("failed to read mnodeEpSet.json, inUse not found"); goto PARSE_OVER; } tsDMnodeInfos.inUse = inUse->valueint; cJSON* nodeNum = cJSON_GetObjectItem(root, "nodeNum"); if (!nodeNum || nodeNum->type != cJSON_Number) { - dError("failed to read mnodeIpList.json, nodeNum not found"); + dError("failed to read mnodeEpSet.json, nodeNum not found"); goto PARSE_OVER; } tsDMnodeInfos.nodeNum = nodeNum->valueint; cJSON* nodeInfos = cJSON_GetObjectItem(root, "nodeInfos"); if (!nodeInfos || nodeInfos->type != cJSON_Array) { - dError("failed to read mnodeIpList.json, nodeInfos not found"); + dError("failed to read mnodeEpSet.json, nodeInfos not found"); goto PARSE_OVER; } int size = cJSON_GetArraySize(nodeInfos); if (size != tsDMnodeInfos.nodeNum) { - dError("failed to read mnodeIpList.json, nodeInfos size not matched"); + dError("failed to read mnodeEpSet.json, nodeInfos size not matched"); goto PARSE_OVER; } @@ -606,14 +606,14 @@ static bool dnodeReadMnodeInfos() { cJSON *nodeId = cJSON_GetObjectItem(nodeInfo, "nodeId"); if (!nodeId || nodeId->type != cJSON_Number) { - dError("failed to read mnodeIpList.json, nodeId not found"); + dError("failed to read mnodeEpSet.json, nodeId not found"); goto PARSE_OVER; } tsDMnodeInfos.nodeInfos[i].nodeId = nodeId->valueint; cJSON *nodeEp = cJSON_GetObjectItem(nodeInfo, "nodeEp"); if (!nodeEp || nodeEp->type != cJSON_String || nodeEp->valuestring == NULL) { - dError("failed to read mnodeIpList.json, nodeName not found"); + dError("failed to read mnodeEpSet.json, nodeName not found"); goto PARSE_OVER; } strncpy(tsDMnodeInfos.nodeInfos[i].nodeEp, nodeEp->valuestring, TSDB_EP_LEN); @@ -621,7 +621,7 @@ static bool dnodeReadMnodeInfos() { ret = true; - dInfo("read mnode iplist successed, numOfIps:%d inUse:%d", tsDMnodeInfos.nodeNum, tsDMnodeInfos.inUse); + dInfo("read mnode epSet successed, numOfEps:%d inUse:%d", tsDMnodeInfos.nodeNum, tsDMnodeInfos.inUse); for (int32_t i = 0; i < tsDMnodeInfos.nodeNum; i++) { dInfo("mnode:%d, %s", tsDMnodeInfos.nodeInfos[i].nodeId, tsDMnodeInfos.nodeInfos[i].nodeEp); } @@ -635,7 +635,7 @@ PARSE_OVER: static void dnodeSaveMnodeInfos() { char ipFile[TSDB_FILENAME_LEN] = {0}; - sprintf(ipFile, "%s/mnodeIpList.json", tsDnodeDir); + sprintf(ipFile, "%s/mnodeEpSet.json", tsDnodeDir); FILE *fp = fopen(ipFile, "w"); if (!fp) return; @@ -663,11 +663,11 @@ static void dnodeSaveMnodeInfos() { fclose(fp); free(content); - dInfo("save mnode iplist successed"); + dInfo("save mnode epSet successed"); } char *dnodeGetMnodeMasterEp() { - return tsDMnodeInfos.nodeInfos[tsDMnodeIpSet.inUse].nodeEp; + return tsDMnodeInfos.nodeInfos[tsDMnodeEpSet.inUse].nodeEp; } void* dnodeGetMnodeInfos() { @@ -726,9 +726,9 @@ static void dnodeSendStatusMsg(void *handle, void *tmrId) { .msgType = TSDB_MSG_TYPE_DM_STATUS }; - SRpcIpSet ipSet; - dnodeGetMnodeIpSetForPeer(&ipSet); - dnodeSendMsgToDnode(&ipSet, &rpcMsg); + SRpcEpSet epSet; + dnodeGetMnodeEpSetForPeer(&epSet); + dnodeSendMsgToDnode(&epSet, &rpcMsg); } static bool dnodeReadDnodeCfg() { @@ -817,20 +817,20 @@ void dnodeSendRedirectMsg(SRpcMsg *rpcMsg, bool forShell) { SRpcConnInfo connInfo = {0}; rpcGetConnInfo(rpcMsg->handle, &connInfo); - SRpcIpSet ipSet = {0}; + SRpcEpSet epSet = {0}; if (forShell) { - dnodeGetMnodeIpSetForShell(&ipSet); + dnodeGetMnodeEpSetForShell(&epSet); } else { - dnodeGetMnodeIpSetForPeer(&ipSet); + dnodeGetMnodeEpSetForPeer(&epSet); } - dDebug("msg:%s will be redirected, dnodeIp:%s user:%s, numOfIps:%d inUse:%d", taosMsg[rpcMsg->msgType], - taosIpStr(connInfo.clientIp), connInfo.user, ipSet.numOfIps, ipSet.inUse); + dDebug("msg:%s will be redirected, dnodeIp:%s user:%s, numOfEps:%d inUse:%d", taosMsg[rpcMsg->msgType], + taosIpStr(connInfo.clientIp), connInfo.user, epSet.numOfEps, epSet.inUse); - for (int i = 0; i < ipSet.numOfIps; ++i) { - dDebug("mnode index:%d %s:%d", i, ipSet.fqdn[i], ipSet.port[i]); - ipSet.port[i] = htons(ipSet.port[i]); + for (int i = 0; i < epSet.numOfEps; ++i) { + dDebug("mnode index:%d %s:%d", i, epSet.fqdn[i], epSet.port[i]); + epSet.port[i] = htons(epSet.port[i]); } - rpcSendRedirectRsp(rpcMsg->handle, &ipSet); + rpcSendRedirectRsp(rpcMsg->handle, &epSet); } diff --git a/src/dnode/src/dnodePeer.c b/src/dnode/src/dnodePeer.c index 2a3436583f..b27f56a871 100644 --- a/src/dnode/src/dnodePeer.c +++ b/src/dnode/src/dnodePeer.c @@ -29,11 +29,11 @@ #include "dnodeVWrite.h" #include "dnodeMPeer.h" -extern void dnodeUpdateMnodeIpSetForPeer(SRpcIpSet *pIpSet); +extern void dnodeUpdateMnodeEpSetForPeer(SRpcEpSet *pEpSet); static void (*dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *); -static void dnodeProcessReqMsgFromDnode(SRpcMsg *pMsg, SRpcIpSet *); +static void dnodeProcessReqMsgFromDnode(SRpcMsg *pMsg, SRpcEpSet *); static void (*dnodeProcessRspMsgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *rpcMsg); -static void dnodeProcessRspFromDnode(SRpcMsg *pMsg, SRpcIpSet *pIpSet); +static void dnodeProcessRspFromDnode(SRpcMsg *pMsg, SRpcEpSet *pEpSet); static void *tsDnodeServerRpc = NULL; static void *tsDnodeClientRpc = NULL; @@ -83,7 +83,7 @@ void dnodeCleanupServer() { } } -static void dnodeProcessReqMsgFromDnode(SRpcMsg *pMsg, SRpcIpSet *pIpSet) { +static void dnodeProcessReqMsgFromDnode(SRpcMsg *pMsg, SRpcEpSet *pEpSet) { SRpcMsg rspMsg = { .handle = pMsg->handle, .pCont = NULL, @@ -148,9 +148,9 @@ void dnodeCleanupClient() { } } -static void dnodeProcessRspFromDnode(SRpcMsg *pMsg, SRpcIpSet *pIpSet) { - if (pMsg->msgType == TSDB_MSG_TYPE_DM_STATUS_RSP && pIpSet) { - dnodeUpdateMnodeIpSetForPeer(pIpSet); +static void dnodeProcessRspFromDnode(SRpcMsg *pMsg, SRpcEpSet *pEpSet) { + if (pMsg->msgType == TSDB_MSG_TYPE_DM_STATUS_RSP && pEpSet) { + dnodeUpdateMnodeEpSetForPeer(pEpSet); } if (dnodeProcessRspMsgFp[pMsg->msgType]) { @@ -166,12 +166,12 @@ void dnodeAddClientRspHandle(uint8_t msgType, void (*fp)(SRpcMsg *rpcMsg)) { dnodeProcessRspMsgFp[msgType] = fp; } -void dnodeSendMsgToDnode(SRpcIpSet *ipSet, SRpcMsg *rpcMsg) { - rpcSendRequest(tsDnodeClientRpc, ipSet, rpcMsg); +void dnodeSendMsgToDnode(SRpcEpSet *epSet, SRpcMsg *rpcMsg) { + rpcSendRequest(tsDnodeClientRpc, epSet, rpcMsg); } void dnodeSendMsgToDnodeRecv(SRpcMsg *rpcMsg, SRpcMsg *rpcRsp) { - SRpcIpSet ipSet = {0}; - dnodeGetMnodeIpSetForPeer(&ipSet); - rpcSendRecv(tsDnodeClientRpc, &ipSet, rpcMsg, rpcRsp); + SRpcEpSet epSet = {0}; + dnodeGetMnodeEpSetForPeer(&epSet); + rpcSendRecv(tsDnodeClientRpc, &epSet, rpcMsg, rpcRsp); } diff --git a/src/dnode/src/dnodeShell.c b/src/dnode/src/dnodeShell.c index d089495079..f9d137bb99 100644 --- a/src/dnode/src/dnodeShell.c +++ b/src/dnode/src/dnodeShell.c @@ -31,7 +31,7 @@ #include "dnodeShell.h" static void (*dnodeProcessShellMsgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *); -static void dnodeProcessMsgFromShell(SRpcMsg *pMsg, SRpcIpSet *); +static void dnodeProcessMsgFromShell(SRpcMsg *pMsg, SRpcEpSet *); static int dnodeRetrieveUserAuthInfo(char *user, char *spi, char *encrypt, char *secret, char *ckey); static void * tsDnodeShellRpc = NULL; static int32_t tsDnodeQueryReqNum = 0; @@ -108,7 +108,7 @@ void dnodeCleanupShell() { } } -void dnodeProcessMsgFromShell(SRpcMsg *pMsg, SRpcIpSet *pIpSet) { +void dnodeProcessMsgFromShell(SRpcMsg *pMsg, SRpcEpSet *pEpSet) { SRpcMsg rpcMsg = { .handle = pMsg->handle, .pCont = NULL, diff --git a/src/inc/dnode.h b/src/inc/dnode.h index b561c407a3..5a059c93a6 100644 --- a/src/inc/dnode.h +++ b/src/inc/dnode.h @@ -39,13 +39,13 @@ SDnodeStatisInfo dnodeGetStatisInfo(); bool dnodeIsFirstDeploy(); char * dnodeGetMnodeMasterEp(); -void dnodeGetMnodeIpSetForPeer(void *ipSet); -void dnodeGetMnodeIpSetForShell(void *ipSet); +void dnodeGetMnodeEpSetForPeer(void *epSet); +void dnodeGetMnodeEpSetForShell(void *epSet); void * dnodeGetMnodeInfos(); int32_t dnodeGetDnodeId(); void dnodeAddClientRspHandle(uint8_t msgType, void (*fp)(SRpcMsg *rpcMsg)); -void dnodeSendMsgToDnode(SRpcIpSet *ipSet, SRpcMsg *rpcMsg); +void dnodeSendMsgToDnode(SRpcEpSet *epSet, SRpcMsg *rpcMsg); void dnodeSendMsgToDnodeRecv(SRpcMsg *rpcMsg, SRpcMsg *rpcRsp); void *dnodeSendCfgTableToRecv(int32_t vgId, int32_t sid); diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index a367d6b93b..96386aab65 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -176,7 +176,7 @@ extern char *taosMsg[]; typedef struct { char fqdn[TSDB_FQDN_LEN]; uint16_t port; -} SIpAddr; +} SEpAddr; typedef struct { int32_t numOfVnodes; @@ -306,7 +306,7 @@ typedef struct { int8_t reserved1; int8_t reserved2; int32_t connId; - SRpcIpSet ipList; + SRpcEpSet epSet; } SCMConnectRsp; typedef struct { @@ -648,8 +648,8 @@ typedef struct SCMSTableVgroupMsg { typedef struct { int32_t vgId; - int8_t numOfIps; - SIpAddr ipAddr[TSDB_MAX_REPLICA]; + int8_t numOfEps; + SEpAddr epAddr[TSDB_MAX_REPLICA]; } SCMVgroupInfo; typedef struct { @@ -753,7 +753,7 @@ typedef struct { uint32_t onlineDnodes; uint32_t connId; int8_t killConnection; - SRpcIpSet ipList; + SRpcEpSet epSet; } SCMHeartBeatRsp; typedef struct { diff --git a/src/inc/trpc.h b/src/inc/trpc.h index d1adfb7494..1af6a5eb0f 100644 --- a/src/inc/trpc.h +++ b/src/inc/trpc.h @@ -28,12 +28,12 @@ extern "C" { extern int tsRpcHeadSize; -typedef struct SRpcIpSet { +typedef struct SRpcEpSet { int8_t inUse; - int8_t numOfIps; + int8_t numOfEps; uint16_t port[TSDB_MAX_REPLICA]; char fqdn[TSDB_MAX_REPLICA][TSDB_FQDN_LEN]; -} SRpcIpSet; +} SRpcEpSet; typedef struct SRpcConnInfo { uint32_t clientIp; @@ -67,7 +67,7 @@ typedef struct SRpcInit { char *ckey; // ciphering key // call back to process incoming msg, code shall be ignored by server app - void (*cfp)(SRpcMsg *, SRpcIpSet *); + void (*cfp)(SRpcMsg *, SRpcEpSet *); // call back to retrieve the client auth info, for server app only int (*afp)(char *tableId, char *spi, char *encrypt, char *secret, char *ckey); @@ -78,11 +78,11 @@ void rpcClose(void *); void *rpcMallocCont(int contLen); void rpcFreeCont(void *pCont); void *rpcReallocCont(void *ptr, int contLen); -void rpcSendRequest(void *thandle, const SRpcIpSet *pIpSet, SRpcMsg *pMsg); +void rpcSendRequest(void *thandle, const SRpcEpSet *pEpSet, SRpcMsg *pMsg); void rpcSendResponse(const SRpcMsg *pMsg); -void rpcSendRedirectRsp(void *pConn, const SRpcIpSet *pIpSet); +void rpcSendRedirectRsp(void *pConn, const SRpcEpSet *pEpSet); int rpcGetConnInfo(void *thandle, SRpcConnInfo *pInfo); -void rpcSendRecv(void *shandle, SRpcIpSet *pIpSet, SRpcMsg *pReq, SRpcMsg *pRsp); +void rpcSendRecv(void *shandle, SRpcEpSet *pEpSet, SRpcMsg *pReq, SRpcMsg *pRsp); int rpcReportProgress(void *pConn, char *pCont, int contLen); void rpcCancelRequest(void *pContext); diff --git a/src/kit/taosmigrate/taosmigrate.c b/src/kit/taosmigrate/taosmigrate.c index b80ca44a10..3ef73ef169 100644 --- a/src/kit/taosmigrate/taosmigrate.c +++ b/src/kit/taosmigrate/taosmigrate.c @@ -210,10 +210,10 @@ int32_t main(int32_t argc, char *argv[]) { (void)snprintf(mnodeWal, TSDB_FILENAME_LEN*2, "%s/mnode/wal/wal0", arguments.dataDir); walModWalFile(mnodeWal); - // 2. modfiy dnode config: mnodeIpList.json - char dnodeIpList[TSDB_FILENAME_LEN*2] = {0}; - (void)snprintf(dnodeIpList, TSDB_FILENAME_LEN*2, "%s/dnode/mnodeIpList.json", arguments.dataDir); - modDnodeIpList(dnodeIpList); + // 2. modfiy dnode config: mnodeEpSet.json + char dnodeEpSet[TSDB_FILENAME_LEN*2] = {0}; + (void)snprintf(dnodeEpSet, TSDB_FILENAME_LEN*2, "%s/dnode/mnodeEpSet.json", arguments.dataDir); + modDnodeEpSet(dnodeEpSet); // 3. modify vnode config: config.json char vnodeDir[TSDB_FILENAME_LEN*2] = {0}; diff --git a/src/kit/taosmigrate/taosmigrate.h b/src/kit/taosmigrate/taosmigrate.h index a0a02e651c..9fb3c92db2 100644 --- a/src/kit/taosmigrate/taosmigrate.h +++ b/src/kit/taosmigrate/taosmigrate.h @@ -71,7 +71,7 @@ int tSystemShell(const char * cmd); void taosMvFile(char* destFile, char *srcFile) ; void walModWalFile(char* walfile); SdnodeIfo* getDnodeInfo(int32_t dnodeId); -void modDnodeIpList(char* dnodeIpList); +void modDnodeEpSet(char* dnodeEpSet); void modAllVnode(char *vnodeDir); #endif diff --git a/src/kit/taosmigrate/taosmigrateDnodeCfg.c b/src/kit/taosmigrate/taosmigrateDnodeCfg.c index 263d5521e9..7f6fd03fea 100644 --- a/src/kit/taosmigrate/taosmigrateDnodeCfg.c +++ b/src/kit/taosmigrate/taosmigrateDnodeCfg.c @@ -23,10 +23,10 @@ static SDMMnodeInfos tsDnodeIpInfos = {0}; -static bool dnodeReadMnodeInfos(char* dnodeIpList) { - FILE *fp = fopen(dnodeIpList, "r"); +static bool dnodeReadMnodeInfos(char* dnodeEpSet) { + FILE *fp = fopen(dnodeEpSet, "r"); if (!fp) { - printf("failed to read mnodeIpList.json, file not exist\n"); + printf("failed to read mnodeEpSet.json, file not exist\n"); return false; } @@ -37,40 +37,40 @@ static bool dnodeReadMnodeInfos(char* dnodeIpList) { if (len <= 0) { free(content); fclose(fp); - printf("failed to read mnodeIpList.json, content is null\n"); + printf("failed to read mnodeEpSet.json, content is null\n"); return false; } content[len] = 0; cJSON* root = cJSON_Parse(content); if (root == NULL) { - printf("failed to read mnodeIpList.json, invalid json format\n"); + printf("failed to read mnodeEpSet.json, invalid json format\n"); goto PARSE_OVER; } cJSON* inUse = cJSON_GetObjectItem(root, "inUse"); if (!inUse || inUse->type != cJSON_Number) { - printf("failed to read mnodeIpList.json, inUse not found\n"); + printf("failed to read mnodeEpSet.json, inUse not found\n"); goto PARSE_OVER; } tsDnodeIpInfos.inUse = inUse->valueint; cJSON* nodeNum = cJSON_GetObjectItem(root, "nodeNum"); if (!nodeNum || nodeNum->type != cJSON_Number) { - printf("failed to read mnodeIpList.json, nodeNum not found\n"); + printf("failed to read mnodeEpSet.json, nodeNum not found\n"); goto PARSE_OVER; } tsDnodeIpInfos.nodeNum = nodeNum->valueint; cJSON* nodeInfos = cJSON_GetObjectItem(root, "nodeInfos"); if (!nodeInfos || nodeInfos->type != cJSON_Array) { - printf("failed to read mnodeIpList.json, nodeInfos not found\n"); + printf("failed to read mnodeEpSet.json, nodeInfos not found\n"); goto PARSE_OVER; } int size = cJSON_GetArraySize(nodeInfos); if (size != tsDnodeIpInfos.nodeNum) { - printf("failed to read mnodeIpList.json, nodeInfos size not matched\n"); + printf("failed to read mnodeEpSet.json, nodeInfos size not matched\n"); goto PARSE_OVER; } @@ -80,14 +80,14 @@ static bool dnodeReadMnodeInfos(char* dnodeIpList) { cJSON *nodeId = cJSON_GetObjectItem(nodeInfo, "nodeId"); if (!nodeId || nodeId->type != cJSON_Number) { - printf("failed to read mnodeIpList.json, nodeId not found\n"); + printf("failed to read mnodeEpSet.json, nodeId not found\n"); goto PARSE_OVER; } tsDnodeIpInfos.nodeInfos[i].nodeId = nodeId->valueint; cJSON *nodeEp = cJSON_GetObjectItem(nodeInfo, "nodeEp"); if (!nodeEp || nodeEp->type != cJSON_String || nodeEp->valuestring == NULL) { - printf("failed to read mnodeIpList.json, nodeName not found\n"); + printf("failed to read mnodeEpSet.json, nodeName not found\n"); goto PARSE_OVER; } strncpy(tsDnodeIpInfos.nodeInfos[i].nodeEp, nodeEp->valuestring, TSDB_EP_LEN); @@ -102,7 +102,7 @@ static bool dnodeReadMnodeInfos(char* dnodeIpList) { ret = true; - //printf("read mnode iplist successed, numOfIps:%d inUse:%d\n", tsDnodeIpInfos.nodeNum, tsDnodeIpInfos.inUse); + //printf("read mnode epSet successed, numOfEps:%d inUse:%d\n", tsDnodeIpInfos.nodeNum, tsDnodeIpInfos.inUse); //for (int32_t i = 0; i < tsDnodeIpInfos.nodeNum; i++) { // printf("mnode:%d, %s\n", tsDnodeIpInfos.nodeInfos[i].nodeId, tsDnodeIpInfos.nodeInfos[i].nodeEp); //} @@ -115,8 +115,8 @@ PARSE_OVER: } -static void dnodeSaveMnodeInfos(char* dnodeIpList) { - FILE *fp = fopen(dnodeIpList, "w"); +static void dnodeSaveMnodeInfos(char* dnodeEpSet) { + FILE *fp = fopen(dnodeEpSet, "w"); if (!fp) return; int32_t len = 0; @@ -143,13 +143,13 @@ static void dnodeSaveMnodeInfos(char* dnodeIpList) { fclose(fp); free(content); - printf("mod mnode iplist successed\n"); + printf("mod mnode epSet successed\n"); } -void modDnodeIpList(char* dnodeIpList) +void modDnodeEpSet(char* dnodeEpSet) { - (void)dnodeReadMnodeInfos(dnodeIpList); - dnodeSaveMnodeInfos(dnodeIpList); + (void)dnodeReadMnodeInfos(dnodeEpSet); + dnodeSaveMnodeInfos(dnodeEpSet); return; } diff --git a/src/mnode/inc/mnodeMnode.h b/src/mnode/inc/mnodeMnode.h index 1060907234..0976ea8acd 100644 --- a/src/mnode/inc/mnodeMnode.h +++ b/src/mnode/inc/mnodeMnode.h @@ -42,12 +42,12 @@ void mnodeIncMnodeRef(struct SMnodeObj *pMnode); void mnodeDecMnodeRef(struct SMnodeObj *pMnode); char * mnodeGetMnodeRoleStr(); -void mnodeGetMnodeIpSetForPeer(SRpcIpSet *ipSet); -void mnodeGetMnodeIpSetForShell(SRpcIpSet *ipSet); +void mnodeGetMnodeEpSetForPeer(SRpcEpSet *epSet); +void mnodeGetMnodeEpSetForShell(SRpcEpSet *epSet); char* mnodeGetMnodeMasterEp(); void mnodeGetMnodeInfos(void *mnodes); -void mnodeUpdateMnodeIpSet(); +void mnodeUpdateMnodeEpSet(); #ifdef __cplusplus } diff --git a/src/mnode/inc/mnodeVgroup.h b/src/mnode/inc/mnodeVgroup.h index 3f1da89605..9c5b201e93 100644 --- a/src/mnode/inc/mnodeVgroup.h +++ b/src/mnode/inc/mnodeVgroup.h @@ -44,12 +44,12 @@ int32_t mnodeGetAvailableVgroup(struct SMnodeMsg *pMsg, SVgObj **pVgroup, int32_ void mnodeAddTableIntoVgroup(SVgObj *pVgroup, SChildTableObj *pTable); void mnodeRemoveTableFromVgroup(SVgObj *pVgroup, SChildTableObj *pTable); -void mnodeSendDropVnodeMsg(int32_t vgId, SRpcIpSet *ipSet, void *ahandle); +void mnodeSendDropVnodeMsg(int32_t vgId, SRpcEpSet *epSet, void *ahandle); void mnodeSendCreateVgroupMsg(SVgObj *pVgroup, void *ahandle); void mnodeSendAlterVgroupMsg(SVgObj *pVgroup); -SRpcIpSet mnodeGetIpSetFromVgroup(SVgObj *pVgroup); -SRpcIpSet mnodeGetIpSetFromIp(char *ep); +SRpcEpSet mnodeGetEpSetFromVgroup(SVgObj *pVgroup); +SRpcEpSet mnodeGetEpSetFromIp(char *ep); #ifdef __cplusplus } diff --git a/src/mnode/src/mnodeDnode.c b/src/mnode/src/mnodeDnode.c index 9085f1a9f7..7edba8662e 100644 --- a/src/mnode/src/mnodeDnode.c +++ b/src/mnode/src/mnodeDnode.c @@ -289,14 +289,14 @@ static int32_t mnodeProcessCfgDnodeMsg(SMnodeMsg *pMsg) { } } - SRpcIpSet ipSet = mnodeGetIpSetFromIp(pCmCfgDnode->ep); + SRpcEpSet epSet = mnodeGetEpSetFromIp(pCmCfgDnode->ep); if (dnodeId != 0) { SDnodeObj *pDnode = mnodeGetDnode(dnodeId); if (pDnode == NULL) { mError("failed to cfg dnode, invalid dnodeId:%d", dnodeId); return TSDB_CODE_MND_DNODE_NOT_EXIST; } - ipSet = mnodeGetIpSetFromIp(pDnode->dnodeEp); + epSet = mnodeGetEpSetFromIp(pDnode->dnodeEp); mnodeDecDnodeRef(pDnode); } @@ -313,7 +313,7 @@ static int32_t mnodeProcessCfgDnodeMsg(SMnodeMsg *pMsg) { }; mInfo("dnode:%s, is configured by %s", pCmCfgDnode->ep, pMsg->pUser->user); - dnodeSendMsgToDnode(&ipSet, &rpcMdCfgDnodeMsg); + dnodeSendMsgToDnode(&epSet, &rpcMdCfgDnodeMsg); return TSDB_CODE_SUCCESS; } @@ -399,9 +399,9 @@ static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg) { SVgObj *pVgroup = mnodeGetVgroup(pVload->vgId); if (pVgroup == NULL) { - SRpcIpSet ipSet = mnodeGetIpSetFromIp(pDnode->dnodeEp); + SRpcEpSet epSet = mnodeGetEpSetFromIp(pDnode->dnodeEp); mInfo("dnode:%d, vgId:%d not exist in mnode, drop it", pDnode->dnodeId, pVload->vgId); - mnodeSendDropVnodeMsg(pVload->vgId, &ipSet, NULL); + mnodeSendDropVnodeMsg(pVload->vgId, &epSet, NULL); } else { mnodeUpdateVgroupStatus(pVgroup, pDnode, pVload); pAccess->vgId = htonl(pVload->vgId); diff --git a/src/mnode/src/mnodeMnode.c b/src/mnode/src/mnodeMnode.c index f74de2b325..d2f389ca0b 100644 --- a/src/mnode/src/mnodeMnode.c +++ b/src/mnode/src/mnodeMnode.c @@ -35,8 +35,8 @@ static void * tsMnodeSdb = NULL; static int32_t tsMnodeUpdateSize = 0; -static SRpcIpSet tsMnodeIpSetForShell; -static SRpcIpSet tsMnodeIpSetForPeer; +static SRpcEpSet tsMnodeEpSetForShell; +static SRpcEpSet tsMnodeEpSetForPeer; static SDMMnodeInfos tsMnodeInfos; static int32_t mnodeGetMnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); static int32_t mnodeRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn); @@ -123,7 +123,7 @@ static int32_t mnodeMnodeActionRestored() { sdbFreeIter(pIter); } - mnodeUpdateMnodeIpSet(); + mnodeUpdateMnodeEpSet(); return TSDB_CODE_SUCCESS; } @@ -204,13 +204,13 @@ char *mnodeGetMnodeRoleStr(int32_t role) { } } -void mnodeUpdateMnodeIpSet() { - mInfo("update mnodes ipset, numOfIps:%d ", mnodeGetMnodesNum()); +void mnodeUpdateMnodeEpSet() { + mInfo("update mnodes epSet, numOfEps:%d ", mnodeGetMnodesNum()); mnodeMnodeWrLock(); - memset(&tsMnodeIpSetForShell, 0, sizeof(SRpcIpSet)); - memset(&tsMnodeIpSetForPeer, 0, sizeof(SRpcIpSet)); + memset(&tsMnodeEpSetForShell, 0, sizeof(SRpcEpSet)); + memset(&tsMnodeEpSetForPeer, 0, sizeof(SRpcEpSet)); memset(&tsMnodeInfos, 0, sizeof(SDMMnodeInfos)); int32_t index = 0; @@ -222,20 +222,20 @@ void mnodeUpdateMnodeIpSet() { SDnodeObj *pDnode = mnodeGetDnode(pMnode->mnodeId); if (pDnode != NULL) { - strcpy(tsMnodeIpSetForShell.fqdn[index], pDnode->dnodeFqdn); - tsMnodeIpSetForShell.port[index] = htons(pDnode->dnodePort); - mDebug("mnode:%d, for shell fqdn:%s %d", pDnode->dnodeId, tsMnodeIpSetForShell.fqdn[index], htons(tsMnodeIpSetForShell.port[index])); + strcpy(tsMnodeEpSetForShell.fqdn[index], pDnode->dnodeFqdn); + tsMnodeEpSetForShell.port[index] = htons(pDnode->dnodePort); + mDebug("mnode:%d, for shell fqdn:%s %d", pDnode->dnodeId, tsMnodeEpSetForShell.fqdn[index], htons(tsMnodeEpSetForShell.port[index])); - strcpy(tsMnodeIpSetForPeer.fqdn[index], pDnode->dnodeFqdn); - tsMnodeIpSetForPeer.port[index] = htons(pDnode->dnodePort + TSDB_PORT_DNODEDNODE); - mDebug("mnode:%d, for peer fqdn:%s %d", pDnode->dnodeId, tsMnodeIpSetForPeer.fqdn[index], htons(tsMnodeIpSetForPeer.port[index])); + strcpy(tsMnodeEpSetForPeer.fqdn[index], pDnode->dnodeFqdn); + tsMnodeEpSetForPeer.port[index] = htons(pDnode->dnodePort + TSDB_PORT_DNODEDNODE); + mDebug("mnode:%d, for peer fqdn:%s %d", pDnode->dnodeId, tsMnodeEpSetForPeer.fqdn[index], htons(tsMnodeEpSetForPeer.port[index])); tsMnodeInfos.nodeInfos[index].nodeId = htonl(pMnode->mnodeId); strcpy(tsMnodeInfos.nodeInfos[index].nodeEp, pDnode->dnodeEp); if (pMnode->role == TAOS_SYNC_ROLE_MASTER) { - tsMnodeIpSetForShell.inUse = index; - tsMnodeIpSetForPeer.inUse = index; + tsMnodeEpSetForShell.inUse = index; + tsMnodeEpSetForPeer.inUse = index; tsMnodeInfos.inUse = index; } @@ -248,23 +248,23 @@ void mnodeUpdateMnodeIpSet() { } tsMnodeInfos.nodeNum = index; - tsMnodeIpSetForShell.numOfIps = index; - tsMnodeIpSetForPeer.numOfIps = index; + tsMnodeEpSetForShell.numOfEps = index; + tsMnodeEpSetForPeer.numOfEps = index; sdbFreeIter(pIter); mnodeMnodeUnLock(); } -void mnodeGetMnodeIpSetForPeer(SRpcIpSet *ipSet) { +void mnodeGetMnodeEpSetForPeer(SRpcEpSet *epSet) { mnodeMnodeRdLock(); - *ipSet = tsMnodeIpSetForPeer; + *epSet = tsMnodeEpSetForPeer; mnodeMnodeUnLock(); } -void mnodeGetMnodeIpSetForShell(SRpcIpSet *ipSet) { +void mnodeGetMnodeEpSetForShell(SRpcEpSet *epSet) { mnodeMnodeRdLock(); - *ipSet = tsMnodeIpSetForShell; + *epSet = tsMnodeEpSetForShell; mnodeMnodeUnLock(); } @@ -295,7 +295,7 @@ int32_t mnodeAddMnode(int32_t dnodeId) { code = TSDB_CODE_MND_SDB_ERROR; } - mnodeUpdateMnodeIpSet(); + mnodeUpdateMnodeEpSet(); return code; } @@ -308,7 +308,7 @@ void mnodeDropMnodeLocal(int32_t dnodeId) { mnodeDecMnodeRef(pMnode); } - mnodeUpdateMnodeIpSet(); + mnodeUpdateMnodeEpSet(); } int32_t mnodeDropMnode(int32_t dnodeId) { @@ -330,7 +330,7 @@ int32_t mnodeDropMnode(int32_t dnodeId) { sdbDecRef(tsMnodeSdb, pMnode); - mnodeUpdateMnodeIpSet(); + mnodeUpdateMnodeEpSet(); return code; } diff --git a/src/mnode/src/mnodePeer.c b/src/mnode/src/mnodePeer.c index 04f2889607..71b8b1ea84 100644 --- a/src/mnode/src/mnodePeer.c +++ b/src/mnode/src/mnodePeer.c @@ -53,14 +53,14 @@ int32_t mnodeProcessPeerReq(SMnodeMsg *pMsg) { if (!sdbIsMaster()) { SMnodeRsp *rpcRsp = &pMsg->rpcRsp; - SRpcIpSet *ipSet = rpcMallocCont(sizeof(SRpcIpSet)); - mnodeGetMnodeIpSetForPeer(ipSet); - rpcRsp->rsp = ipSet; - rpcRsp->len = sizeof(SRpcIpSet); + SRpcEpSet *epSet = rpcMallocCont(sizeof(SRpcEpSet)); + mnodeGetMnodeEpSetForPeer(epSet); + rpcRsp->rsp = epSet; + rpcRsp->len = sizeof(SRpcEpSet); - mDebug("%p, msg:%s in mpeer queue, will be redireced inUse:%d", pMsg->rpcMsg.ahandle, taosMsg[pMsg->rpcMsg.msgType], ipSet->inUse); - for (int32_t i = 0; i < ipSet->numOfIps; ++i) { - mDebug("mnode index:%d ip:%s:%d", i, ipSet->fqdn[i], htons(ipSet->port[i])); + mDebug("%p, msg:%s in mpeer queue, will be redireced inUse:%d", pMsg->rpcMsg.ahandle, taosMsg[pMsg->rpcMsg.msgType], epSet->inUse); + for (int32_t i = 0; i < epSet->numOfEps; ++i) { + mDebug("mnode index:%d ep:%s:%d", i, epSet->fqdn[i], htons(epSet->port[i])); } return TSDB_CODE_RPC_REDIRECT; diff --git a/src/mnode/src/mnodeRead.c b/src/mnode/src/mnodeRead.c index dfecf1f693..af2fed3408 100644 --- a/src/mnode/src/mnodeRead.c +++ b/src/mnode/src/mnodeRead.c @@ -49,14 +49,14 @@ int32_t mnodeProcessRead(SMnodeMsg *pMsg) { if (!sdbIsMaster()) { SMnodeRsp *rpcRsp = &pMsg->rpcRsp; - SRpcIpSet *ipSet = rpcMallocCont(sizeof(SRpcIpSet)); - mnodeGetMnodeIpSetForShell(ipSet); - rpcRsp->rsp = ipSet; - rpcRsp->len = sizeof(SRpcIpSet); + SRpcEpSet *epSet = rpcMallocCont(sizeof(SRpcEpSet)); + mnodeGetMnodeEpSetForShell(epSet); + rpcRsp->rsp = epSet; + rpcRsp->len = sizeof(SRpcEpSet); - mDebug("%p, msg:%s in mread queue, will be redireced, inUse:%d", pMsg->rpcMsg.ahandle, taosMsg[pMsg->rpcMsg.msgType], ipSet->inUse); - for (int32_t i = 0; i < ipSet->numOfIps; ++i) { - mDebug("mnode index:%d ip:%s:%d", i, ipSet->fqdn[i], htons(ipSet->port[i])); + mDebug("%p, msg:%s in mread queue, will be redireced, inUse:%d", pMsg->rpcMsg.ahandle, taosMsg[pMsg->rpcMsg.msgType], epSet->inUse); + for (int32_t i = 0; i < epSet->numOfEps; ++i) { + mDebug("mnode index:%d ep:%s:%d", i, epSet->fqdn[i], htons(epSet->port[i])); } return TSDB_CODE_RPC_REDIRECT; diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index e08f8ca0db..0f72dbdec4 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -219,7 +219,7 @@ void sdbUpdateMnodeRoles() { } } - mnodeUpdateMnodeIpSet(); + mnodeUpdateMnodeEpSet(); } static uint32_t sdbGetFileInfo(void *ahandle, char *name, uint32_t *index, uint32_t eindex, int32_t *size, uint64_t *fversion) { diff --git a/src/mnode/src/mnodeShow.c b/src/mnode/src/mnodeShow.c index a56ad34a25..e3d5b41be3 100644 --- a/src/mnode/src/mnodeShow.c +++ b/src/mnode/src/mnodeShow.c @@ -270,7 +270,7 @@ static int32_t mnodeProcessHeartBeatMsg(SMnodeMsg *pMsg) { pHBRsp->onlineDnodes = htonl(mnodeGetOnlinDnodesNum()); pHBRsp->totalDnodes = htonl(mnodeGetDnodesNum()); - mnodeGetMnodeIpSetForShell(&pHBRsp->ipList); + mnodeGetMnodeEpSetForShell(&pHBRsp->epSet); pMsg->rpcRsp.rsp = pHBRsp; pMsg->rpcRsp.len = sizeof(SCMHeartBeatRsp); @@ -335,7 +335,7 @@ static int32_t mnodeProcessConnectMsg(SMnodeMsg *pMsg) { pConnectRsp->writeAuth = pUser->writeAuth; pConnectRsp->superAuth = pUser->superAuth; - mnodeGetMnodeIpSetForShell(&pConnectRsp->ipList); + mnodeGetMnodeEpSetForShell(&pConnectRsp->epSet); connect_over: if (code != TSDB_CODE_SUCCESS) { diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index 80c4fc95ab..75ed442cd4 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -910,9 +910,9 @@ static int32_t mnodeProcessDropSuperTableMsg(SMnodeMsg *pMsg) { mInfo("app:%p:%p, stable:%s, send drop stable msg to vgId:%d", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId, pVgroup->vgId); - SRpcIpSet ipSet = mnodeGetIpSetFromVgroup(pVgroup); + SRpcEpSet epSet = mnodeGetEpSetFromVgroup(pVgroup); SRpcMsg rpcMsg = {.pCont = pDrop, .contLen = sizeof(SMDDropSTableMsg), .msgType = TSDB_MSG_TYPE_MD_DROP_STABLE}; - dnodeSendMsgToDnode(&ipSet, &rpcMsg); + dnodeSendMsgToDnode(&epSet, &rpcMsg); mnodeDecVgroupRef(pVgroup); } taosHashDestroyIter(pIter); @@ -1484,10 +1484,10 @@ static int32_t mnodeProcessSuperTableVgroupMsg(SMnodeMsg *pMsg) { SDnodeObj *pDnode = pVgroup->vnodeGid[vn].pDnode; if (pDnode == NULL) break; - tstrncpy(pVgroupInfo->vgroups[vgSize].ipAddr[vn].fqdn, pDnode->dnodeFqdn, TSDB_FQDN_LEN); - pVgroupInfo->vgroups[vgSize].ipAddr[vn].port = htons(pDnode->dnodePort); + tstrncpy(pVgroupInfo->vgroups[vgSize].epAddr[vn].fqdn, pDnode->dnodeFqdn, TSDB_FQDN_LEN); + pVgroupInfo->vgroups[vgSize].epAddr[vn].port = htons(pDnode->dnodePort); - pVgroupInfo->vgroups[vgSize].numOfIps++; + pVgroupInfo->vgroups[vgSize].numOfEps++; } vgSize++; @@ -1615,7 +1615,7 @@ static int32_t mnodeDoCreateChildTableCb(SMnodeMsg *pMsg, int32_t code) { return terrno; } - SRpcIpSet ipSet = mnodeGetIpSetFromVgroup(pMsg->pVgroup); + SRpcEpSet epSet = mnodeGetEpSetFromVgroup(pMsg->pVgroup); SRpcMsg rpcMsg = { .ahandle = pMsg, .pCont = pMDCreate, @@ -1624,7 +1624,7 @@ static int32_t mnodeDoCreateChildTableCb(SMnodeMsg *pMsg, int32_t code) { .msgType = TSDB_MSG_TYPE_MD_CREATE_TABLE }; - dnodeSendMsgToDnode(&ipSet, &rpcMsg); + dnodeSendMsgToDnode(&epSet, &rpcMsg); return TSDB_CODE_MND_ACTION_IN_PROGRESS; } @@ -1788,7 +1788,7 @@ static int32_t mnodeProcessDropChildTableMsg(SMnodeMsg *pMsg, bool needReturn) { pDrop->sid = htonl(pTable->sid); pDrop->uid = htobe64(pTable->uid); - SRpcIpSet ipSet = mnodeGetIpSetFromVgroup(pMsg->pVgroup); + SRpcEpSet epSet = mnodeGetEpSetFromVgroup(pMsg->pVgroup); mInfo("app:%p:%p, table:%s, send drop ctable msg, vgId:%d sid:%d uid:%" PRIu64, pMsg->rpcMsg.ahandle, pMsg, pDrop->tableId, pTable->vgId, pTable->sid, pTable->uid); @@ -1803,7 +1803,7 @@ static int32_t mnodeProcessDropChildTableMsg(SMnodeMsg *pMsg, bool needReturn) { if (!needReturn) rpcMsg.ahandle = NULL; - dnodeSendMsgToDnode(&ipSet, &rpcMsg); + dnodeSendMsgToDnode(&epSet, &rpcMsg); return TSDB_CODE_MND_ACTION_IN_PROGRESS; } @@ -1842,7 +1842,7 @@ static int32_t mnodeAlterNormalTableColumnCb(SMnodeMsg *pMsg, int32_t code) { } } - SRpcIpSet ipSet = mnodeGetIpSetFromVgroup(pMsg->pVgroup); + SRpcEpSet epSet = mnodeGetEpSetFromVgroup(pMsg->pVgroup); SRpcMsg rpcMsg = { .ahandle = pMsg, .pCont = pMDCreate, @@ -1854,7 +1854,7 @@ static int32_t mnodeAlterNormalTableColumnCb(SMnodeMsg *pMsg, int32_t code) { mDebug("app:%p:%p, ctable %s, send alter column msg to vgId:%d", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId, pMsg->pVgroup->vgId); - dnodeSendMsgToDnode(&ipSet, &rpcMsg); + dnodeSendMsgToDnode(&epSet, &rpcMsg); return TSDB_CODE_MND_ACTION_IN_PROGRESS; } @@ -1996,9 +1996,9 @@ static int32_t mnodeDoGetChildTableMeta(SMnodeMsg *pMsg, STableMetaMsg *pMeta) { for (int32_t i = 0; i < pMsg->pVgroup->numOfVnodes; ++i) { SDnodeObj *pDnode = mnodeGetDnode(pMsg->pVgroup->vnodeGid[i].dnodeId); if (pDnode == NULL) break; - strcpy(pMeta->vgroup.ipAddr[i].fqdn, pDnode->dnodeFqdn); - pMeta->vgroup.ipAddr[i].port = htons(pDnode->dnodePort + TSDB_PORT_DNODESHELL); - pMeta->vgroup.numOfIps++; + strcpy(pMeta->vgroup.epAddr[i].fqdn, pDnode->dnodeFqdn); + pMeta->vgroup.epAddr[i].port = htons(pDnode->dnodePort + TSDB_PORT_DNODESHELL); + pMeta->vgroup.numOfEps++; mnodeDecDnodeRef(pDnode); } pMeta->vgroup.vgId = htonl(pMsg->pVgroup->vgId); diff --git a/src/mnode/src/mnodeVgroup.c b/src/mnode/src/mnodeVgroup.c index 5f24981b50..1de591df7c 100644 --- a/src/mnode/src/mnodeVgroup.c +++ b/src/mnode/src/mnodeVgroup.c @@ -317,9 +317,9 @@ void mnodeUpdateVgroupStatus(SVgObj *pVgroup, SDnodeObj *pDnode, SVnodeLoad *pVl } if (!dnodeExist) { - SRpcIpSet ipSet = mnodeGetIpSetFromIp(pDnode->dnodeEp); + SRpcEpSet epSet = mnodeGetEpSetFromIp(pDnode->dnodeEp); mError("vgId:%d, dnode:%d not exist in mnode, drop it", pVload->vgId, pDnode->dnodeId); - mnodeSendDropVnodeMsg(pVload->vgId, &ipSet, NULL); + mnodeSendDropVnodeMsg(pVload->vgId, &epSet, NULL); return; } @@ -809,29 +809,29 @@ static SMDCreateVnodeMsg *mnodeBuildVnodeMsg(SVgObj *pVgroup) { return pVnode; } -SRpcIpSet mnodeGetIpSetFromVgroup(SVgObj *pVgroup) { - SRpcIpSet ipSet = { - .numOfIps = pVgroup->numOfVnodes, +SRpcEpSet mnodeGetEpSetFromVgroup(SVgObj *pVgroup) { + SRpcEpSet epSet = { + .numOfEps = pVgroup->numOfVnodes, .inUse = 0, }; for (int i = 0; i < pVgroup->numOfVnodes; ++i) { - strcpy(ipSet.fqdn[i], pVgroup->vnodeGid[i].pDnode->dnodeFqdn); - ipSet.port[i] = pVgroup->vnodeGid[i].pDnode->dnodePort + TSDB_PORT_DNODEDNODE; + strcpy(epSet.fqdn[i], pVgroup->vnodeGid[i].pDnode->dnodeFqdn); + epSet.port[i] = pVgroup->vnodeGid[i].pDnode->dnodePort + TSDB_PORT_DNODEDNODE; } - return ipSet; + return epSet; } -SRpcIpSet mnodeGetIpSetFromIp(char *ep) { - SRpcIpSet ipSet; +SRpcEpSet mnodeGetEpSetFromIp(char *ep) { + SRpcEpSet epSet; - ipSet.numOfIps = 1; - ipSet.inUse = 0; - taosGetFqdnPortFromEp(ep, ipSet.fqdn[0], &ipSet.port[0]); - ipSet.port[0] += TSDB_PORT_DNODEDNODE; - return ipSet; + epSet.numOfEps = 1; + epSet.inUse = 0; + taosGetFqdnPortFromEp(ep, epSet.fqdn[0], &epSet.port[0]); + epSet.port[0] += TSDB_PORT_DNODEDNODE; + return epSet; } -static void mnodeSendAlterVnodeMsg(SVgObj *pVgroup, SRpcIpSet *ipSet) { +static void mnodeSendAlterVnodeMsg(SVgObj *pVgroup, SRpcEpSet *epSet) { SMDAlterVnodeMsg *pAlter = mnodeBuildVnodeMsg(pVgroup); SRpcMsg rpcMsg = { .ahandle = NULL, @@ -840,21 +840,21 @@ static void mnodeSendAlterVnodeMsg(SVgObj *pVgroup, SRpcIpSet *ipSet) { .code = 0, .msgType = TSDB_MSG_TYPE_MD_ALTER_VNODE }; - dnodeSendMsgToDnode(ipSet, &rpcMsg); + dnodeSendMsgToDnode(epSet, &rpcMsg); } void mnodeSendAlterVgroupMsg(SVgObj *pVgroup) { mDebug("vgId:%d, send alter all vnodes msg, numOfVnodes:%d db:%s", pVgroup->vgId, pVgroup->numOfVnodes, pVgroup->dbName); for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) { - SRpcIpSet ipSet = mnodeGetIpSetFromIp(pVgroup->vnodeGid[i].pDnode->dnodeEp); + SRpcEpSet epSet = mnodeGetEpSetFromIp(pVgroup->vnodeGid[i].pDnode->dnodeEp); mDebug("vgId:%d, index:%d, send alter vnode msg to dnode %s", pVgroup->vgId, i, pVgroup->vnodeGid[i].pDnode->dnodeEp); - mnodeSendAlterVnodeMsg(pVgroup, &ipSet); + mnodeSendAlterVnodeMsg(pVgroup, &epSet); } } -static void mnodeSendCreateVnodeMsg(SVgObj *pVgroup, SRpcIpSet *ipSet, void *ahandle) { +static void mnodeSendCreateVnodeMsg(SVgObj *pVgroup, SRpcEpSet *epSet, void *ahandle) { SMDCreateVnodeMsg *pCreate = mnodeBuildVnodeMsg(pVgroup); SRpcMsg rpcMsg = { .ahandle = ahandle, @@ -863,17 +863,17 @@ static void mnodeSendCreateVnodeMsg(SVgObj *pVgroup, SRpcIpSet *ipSet, void *aha .code = 0, .msgType = TSDB_MSG_TYPE_MD_CREATE_VNODE }; - dnodeSendMsgToDnode(ipSet, &rpcMsg); + dnodeSendMsgToDnode(epSet, &rpcMsg); } void mnodeSendCreateVgroupMsg(SVgObj *pVgroup, void *ahandle) { mDebug("vgId:%d, send create all vnodes msg, numOfVnodes:%d db:%s", pVgroup->vgId, pVgroup->numOfVnodes, pVgroup->dbName); for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) { - SRpcIpSet ipSet = mnodeGetIpSetFromIp(pVgroup->vnodeGid[i].pDnode->dnodeEp); + SRpcEpSet epSet = mnodeGetEpSetFromIp(pVgroup->vnodeGid[i].pDnode->dnodeEp); mDebug("vgId:%d, index:%d, send create vnode msg to dnode %s, ahandle:%p", pVgroup->vgId, i, pVgroup->vnodeGid[i].pDnode->dnodeEp, ahandle); - mnodeSendCreateVnodeMsg(pVgroup, &ipSet, ahandle); + mnodeSendCreateVnodeMsg(pVgroup, &epSet, ahandle); } } @@ -926,7 +926,7 @@ static SMDDropVnodeMsg *mnodeBuildDropVnodeMsg(int32_t vgId) { return pDrop; } -void mnodeSendDropVnodeMsg(int32_t vgId, SRpcIpSet *ipSet, void *ahandle) { +void mnodeSendDropVnodeMsg(int32_t vgId, SRpcEpSet *epSet, void *ahandle) { SMDDropVnodeMsg *pDrop = mnodeBuildDropVnodeMsg(vgId); SRpcMsg rpcMsg = { .ahandle = ahandle, @@ -935,16 +935,16 @@ void mnodeSendDropVnodeMsg(int32_t vgId, SRpcIpSet *ipSet, void *ahandle) { .code = 0, .msgType = TSDB_MSG_TYPE_MD_DROP_VNODE }; - dnodeSendMsgToDnode(ipSet, &rpcMsg); + dnodeSendMsgToDnode(epSet, &rpcMsg); } static void mnodeSendDropVgroupMsg(SVgObj *pVgroup, void *ahandle) { pVgroup->status = TAOS_VG_STATUS_DROPPING; // deleting mDebug("vgId:%d, send drop all vnodes msg, ahandle:%p", pVgroup->vgId, ahandle); for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) { - SRpcIpSet ipSet = mnodeGetIpSetFromIp(pVgroup->vnodeGid[i].pDnode->dnodeEp); + SRpcEpSet epSet = mnodeGetEpSetFromIp(pVgroup->vnodeGid[i].pDnode->dnodeEp); mDebug("vgId:%d, send drop vnode msg to dnode:%d, ahandle:%p", pVgroup->vgId, pVgroup->vnodeGid[i].dnodeId, ahandle); - mnodeSendDropVnodeMsg(pVgroup->vgId, &ipSet, ahandle); + mnodeSendDropVnodeMsg(pVgroup->vgId, &epSet, ahandle); } } @@ -998,8 +998,8 @@ static int32_t mnodeProcessVnodeCfgMsg(SMnodeMsg *pMsg) { } mDebug("vgId:%d, send create vnode msg to dnode %s for vnode cfg msg", pVgroup->vgId, pDnode->dnodeEp); - SRpcIpSet ipSet = mnodeGetIpSetFromIp(pDnode->dnodeEp); - mnodeSendCreateVnodeMsg(pVgroup, &ipSet, NULL); + SRpcEpSet epSet = mnodeGetEpSetFromIp(pDnode->dnodeEp); + mnodeSendCreateVnodeMsg(pVgroup, &epSet, NULL); mnodeDecDnodeRef(pDnode); mnodeDecVgroupRef(pVgroup); diff --git a/src/mnode/src/mnodeWrite.c b/src/mnode/src/mnodeWrite.c index 326ed50981..ab3cfa2dad 100644 --- a/src/mnode/src/mnodeWrite.c +++ b/src/mnode/src/mnodeWrite.c @@ -49,16 +49,16 @@ int32_t mnodeProcessWrite(SMnodeMsg *pMsg) { if (!sdbIsMaster()) { SMnodeRsp *rpcRsp = &pMsg->rpcRsp; - SRpcIpSet *ipSet = rpcMallocCont(sizeof(SRpcIpSet)); - mnodeGetMnodeIpSetForShell(ipSet); - rpcRsp->rsp = ipSet; - rpcRsp->len = sizeof(SRpcIpSet); + SRpcEpSet *epSet = rpcMallocCont(sizeof(SRpcEpSet)); + mnodeGetMnodeEpSetForShell(epSet); + rpcRsp->rsp = epSet; + rpcRsp->len = sizeof(SRpcEpSet); mDebug("app:%p:%p, msg:%s will be redireced inUse:%d", pMsg->rpcMsg.ahandle, pMsg, taosMsg[pMsg->rpcMsg.msgType], - ipSet->inUse); - for (int32_t i = 0; i < ipSet->numOfIps; ++i) { - mDebug("app:%p:%p, mnode index:%d ip:%s:%d", pMsg->rpcMsg.ahandle, pMsg, i, ipSet->fqdn[i], - htons(ipSet->port[i])); + epSet->inUse); + for (int32_t i = 0; i < epSet->numOfEps; ++i) { + mDebug("app:%p:%p, mnode index:%d ep:%s:%d", pMsg->rpcMsg.ahandle, pMsg, i, epSet->fqdn[i], + htons(epSet->port[i])); } return TSDB_CODE_RPC_REDIRECT; diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index 5d67d5e615..2325d12d92 100644 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -55,7 +55,7 @@ typedef struct { char secret[TSDB_KEY_LEN]; // secret for the link char ckey[TSDB_KEY_LEN]; // ciphering key - void (*cfp)(SRpcMsg *, SRpcIpSet *); + void (*cfp)(SRpcMsg *, SRpcEpSet *); int (*afp)(char *user, char *spi, char *encrypt, char *secret, char *ckey); int32_t refCount; @@ -71,7 +71,7 @@ typedef struct { typedef struct { SRpcInfo *pRpc; // associated SRpcInfo - SRpcIpSet ipSet; // ip list provided by app + SRpcEpSet epSet; // ip list provided by app void *ahandle; // handle provided by app void *signature; // for validation struct SRpcConn *pConn; // pConn allocated @@ -80,12 +80,12 @@ typedef struct { int32_t contLen; // content length int32_t code; // error code int16_t numOfTry; // number of try for different servers - int8_t oldInUse; // server IP inUse passed by app + int8_t oldInUse; // server EP inUse passed by app int8_t redirect; // flag to indicate redirect int8_t connType; // connection type SRpcMsg *pRsp; // for synchronous API tsem_t *pSem; // for synchronous API - SRpcIpSet *pSet; // for synchronous API + SRpcEpSet *pSet; // for synchronous API char msg[0]; // RpcHead starts from here } SRpcReqContext; @@ -355,7 +355,7 @@ void *rpcReallocCont(void *ptr, int contLen) { return start + sizeof(SRpcReqContext) + sizeof(SRpcHead); } -void rpcSendRequest(void *shandle, const SRpcIpSet *pIpSet, SRpcMsg *pMsg) { +void rpcSendRequest(void *shandle, const SRpcEpSet *pEpSet, SRpcMsg *pMsg) { SRpcInfo *pRpc = (SRpcInfo *)shandle; SRpcReqContext *pContext; @@ -364,11 +364,11 @@ void rpcSendRequest(void *shandle, const SRpcIpSet *pIpSet, SRpcMsg *pMsg) { pContext->ahandle = pMsg->ahandle; pContext->signature = pContext; pContext->pRpc = (SRpcInfo *)shandle; - pContext->ipSet = *pIpSet; + pContext->epSet = *pEpSet; pContext->contLen = contLen; pContext->pCont = pMsg->pCont; pContext->msgType = pMsg->msgType; - pContext->oldInUse = pIpSet->inUse; + pContext->oldInUse = pEpSet->inUse; pContext->connType = RPC_CONN_UDPC; if (contLen > tsRpcMaxUdpSize) pContext->connType = RPC_CONN_TCPC; @@ -458,15 +458,15 @@ void rpcSendResponse(const SRpcMsg *pRsp) { return; } -void rpcSendRedirectRsp(void *thandle, const SRpcIpSet *pIpSet) { +void rpcSendRedirectRsp(void *thandle, const SRpcEpSet *pEpSet) { SRpcMsg rpcMsg; memset(&rpcMsg, 0, sizeof(rpcMsg)); - rpcMsg.contLen = sizeof(SRpcIpSet); + rpcMsg.contLen = sizeof(SRpcEpSet); rpcMsg.pCont = rpcMallocCont(rpcMsg.contLen); if (rpcMsg.pCont == NULL) return; - memcpy(rpcMsg.pCont, pIpSet, sizeof(SRpcIpSet)); + memcpy(rpcMsg.pCont, pEpSet, sizeof(SRpcEpSet)); rpcMsg.code = TSDB_CODE_RPC_REDIRECT; rpcMsg.handle = thandle; @@ -488,7 +488,7 @@ int rpcGetConnInfo(void *thandle, SRpcConnInfo *pInfo) { return 0; } -void rpcSendRecv(void *shandle, SRpcIpSet *pIpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { +void rpcSendRecv(void *shandle, SRpcEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { SRpcReqContext *pContext; pContext = (SRpcReqContext *) (pMsg->pCont-sizeof(SRpcHead)-sizeof(SRpcReqContext)); @@ -498,9 +498,9 @@ void rpcSendRecv(void *shandle, SRpcIpSet *pIpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) tsem_init(&sem, 0, 0); pContext->pSem = &sem; pContext->pRsp = pRsp; - pContext->pSet = pIpSet; + pContext->pSet = pEpSet; - rpcSendRequest(shandle, pIpSet, pMsg); + rpcSendRequest(shandle, pEpSet, pMsg); tsem_wait(&sem); tsem_destroy(&sem); @@ -755,11 +755,11 @@ static SRpcConn *rpcGetConnObj(SRpcInfo *pRpc, int sid, SRecvInfo *pRecv) { static SRpcConn *rpcSetupConnToServer(SRpcReqContext *pContext) { SRpcConn *pConn; SRpcInfo *pRpc = pContext->pRpc; - SRpcIpSet *pIpSet = &pContext->ipSet; + SRpcEpSet *pEpSet = &pContext->epSet; - pConn = rpcGetConnFromCache(pRpc->pCache, pIpSet->fqdn[pIpSet->inUse], pIpSet->port[pIpSet->inUse], pContext->connType); + pConn = rpcGetConnFromCache(pRpc->pCache, pEpSet->fqdn[pEpSet->inUse], pEpSet->port[pEpSet->inUse], pContext->connType); if ( pConn == NULL || pConn->user[0] == 0) { - pConn = rpcOpenConn(pRpc, pIpSet->fqdn[pIpSet->inUse], pIpSet->port[pIpSet->inUse], pContext->connType); + pConn = rpcOpenConn(pRpc, pEpSet->fqdn[pEpSet->inUse], pEpSet->port[pEpSet->inUse], pContext->connType); } if (pConn) { @@ -1020,16 +1020,16 @@ static void rpcNotifyClient(SRpcReqContext *pContext, SRpcMsg *pMsg) { pContext->pConn = NULL; if (pContext->pRsp) { // for synchronous API - memcpy(pContext->pSet, &pContext->ipSet, sizeof(SRpcIpSet)); + memcpy(pContext->pSet, &pContext->epSet, sizeof(SRpcEpSet)); memcpy(pContext->pRsp, pMsg, sizeof(SRpcMsg)); tsem_post(pContext->pSem); } else { // for asynchronous API - SRpcIpSet *pIpSet = NULL; - if (pContext->ipSet.inUse != pContext->oldInUse || pContext->redirect) - pIpSet = &pContext->ipSet; + SRpcEpSet *pEpSet = NULL; + if (pContext->epSet.inUse != pContext->oldInUse || pContext->redirect) + pEpSet = &pContext->epSet; - (*pRpc->cfp)(pMsg, pIpSet); + (*pRpc->cfp)(pMsg, pEpSet); } // free the request message @@ -1070,9 +1070,9 @@ static void rpcProcessIncomingMsg(SRpcConn *pConn, SRpcHead *pHead) { pConn->pContext = NULL; pConn->pReqMsg = NULL; - // for UDP, port may be changed by server, the port in ipSet shall be used for cache + // for UDP, port may be changed by server, the port in epSet shall be used for cache if (pHead->code != TSDB_CODE_RPC_TOO_SLOW) { - rpcAddConnIntoCache(pRpc->pCache, pConn, pConn->peerFqdn, pContext->ipSet.port[pContext->ipSet.inUse], pConn->connType); + rpcAddConnIntoCache(pRpc->pCache, pConn, pConn->peerFqdn, pContext->epSet.port[pContext->epSet.inUse], pConn->connType); } else { rpcCloseConn(pConn); } @@ -1087,10 +1087,10 @@ static void rpcProcessIncomingMsg(SRpcConn *pConn, SRpcHead *pHead) { if (pHead->code == TSDB_CODE_RPC_REDIRECT) { pContext->numOfTry = 0; - memcpy(&pContext->ipSet, pHead->content, sizeof(pContext->ipSet)); - tDebug("%s, redirect is received, numOfIps:%d", pConn->info, pContext->ipSet.numOfIps); - for (int i=0; iipSet.numOfIps; ++i) - pContext->ipSet.port[i] = htons(pContext->ipSet.port[i]); + memcpy(&pContext->epSet, pHead->content, sizeof(pContext->epSet)); + tDebug("%s, redirect is received, numOfEps:%d", pConn->info, pContext->epSet.numOfEps); + for (int i=0; iepSet.numOfEps; ++i) + pContext->epSet.port[i] = htons(pContext->epSet.port[i]); rpcSendReqToServer(pRpc, pContext); rpcFreeCont(rpcMsg.pCont); } else if (pHead->code == TSDB_CODE_RPC_NOT_READY) { @@ -1269,7 +1269,7 @@ static void rpcProcessConnError(void *param, void *id) { tDebug("%s %p, connection error happens", pRpc->label, pContext->ahandle); - if (pContext->numOfTry >= pContext->ipSet.numOfIps) { + if (pContext->numOfTry >= pContext->epSet.numOfEps) { rpcMsg.msgType = pContext->msgType+1; rpcMsg.ahandle = pContext->ahandle; rpcMsg.code = pContext->code; @@ -1279,8 +1279,8 @@ static void rpcProcessConnError(void *param, void *id) { rpcNotifyClient(pContext, &rpcMsg); } else { // move to next IP - pContext->ipSet.inUse++; - pContext->ipSet.inUse = pContext->ipSet.inUse % pContext->ipSet.numOfIps; + pContext->epSet.inUse++; + pContext->epSet.inUse = pContext->epSet.inUse % pContext->epSet.numOfEps; rpcSendReqToServer(pRpc, pContext); } } diff --git a/src/rpc/test/rclient.c b/src/rpc/test/rclient.c index f8dbbedb11..e2d9d388f9 100644 --- a/src/rpc/test/rclient.c +++ b/src/rpc/test/rclient.c @@ -22,7 +22,7 @@ typedef struct { int index; - SRpcIpSet ipSet; + SRpcEpSet epSet; int num; int numOfReqs; int msgSize; @@ -32,11 +32,11 @@ typedef struct { void *pRpc; } SInfo; -static void processResponse(SRpcMsg *pMsg, SRpcIpSet *pIpSet) { +static void processResponse(SRpcMsg *pMsg, SRpcEpSet *pEpSet) { SInfo *pInfo = (SInfo *)pMsg->ahandle; tDebug("thread:%d, response is received, type:%d contLen:%d code:0x%x", pInfo->index, pMsg->msgType, pMsg->contLen, pMsg->code); - if (pIpSet) pInfo->ipSet = *pIpSet; + if (pEpSet) pInfo->epSet = *pEpSet; rpcFreeCont(pMsg->pCont); sem_post(&pInfo->rspSem); @@ -57,7 +57,7 @@ static void *sendRequest(void *param) { rpcMsg.ahandle = pInfo; rpcMsg.msgType = 1; tDebug("thread:%d, send request, contLen:%d num:%d", pInfo->index, pInfo->msgSize, pInfo->num); - rpcSendRequest(pInfo->pRpc, &pInfo->ipSet, &rpcMsg); + rpcSendRequest(pInfo->pRpc, &pInfo->epSet, &rpcMsg); if ( pInfo->num % 20000 == 0 ) tInfo("thread:%d, %d requests have been sent", pInfo->index, pInfo->num); sem_wait(&pInfo->rspSem); @@ -71,7 +71,7 @@ static void *sendRequest(void *param) { int main(int argc, char *argv[]) { SRpcInit rpcInit; - SRpcIpSet ipSet; + SRpcEpSet epSet; int msgSize = 128; int numOfReqs = 0; int appThreads = 1; @@ -82,12 +82,12 @@ int main(int argc, char *argv[]) { pthread_attr_t thattr; // server info - ipSet.numOfIps = 1; - ipSet.inUse = 0; - ipSet.port[0] = 7000; - ipSet.port[1] = 7000; - strcpy(ipSet.fqdn[0], serverIp); - strcpy(ipSet.fqdn[1], "192.168.0.1"); + epSet.numOfEps = 1; + epSet.inUse = 0; + epSet.port[0] = 7000; + epSet.port[1] = 7000; + strcpy(epSet.fqdn[0], serverIp); + strcpy(epSet.fqdn[1], "192.168.0.1"); // client info memset(&rpcInit, 0, sizeof(rpcInit)); @@ -105,9 +105,9 @@ int main(int argc, char *argv[]) { for (int i=1; iindex = i; - pInfo->ipSet = ipSet; + pInfo->epSet = epSet; pInfo->numOfReqs = numOfReqs; pInfo->msgSize = msgSize; sem_init(&pInfo->rspSem, 0, 0); diff --git a/src/rpc/test/rsclient.c b/src/rpc/test/rsclient.c index 266697d555..6e6961784b 100644 --- a/src/rpc/test/rsclient.c +++ b/src/rpc/test/rsclient.c @@ -23,7 +23,7 @@ typedef struct { int index; - SRpcIpSet ipSet; + SRpcEpSet epSet; int num; int numOfReqs; int msgSize; @@ -51,7 +51,7 @@ static void *sendRequest(void *param) { rpcMsg.msgType = 1; tDebug("thread:%d, send request, contLen:%d num:%d", pInfo->index, pInfo->msgSize, pInfo->num); - rpcSendRecv(pInfo->pRpc, &pInfo->ipSet, &rpcMsg, &rspMsg); + rpcSendRecv(pInfo->pRpc, &pInfo->epSet, &rpcMsg, &rspMsg); // handle response if (rspMsg.code != 0) terror++; @@ -72,7 +72,7 @@ static void *sendRequest(void *param) { int main(int argc, char *argv[]) { SRpcInit rpcInit; - SRpcIpSet ipSet; + SRpcEpSet epSet; int msgSize = 128; int numOfReqs = 0; int appThreads = 1; @@ -83,12 +83,12 @@ int main(int argc, char *argv[]) { pthread_attr_t thattr; // server info - ipSet.numOfIps = 1; - ipSet.inUse = 0; - ipSet.port[0] = 7000; - ipSet.port[1] = 7000; - strcpy(ipSet.fqdn[0], serverIp); - strcpy(ipSet.fqdn[1], "192.168.0.1"); + epSet.numOfEps = 1; + epSet.inUse = 0; + epSet.port[0] = 7000; + epSet.port[1] = 7000; + strcpy(epSet.fqdn[0], serverIp); + strcpy(epSet.fqdn[1], "192.168.0.1"); // client info memset(&rpcInit, 0, sizeof(rpcInit)); @@ -106,9 +106,9 @@ int main(int argc, char *argv[]) { for (int i=1; iindex = i; - pInfo->ipSet = ipSet; + pInfo->epSet = epSet; pInfo->numOfReqs = numOfReqs; pInfo->msgSize = msgSize; sem_init(&pInfo->rspSem, 0, 0); diff --git a/src/rpc/test/rserver.c b/src/rpc/test/rserver.c index d06e9df64b..44c5cd6ab4 100644 --- a/src/rpc/test/rserver.c +++ b/src/rpc/test/rserver.c @@ -103,7 +103,7 @@ int retrieveAuthInfo(char *meterId, char *spi, char *encrypt, char *secret, char return ret; } -void processRequestMsg(SRpcMsg *pMsg, SRpcIpSet *pIpSet) { +void processRequestMsg(SRpcMsg *pMsg, SRpcEpSet *pEpSet) { SRpcMsg *pTemp; pTemp = taosAllocateQitem(sizeof(SRpcMsg)); From 710ea32000c4221daf5896d5892c35b9b4baa866 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 19 Jul 2020 03:11:44 +0000 Subject: [PATCH 025/100] Bump lodash from 4.17.13 to 4.17.19 in /src/connector/grafana/tdengine Bumps [lodash](https://github.com/lodash/lodash) from 4.17.13 to 4.17.19. - [Release notes](https://github.com/lodash/lodash/releases) - [Commits](https://github.com/lodash/lodash/compare/4.17.13...4.17.19) Signed-off-by: dependabot[bot] --- src/connector/grafana/tdengine/package-lock.json | 8 ++++---- src/connector/grafana/tdengine/package.json | 2 +- src/connector/grafana/tdengine/yarn.lock | 12 ++++-------- 3 files changed, 9 insertions(+), 13 deletions(-) diff --git a/src/connector/grafana/tdengine/package-lock.json b/src/connector/grafana/tdengine/package-lock.json index 7c8853b99c..013a005d8d 100644 --- a/src/connector/grafana/tdengine/package-lock.json +++ b/src/connector/grafana/tdengine/package-lock.json @@ -1,6 +1,6 @@ { "name": "TDengine", - "version": "1.0.0", + "version": "2.0.0", "lockfileVersion": 1, "requires": true, "dependencies": { @@ -2302,9 +2302,9 @@ } }, "lodash": { - "version": "4.17.13", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.13.tgz", - "integrity": "sha512-vm3/XWXfWtRua0FkUyEHBZy8kCPjErNBT9fJx8Zvs+U6zjqPbTUOpkaoum3O5uiA8sm+yNMHXfYkTUHFoMxFNA==" + "version": "4.17.19", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.19.tgz", + "integrity": "sha512-JNvd8XER9GQX0v2qJgsaN/mzFCNA5BRe/j8JN9d+tWyGLSodKQHKFicdwNYzWwI3wjRnaKPsGj1XkBjx/F96DQ==" }, "log-symbols": { "version": "2.2.0", diff --git a/src/connector/grafana/tdengine/package.json b/src/connector/grafana/tdengine/package.json index 678278ec47..b7e6512354 100644 --- a/src/connector/grafana/tdengine/package.json +++ b/src/connector/grafana/tdengine/package.json @@ -38,7 +38,7 @@ "q": "^1.5.0" }, "dependencies": { - "lodash": "^4.17.13", + "lodash": "^4.17.19", "yarn": "^1.22.0" }, "homepage": "https://github.com/taosdata/TDengine/tree/develop/src/connector/grafana/tdengine" diff --git a/src/connector/grafana/tdengine/yarn.lock b/src/connector/grafana/tdengine/yarn.lock index f785e4e478..00bc01df7a 100644 --- a/src/connector/grafana/tdengine/yarn.lock +++ b/src/connector/grafana/tdengine/yarn.lock @@ -1888,14 +1888,10 @@ locate-path@^3.0.0: p-locate "^3.0.0" path-exists "^3.0.0" -lodash@^4.17.10, lodash@^4.17.13, lodash@^4.17.4, lodash@^4.2.0, lodash@~4.17.10, lodash@~4.17.5: - version "4.17.13" - resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.13.tgz#0bdc3a6adc873d2f4e0c4bac285df91b64fc7b93" - -lodash@^4.17.15: - version "4.17.15" - resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.15.tgz#b447f6670a0455bbfeedd11392eff330ea097548" - integrity sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A== +lodash@^4.17.10, lodash@^4.17.15, lodash@^4.17.19, lodash@^4.17.4, lodash@^4.2.0, lodash@~4.17.10, lodash@~4.17.5: + version "4.17.19" + resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.19.tgz#e48ddedbe30b3321783c5b4301fbd353bc1e4a4b" + integrity sha512-JNvd8XER9GQX0v2qJgsaN/mzFCNA5BRe/j8JN9d+tWyGLSodKQHKFicdwNYzWwI3wjRnaKPsGj1XkBjx/F96DQ== log-symbols@2.2.0: version "2.2.0" From 4a6c170d96607251389741df7fbc8fc7ab5019ac Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 19 Jul 2020 17:53:06 +0800 Subject: [PATCH 026/100] change hash data in sdb --- src/mnode/src/mnodeSdb.c | 79 ++++++++++++++-------------------------- 1 file changed, 27 insertions(+), 52 deletions(-) diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index 0f72dbdec4..42ded7ed06 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -77,11 +77,6 @@ typedef struct { pthread_mutex_t mutex; } SSdbObject; -typedef struct { - int32_t rowSize; - void * row; -} SSdbRow; - typedef struct { pthread_t thread; int32_t workerId; @@ -419,32 +414,28 @@ void sdbDecRef(void *handle, void *pObj) { } } -static SSdbRow *sdbGetRowMeta(SSdbTable *pTable, void *key) { +static void *sdbGetRowMeta(SSdbTable *pTable, void *key) { if (pTable == NULL) return NULL; int32_t keySize = sizeof(int32_t); if (pTable->keyType == SDB_KEY_STRING || pTable->keyType == SDB_KEY_VAR_STRING) { keySize = strlen((char *)key); } - - return taosHashGet(pTable->iHandle, key, keySize); + + void **ppRow = (void **)taosHashGet(pTable->iHandle, key, keySize); + if (ppRow == NULL) return NULL; + return *ppRow; } -static SSdbRow *sdbGetRowMetaFromObj(SSdbTable *pTable, void *key) { +static void *sdbGetRowMetaFromObj(SSdbTable *pTable, void *key) { return sdbGetRowMeta(pTable, sdbGetObjKey(pTable, key)); } void *sdbGetRow(void *handle, void *key) { - SSdbTable *pTable = (SSdbTable *)handle; - int32_t keySize = sizeof(int32_t); - if (pTable->keyType == SDB_KEY_STRING || pTable->keyType == SDB_KEY_VAR_STRING) { - keySize = strlen((char *)key); - } - - SSdbRow *pMeta = taosHashGet(pTable->iHandle, key, keySize); - if (pMeta) { - sdbIncRef(pTable, pMeta->row); - return pMeta->row; + void *pRow = sdbGetRowMeta(handle, key); + if (pRow) { + sdbIncRef(handle, pRow); + return pRow; } else { return NULL; } @@ -455,10 +446,6 @@ static void *sdbGetRowFromObj(SSdbTable *pTable, void *key) { } static int32_t sdbInsertHash(SSdbTable *pTable, SSdbOper *pOper) { - SSdbRow rowMeta; - rowMeta.rowSize = pOper->rowSize; - rowMeta.row = pOper->pObj; - void * key = sdbGetObjKey(pTable, pOper->pObj); int32_t keySize = sizeof(int32_t); @@ -466,7 +453,7 @@ static int32_t sdbInsertHash(SSdbTable *pTable, SSdbOper *pOper) { keySize = strlen((char *)key); } - taosHashPut(pTable->iHandle, key, keySize, &rowMeta, sizeof(SSdbRow)); + taosHashPut(pTable->iHandle, key, keySize, &pOper->pObj, sizeof(void **)); sdbIncRef(pTable, pOper->pObj); atomic_add_fetch_32(&pTable->numOfRows, 1); @@ -586,17 +573,17 @@ static int sdbWrite(void *param, void *data, int type) { code = (*pTable->decodeFp)(&oper); return sdbInsertHash(pTable, &oper); } else if (action == SDB_ACTION_DELETE) { - SSdbRow *rowMeta = sdbGetRowMeta(pTable, pHead->cont); - if (rowMeta == NULL || rowMeta->row == NULL) { + void *pRow = sdbGetRowMeta(pTable, pHead->cont); + if (pRow == NULL) { sdbError("table:%s, failed to get object:%s from wal while dispose delete action", pTable->tableName, pHead->cont); return TSDB_CODE_SUCCESS; } - SSdbOper oper = {.table = pTable, .pObj = rowMeta->row}; + SSdbOper oper = {.table = pTable, .pObj = pRow}; return sdbDeleteHash(pTable, &oper); } else if (action == SDB_ACTION_UPDATE) { - SSdbRow *rowMeta = sdbGetRowMeta(pTable, pHead->cont); - if (rowMeta == NULL || rowMeta->row == NULL) { + void *pRow = sdbGetRowMeta(pTable, pHead->cont); + if (pRow == NULL) { sdbError("table:%s, failed to get object:%s from wal while dispose update action", pTable->tableName, pHead->cont); return TSDB_CODE_SUCCESS; @@ -675,18 +662,12 @@ int32_t sdbDeleteRow(SSdbOper *pOper) { SSdbTable *pTable = (SSdbTable *)pOper->table; if (pTable == NULL) return TSDB_CODE_MND_SDB_INVALID_TABLE_TYPE; - SSdbRow *pMeta = sdbGetRowMetaFromObj(pTable, pOper->pObj); - if (pMeta == NULL) { + void *pRow = sdbGetRowMetaFromObj(pTable, pOper->pObj); + if (pRow == NULL) { sdbDebug("table:%s, record is not there, delete failed", pTable->tableName); return TSDB_CODE_MND_SDB_OBJ_NOT_THERE; } - void *pMetaRow = pMeta->row; - if (pMetaRow == NULL) { - sdbError("table:%s, record meta is null", pTable->tableName); - return TSDB_CODE_MND_SDB_INVAID_META_ROW; - } - sdbIncRef(pTable, pOper->pObj); int32_t code = sdbDeleteHash(pTable, pOper); @@ -728,18 +709,12 @@ int32_t sdbUpdateRow(SSdbOper *pOper) { SSdbTable *pTable = (SSdbTable *)pOper->table; if (pTable == NULL) return TSDB_CODE_MND_SDB_INVALID_TABLE_TYPE; - SSdbRow *pMeta = sdbGetRowMetaFromObj(pTable, pOper->pObj); - if (pMeta == NULL) { + void *pRow = sdbGetRowMetaFromObj(pTable, pOper->pObj); + if (pRow == NULL) { sdbDebug("table:%s, record is not there, update failed", pTable->tableName); return TSDB_CODE_MND_SDB_OBJ_NOT_THERE; } - void *pMetaRow = pMeta->row; - if (pMetaRow == NULL) { - sdbError("table:%s, record meta is null", pTable->tableName); - return TSDB_CODE_MND_SDB_INVAID_META_ROW; - } - int32_t code = sdbUpdateHash(pTable, pOper); if (code != TSDB_CODE_SUCCESS) { sdbError("table:%s, failed to update hash", pTable->tableName); @@ -789,14 +764,14 @@ void *sdbFetchRow(void *handle, void *pNode, void **ppRow) { return NULL; } - SSdbRow *pMeta = taosHashIterGet(pIter); - if (pMeta == NULL) { + void **ppMetaRow = taosHashIterGet(pIter); + if (ppMetaRow == NULL) { taosHashDestroyIter(pIter); return NULL; } - *ppRow = pMeta->row; - sdbIncRef(handle, pMeta->row); + *ppRow = *ppMetaRow; + sdbIncRef(handle, *ppMetaRow); return pIter; } @@ -846,11 +821,11 @@ void sdbCloseTable(void *handle) { SHashMutableIterator *pIter = taosHashCreateIter(pTable->iHandle); while (taosHashIterNext(pIter)) { - SSdbRow *pMeta = taosHashIterGet(pIter); - if (pMeta == NULL) continue; + void **ppRow = taosHashIterGet(pIter); + if (ppRow == NULL) continue; SSdbOper oper = { - .pObj = pMeta->row, + .pObj = *ppRow, .table = pTable, }; From 19e2109614435838e028f9db70c780ae5ea8239f Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 19 Jul 2020 18:11:23 +0800 Subject: [PATCH 027/100] fix refcount error while drop stable --- src/mnode/src/mnodeTable.c | 41 +++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index 75ed442cd4..7478d7cd78 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -107,42 +107,41 @@ static int32_t mnodeChildTableActionInsert(SSdbOper *pOper) { SVgObj *pVgroup = mnodeGetVgroup(pTable->vgId); if (pVgroup == NULL) { mError("ctable:%s, not in vgId:%d", pTable->info.tableId, pTable->vgId); - return TSDB_CODE_MND_VGROUP_NOT_EXIST; } - mnodeDecVgroupRef(pVgroup); - SDbObj *pDb = mnodeGetDb(pVgroup->dbName); - if (pDb == NULL) { - mError("ctable:%s, vgId:%d not in db:%s", pTable->info.tableId, pVgroup->vgId, pVgroup->dbName); - return TSDB_CODE_MND_INVALID_DB; + SDbObj *pDb = NULL; + if (pVgroup != NULL) { + pDb = mnodeGetDb(pVgroup->dbName); + if (pDb == NULL) { + mError("ctable:%s, vgId:%d not in db:%s", pTable->info.tableId, pVgroup->vgId, pVgroup->dbName); + } } - - if (pDb->status != TSDB_DB_STATUS_READY) { - mError("db:%s, status:%d, in dropping", pDb->name, pDb->status); - return TSDB_CODE_MND_DB_IN_DROPPING; - } - mnodeDecDbRef(pDb); - SAcctObj *pAcct = mnodeGetAcct(pDb->acct); - if (pAcct == NULL) { - mError("ctable:%s, acct:%s not exists", pTable->info.tableId, pDb->acct); - return TSDB_CODE_MND_INVALID_ACCT; + SAcctObj *pAcct = NULL; + if (pDb != NULL) { + pAcct = mnodeGetAcct(pDb->acct); + if (pAcct == NULL) { + mError("ctable:%s, acct:%s not exists", pTable->info.tableId, pDb->acct); + } } - mnodeDecAcctRef(pAcct); if (pTable->info.type == TSDB_CHILD_TABLE) { // add ref pTable->superTable = mnodeGetSuperTableByUid(pTable->suid); mnodeAddTableIntoStable(pTable->superTable, pTable); grantAdd(TSDB_GRANT_TIMESERIES, pTable->superTable->numOfColumns - 1); - pAcct->acctInfo.numOfTimeSeries += (pTable->superTable->numOfColumns - 1); + if (pAcct) pAcct->acctInfo.numOfTimeSeries += (pTable->superTable->numOfColumns - 1); } else { grantAdd(TSDB_GRANT_TIMESERIES, pTable->numOfColumns - 1); - pAcct->acctInfo.numOfTimeSeries += (pTable->numOfColumns - 1); + if (pAcct) pAcct->acctInfo.numOfTimeSeries += (pTable->numOfColumns - 1); } - mnodeAddTableIntoDb(pDb); - mnodeAddTableIntoVgroup(pVgroup, pTable); + if (pDb) mnodeAddTableIntoDb(pDb); + if (pVgroup) mnodeAddTableIntoVgroup(pVgroup, pTable); + + mnodeDecVgroupRef(pVgroup); + mnodeDecDbRef(pDb); + mnodeDecAcctRef(pAcct); return TSDB_CODE_SUCCESS; } From b40c7018e3fe9b0b0525c4b2fdbb587179e89c5f Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Mon, 20 Jul 2020 09:28:13 +0800 Subject: [PATCH 028/100] print messages in red on error --- tests/examples/c/apitest.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/examples/c/apitest.c b/tests/examples/c/apitest.c index bc628fb58f..759e16d1de 100644 --- a/tests/examples/c/apitest.c +++ b/tests/examples/c/apitest.c @@ -218,9 +218,9 @@ void verify_prepare(TAOS* taos) { int code = taos_errno(result); if (code != 0) { - printf("failed to create database, reason:%s\n", taos_errstr(result)); + printf("\033[31mfailed to create database, reason:%s\033[0m\n", taos_errstr(result)); taos_free_result(result); - exit(1); + return; } taos_free_result(result); @@ -232,9 +232,9 @@ void verify_prepare(TAOS* taos) { result = taos_query(taos, sql); code = taos_errno(result); if (code != 0) { - printf("failed to create table, reason:%s\n", taos_errstr(result)); + printf("\033[31mfailed to create table, reason:%s\033[0m\n", taos_errstr(result)); taos_free_result(result); - exit(1); + return; } taos_free_result(result); @@ -320,7 +320,7 @@ void verify_prepare(TAOS* taos) { sql = "insert into m1 values(?,?,?,?,?,?,?,?,?,?)"; code = taos_stmt_prepare(stmt, sql, 0); if (code != 0){ - printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); + printf("\033[31mfailed to execute taos_stmt_prepare. code:0x%x\033[0m\n", code); } v.ts = 1591060628000; for (int i = 0; i < 10; ++i) { @@ -343,8 +343,8 @@ void verify_prepare(TAOS* taos) { taos_stmt_add_batch(stmt); } if (taos_stmt_execute(stmt) != 0) { - printf("failed to execute insert statement.\n"); - exit(1); + printf("\033[31mfailed to execute insert statement.\033[0m\n"); + return; } taos_stmt_close(stmt); @@ -355,8 +355,8 @@ void verify_prepare(TAOS* taos) { v.v2 = 15; taos_stmt_bind_param(stmt, params + 2); if (taos_stmt_execute(stmt) != 0) { - printf("failed to execute select statement.\n"); - exit(1); + printf("\033[31mfailed to execute select statement.\033[0m\n"); + return; } result = taos_stmt_use_result(stmt); @@ -444,7 +444,7 @@ int main(int argc, char *argv[]) { TAOS* taos = taos_connect(host, user, passwd, "", 0); if (taos == NULL) { - printf("failed to connect to db, reason:%s\n", taos_errstr(taos)); + printf("\033[31mfailed to connect to db, reason:%s\033[0m\n", taos_errstr(taos)); exit(1); } From 1067175746851366b658b0b2b590f77ff915642f Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Mon, 20 Jul 2020 11:15:32 +0800 Subject: [PATCH 029/100] fix compiler error --- src/client/src/tscSub.c | 3 ++- tests/examples/c/makefile | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/client/src/tscSub.c b/src/client/src/tscSub.c index 73614fa0a5..526f380475 100644 --- a/src/client/src/tscSub.c +++ b/src/client/src/tscSub.c @@ -92,6 +92,7 @@ static void asyncCallback(void *param, TAOS_RES *tres, int code) { static SSub* tscCreateSubscription(STscObj* pObj, const char* topic, const char* sql) { int code = TSDB_CODE_SUCCESS, line = __LINE__; + SSqlObj* pSql = NULL; SSub* pSub = calloc(1, sizeof(SSub)); if (pSub == NULL) { @@ -113,7 +114,7 @@ static SSub* tscCreateSubscription(STscObj* pObj, const char* topic, const char* goto fail; } - SSqlObj* pSql = calloc(1, sizeof(SSqlObj)); + pSql = calloc(1, sizeof(SSqlObj)); if (pSql == NULL) { line = __LINE__; code = TSDB_CODE_TSC_OUT_OF_MEMORY; diff --git a/tests/examples/c/makefile b/tests/examples/c/makefile index 0a4b8ee9d2..f9653c9c96 100644 --- a/tests/examples/c/makefile +++ b/tests/examples/c/makefile @@ -4,7 +4,6 @@ ROOT=./ TARGET=exe LFLAGS = '-Wl,-rpath,/usr/local/taos/driver/' -ltaos -lpthread -lm -lrt -#LFLAGS = '-Wl,-rpath,/home/zbm/project/td/debug/build/lib/' -L/home/zbm/project/td/debug/build/lib -ltaos -lpthread -lm -lrt CFLAGS = -O3 -g -Wall -Wno-deprecated -fPIC -Wno-unused-result -Wconversion -Wno-char-subscripts -D_REENTRANT -Wno-format -D_REENTRANT -DLINUX -msse4.2 -Wno-unused-function -D_M_X64 \ -I/usr/local/taos/include -std=gnu99 @@ -16,6 +15,7 @@ exe: gcc $(CFLAGS) ./prepare.c -o $(ROOT)/prepare $(LFLAGS) gcc $(CFLAGS) ./stream.c -o $(ROOT)/stream $(LFLAGS) gcc $(CFLAGS) ./subscribe.c -o $(ROOT)subscribe $(LFLAGS) + gcc $(CFLAGS) ./apitest.c -o $(ROOT)apitest $(LFLAGS) clean: rm $(ROOT)/asyncdemo From caa5442005f28a0e4c22b538ff64bfc6ffa124a2 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sun, 19 Jul 2020 12:27:45 +0000 Subject: [PATCH 030/100] src/inc/taos.h --- src/inc/taos.h | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/inc/taos.h b/src/inc/taos.h index d6f1883572..1d609bc7db 100644 --- a/src/inc/taos.h +++ b/src/inc/taos.h @@ -67,8 +67,6 @@ DLL_EXPORT void taos_init(); DLL_EXPORT void taos_cleanup(); DLL_EXPORT int taos_options(TSDB_OPTION option, const void *arg, ...); DLL_EXPORT TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port); -DLL_EXPORT TAOS *taos_connect_c(const char *ip, uint8_t ipLen, const char *user, uint8_t userLen, - const char *pass, uint8_t passLen, const char *db, uint8_t dbLen, uint16_t port); DLL_EXPORT void taos_close(TAOS *taos); typedef struct TAOS_BIND { @@ -90,7 +88,6 @@ TAOS_RES * taos_stmt_use_result(TAOS_STMT *stmt); int taos_stmt_close(TAOS_STMT *stmt); DLL_EXPORT TAOS_RES *taos_query(TAOS *taos, const char *sql); -DLL_EXPORT TAOS_RES *taos_query_c(TAOS *taos, const char *sql, uint32_t sqlLen); DLL_EXPORT TAOS_ROW taos_fetch_row(TAOS_RES *res); DLL_EXPORT int taos_result_precision(TAOS_RES *res); // get the time precision of result DLL_EXPORT void taos_free_result(TAOS_RES *res); From 55e3352d7be4c3739bc6355f41c143bef0c37d33 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 19 Jul 2020 23:29:17 +0800 Subject: [PATCH 031/100] [td-225] refactor codes. opt resbuf perf. rename sources file --- src/client/inc/tsclient.h | 2 +- src/client/src/tscFunctionImpl.c | 8 +- src/client/src/tscServer.c | 63 ++--- src/query/inc/{qExecutor.h => qexecutor.h} | 6 +- src/query/inc/{qresultBuf.h => qresultbuf.h} | 24 +- src/query/inc/{qUtil.h => qutil.h} | 4 +- src/query/inc/tsqlfunction.h | 10 +- src/query/src/{qExecutor.c => qexecutor.c} | 132 +++++++---- .../src/{qFilterFunc.c => qfilterfunc.c} | 4 +- src/query/src/qresultBuf.c | 184 --------------- src/query/src/qresultbuf.c | 218 ++++++++++++++++++ src/query/src/{qUtil.c => qutil.c} | 8 +- src/query/tests/resultBufferTest.cpp | 2 +- src/tsdb/src/tsdbRead.c | 2 +- src/util/src/tutil.c | 2 +- 15 files changed, 365 insertions(+), 304 deletions(-) rename src/query/inc/{qExecutor.h => qexecutor.h} (99%) rename src/query/inc/{qresultBuf.h => qresultbuf.h} (75%) rename src/query/inc/{qUtil.h => qutil.h} (95%) rename src/query/src/{qExecutor.c => qexecutor.c} (98%) rename src/query/src/{qFilterFunc.c => qfilterfunc.c} (99%) delete mode 100644 src/query/src/qresultBuf.c create mode 100644 src/query/src/qresultbuf.c rename src/query/src/{qUtil.c => qutil.c} (98%) diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index ef8184990d..3f548c18ca 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -30,7 +30,7 @@ extern "C" { #include "tsqlfunction.h" #include "tutil.h" -#include "qExecutor.h" +#include "qexecutor.h" #include "qsqlparser.h" #include "qtsbuf.h" #include "tcmdtype.h" diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c index 262b7ab3f6..fb15e4efd5 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/client/src/tscFunctionImpl.c @@ -74,7 +74,7 @@ for (int32_t i = 0; i < (ctx)->tagInfo.numOfTagCols; ++i) { \ void noop1(SQLFunctionCtx *UNUSED_PARAM(pCtx)) {} void noop2(SQLFunctionCtx *UNUSED_PARAM(pCtx), int32_t UNUSED_PARAM(index)) {} -void doFinalizer(SQLFunctionCtx *pCtx) { resetResultInfo(GET_RES_INFO(pCtx)); } +void doFinalizer(SQLFunctionCtx *pCtx) { RESET_RESULT_INFO(GET_RES_INFO(pCtx)); } typedef struct tValuePair { tVariant v; @@ -330,12 +330,6 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI return TSDB_CODE_SUCCESS; } -/** - * the numOfRes should be kept, since it may be used later - * and allow the ResultInfo to be re initialized - */ -void resetResultInfo(SResultInfo *pResInfo) { pResInfo->initialized = false; } - void setResultInfoBuf(SResultInfo *pResInfo, int32_t size, bool superTable, char* buf) { assert(pResInfo->interResultBuf == NULL); diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 398dea9d09..2bc41a834d 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -242,52 +242,31 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) { if (pEpSet) tscMgmtEpSet = *pEpSet; } - if (rpcMsg->pCont == NULL) { - rpcMsg->code = TSDB_CODE_RPC_NETWORK_UNAVAIL; - } else { - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); - // if (rpcMsg->code != TSDB_CODE_RPC_NETWORK_UNAVAIL) { - // if (pCmd->command == TSDB_SQL_CONNECT) { - // rpcMsg->code = TSDB_CODE_RPC_NETWORK_UNAVAIL; - // rpcFreeCont(rpcMsg->pCont); - // return; - // } + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); - // if (pCmd->command == TSDB_SQL_HB) { - // rpcMsg->code = TSDB_CODE_RPC_NOT_READY; - // rpcFreeCont(rpcMsg->pCont); - // return; - // } + int32_t cmd = pCmd->command; + if ((cmd == TSDB_SQL_SELECT || cmd == TSDB_SQL_FETCH || cmd == TSDB_SQL_INSERT || cmd == TSDB_SQL_UPDATE_TAGS_VAL) && + (rpcMsg->code == TSDB_CODE_TDB_INVALID_TABLE_ID || + rpcMsg->code == TSDB_CODE_VND_INVALID_VGROUP_ID || + rpcMsg->code == TSDB_CODE_RPC_NETWORK_UNAVAIL || + rpcMsg->code == TSDB_CODE_TDB_TABLE_RECONFIGURE)) { + tscWarn("%p it shall renew table meta, code:%s, retry:%d", pSql, tstrerror(rpcMsg->code), ++pSql->retry); - // if (pCmd->command == TSDB_SQL_META || pCmd->command == TSDB_SQL_DESCRIBE_TABLE || - // pCmd->command == TSDB_SQL_STABLEVGROUP || pCmd->command == TSDB_SQL_SHOW || - // pCmd->command == TSDB_SQL_RETRIEVE) { - // // get table meta/vgroup query will not retry, do nothing - // } - // } + // set the flag to denote that sql string needs to be re-parsed and build submit block with table schema + if (rpcMsg->code == TSDB_CODE_TDB_TABLE_RECONFIGURE) { + pSql->cmd.submitSchema = 1; + } - if ((pCmd->command == TSDB_SQL_SELECT || pCmd->command == TSDB_SQL_FETCH || pCmd->command == TSDB_SQL_INSERT || - pCmd->command == TSDB_SQL_UPDATE_TAGS_VAL) && - (rpcMsg->code == TSDB_CODE_TDB_INVALID_TABLE_ID || rpcMsg->code == TSDB_CODE_VND_INVALID_VGROUP_ID || - rpcMsg->code == TSDB_CODE_RPC_NETWORK_UNAVAIL || rpcMsg->code == TSDB_CODE_TDB_TABLE_RECONFIGURE)) { - tscWarn("%p it shall renew table meta, code:%s, retry:%d", pSql, tstrerror(rpcMsg->code), ++pSql->retry); - // set the flag to denote that sql string needs to be re-parsed and build submit block with table schema - if (rpcMsg->code == TSDB_CODE_TDB_TABLE_RECONFIGURE) { - pSql->cmd.submitSchema = 1; - } + pSql->res.code = rpcMsg->code; // keep the previous error code + if (pSql->retry > pSql->maxRetry) { + tscError("%p max retry %d reached, give up", pSql, pSql->maxRetry); + } else { + rpcMsg->code = tscRenewTableMeta(pSql, pTableMetaInfo->name); - pSql->res.code = rpcMsg->code; // keep the previous error code - if (pSql->retry > pSql->maxRetry) { - tscError("%p max retry %d reached, give up", pSql, pSql->maxRetry); - } else { - rpcMsg->code = tscRenewTableMeta(pSql, pTableMetaInfo->name); - - // if there is an error occurring, proceed to the following error handling procedure. - // todo add test cases - if (rpcMsg->code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { - rpcFreeCont(rpcMsg->pCont); - return; - } + // if there is an error occurring, proceed to the following error handling procedure. + if (rpcMsg->code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { + rpcFreeCont(rpcMsg->pCont); + return; } } } diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qexecutor.h similarity index 99% rename from src/query/inc/qExecutor.h rename to src/query/inc/qexecutor.h index fb8750323f..97a7565de8 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qexecutor.h @@ -19,15 +19,15 @@ #include "hash.h" #include "qfill.h" -#include "qresultBuf.h" +#include "qresultbuf.h" #include "qsqlparser.h" #include "qtsbuf.h" +#include "query.h" #include "taosdef.h" #include "tarray.h" #include "tlockfree.h" #include "tsdb.h" #include "tsqlfunction.h" -#include "query.h" struct SColumnFilterElem; typedef bool (*__filter_func_t)(struct SColumnFilterElem* pFilter, char* val1, char* val2); @@ -158,7 +158,7 @@ typedef struct SQueryRuntimeEnv { SResultInfo* resultInfo; // todo refactor to merge with SWindowResInfo SQuery* pQuery; SQLFunctionCtx* pCtx; - int16_t numOfRowsPerPage; + int32_t numOfRowsPerPage; int16_t offset[TSDB_MAX_COLUMNS]; uint16_t scanFlag; // denotes reversed scan of data or not SFillInfo* pFillInfo; diff --git a/src/query/inc/qresultBuf.h b/src/query/inc/qresultbuf.h similarity index 75% rename from src/query/inc/qresultBuf.h rename to src/query/inc/qresultbuf.h index a323d530a2..f901474cc1 100644 --- a/src/query/inc/qresultBuf.h +++ b/src/query/inc/qresultbuf.h @@ -33,14 +33,19 @@ typedef struct SDiskbasedResultBuf { int32_t fd; // data file fd int32_t allocateId; // allocated page id int32_t incStep; // minimum allocated pages - char* pBuf; // mmap buffer pointer + void* pBuf; // mmap buffer pointer char* path; // file path - + int32_t pageSize; // current used page size + int32_t inMemPages; // numOfPages that are allocated in memory SHashObj* idsTable; // id hash table SIDList list; // for each id, there is a page id list + + void* iBuf; // inmemory buf + void* handle; // for debug purpose } SDiskbasedResultBuf; -#define DEFAULT_INTERN_BUF_PAGE_SIZE (8192L*5) +#define DEFAULT_INTERN_BUF_PAGE_SIZE (1024L) +#define DEFAULT_INMEM_BUF_PAGES 10 /** * create disk-based result buffer @@ -49,7 +54,8 @@ typedef struct SDiskbasedResultBuf { * @param rowSize * @return */ -int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t size, int32_t rowSize, void* handle); +int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t numOfPages, int32_t rowSize, int32_t pagesize, + int32_t inMemPages, void* handle); /** * @@ -81,8 +87,14 @@ SIDList getDataBufPagesIdList(SDiskbasedResultBuf* pResultBuf, int32_t groupId); * @param id * @return */ -#define GET_RES_BUF_PAGE_BY_ID(buf, id) ((tFilePage*)((buf)->pBuf + DEFAULT_INTERN_BUF_PAGE_SIZE*(id))) - +//#define getResBufPage(buf, id) ((tFilePage*)((buf)->pBuf + (buf)->pageSize * (id))) +static FORCE_INLINE tFilePage* getResBufPage(SDiskbasedResultBuf* pResultBuf, int32_t id) { + if (id < pResultBuf->inMemPages) { + return pResultBuf->iBuf + id * pResultBuf->pageSize; + } else { + return pResultBuf->pBuf + (id - pResultBuf->inMemPages) * pResultBuf->pageSize; + } +} /** * get the total buffer size in the format of disk file * @param pResultBuf diff --git a/src/query/inc/qUtil.h b/src/query/inc/qutil.h similarity index 95% rename from src/query/inc/qUtil.h rename to src/query/inc/qutil.h index 78ae7be030..7119cb75fe 100644 --- a/src/query/inc/qUtil.h +++ b/src/query/inc/qutil.h @@ -49,7 +49,7 @@ static FORCE_INLINE char *getPosInResultPage(SQueryRuntimeEnv *pRuntimeEnv, int3 assert(pResult != NULL && pRuntimeEnv != NULL); SQuery *pQuery = pRuntimeEnv->pQuery; - tFilePage *page = GET_RES_BUF_PAGE_BY_ID(pRuntimeEnv->pResultBuf, pResult->pos.pageId); + tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pResult->pos.pageId); int32_t realRowId = pResult->pos.rowId * GET_ROW_PARAM_FOR_MULTIOUTPUT(pQuery, pRuntimeEnv->topBotQuery, pRuntimeEnv->stableQuery); return ((char *)page->data) + pRuntimeEnv->offset[columnIndex] * pRuntimeEnv->numOfRowsPerPage + @@ -59,6 +59,4 @@ static FORCE_INLINE char *getPosInResultPage(SQueryRuntimeEnv *pRuntimeEnv, int3 __filter_func_t *getRangeFilterFuncArray(int32_t type); __filter_func_t *getValueFilterFuncArray(int32_t type); -bool supportPrefilter(int32_t type); - #endif // TDENGINE_QUERYUTIL_H diff --git a/src/query/inc/tsqlfunction.h b/src/query/inc/tsqlfunction.h index c687f01cbc..5ce9121cf1 100644 --- a/src/query/inc/tsqlfunction.h +++ b/src/query/inc/tsqlfunction.h @@ -255,7 +255,15 @@ extern int32_t functionCompatList[]; // compatible check array list bool topbot_datablock_filter(SQLFunctionCtx *pCtx, int32_t functionId, const char *minval, const char *maxval); -void resetResultInfo(SResultInfo *pResInfo); +/** + * the numOfRes should be kept, since it may be used later + * and allow the ResultInfo to be re initialized + */ +#define RESET_RESULT_INFO(_r) \ + do { \ + (_r)->initialized = false; \ + } while (0) + void setResultInfoBuf(SResultInfo *pResInfo, int32_t size, bool superTable, char* buf); static FORCE_INLINE void initResultInfo(SResultInfo *pResInfo) { diff --git a/src/query/src/qExecutor.c b/src/query/src/qexecutor.c similarity index 98% rename from src/query/src/qExecutor.c rename to src/query/src/qexecutor.c index 42fc90f478..9348f702d0 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qexecutor.c @@ -20,12 +20,12 @@ #include "exception.h" #include "hash.h" -#include "qExecutor.h" -#include "qUtil.h" -#include "qresultBuf.h" +#include "qast.h" +#include "qexecutor.h" +#include "qresultbuf.h" #include "query.h" #include "queryLog.h" -#include "qast.h" +#include "qutil.h" #include "tlosertree.h" #include "tscompression.h" #include "ttime.h" @@ -233,9 +233,7 @@ bool isGroupbyNormalCol(SSqlGroupbyExpr *pGroupbyExpr) { for (int32_t i = 0; i < pGroupbyExpr->numOfGroupCols; ++i) { SColIndex *pColIndex = taosArrayGet(pGroupbyExpr->columnInfo, i); if (pColIndex->flag == TSDB_COL_NORMAL) { - /* - * make sure the normal column locates at the second position if tbname exists in group by clause - */ + //make sure the normal column locates at the second position if tbname exists in group by clause if (pGroupbyExpr->numOfGroupCols > 1) { assert(pColIndex->colIndex > 0); } @@ -294,6 +292,17 @@ bool isSelectivityWithTagsQuery(SQuery *pQuery) { return false; } +bool isProjQuery(SQuery *pQuery) { + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { + int32_t functId = pQuery->pSelectExpr[i].base.functionId; + if (functId != TSDB_FUNC_PRJ && functId != TSDB_FUNC_TAGPRJ) { + return false; + } + } + + return true; +} + bool isTSCompQuery(SQuery *pQuery) { return pQuery->pSelectExpr[0].base.functionId == TSDB_FUNC_TS_COMP; } static bool limitResults(SQueryRuntimeEnv* pRuntimeEnv) { @@ -470,7 +479,7 @@ static int32_t addNewWindowResultBuf(SWindowResult *pWindowRes, SDiskbasedResult pData = getNewDataBuf(pResultBuf, sid, &pageId); } else { pageId = getLastPageId(list); - pData = GET_RES_BUF_PAGE_BY_ID(pResultBuf, pageId); + pData = getResBufPage(pResultBuf, pageId); if (pData->num >= numOfRowsPerPage) { pData = getNewDataBuf(pResultBuf, sid, &pageId); @@ -1003,7 +1012,6 @@ static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, char *pDat case TSDB_DATA_TYPE_BIGINT: v = GET_INT64_VAL(pData); break; } -// assert(pRuntimeEnv->windowResInfo.hashList->size <= 2); SWindowResult *pWindowRes = doSetTimeWindowFromKey(pRuntimeEnv, &pRuntimeEnv->windowResInfo, pData, bytes, true); if (pWindowRes == NULL) { return -1; @@ -1201,9 +1209,8 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS continue; } - // interval window query + // interval window query, decide the time window according to the primary timestamp if (QUERY_IS_INTERVAL_QUERY(pQuery)) { - // decide the time window according to the primary timestamp int64_t ts = tsCols[offset]; STimeWindow win = getActiveTimeWindow(pWindowResInfo, ts, pQuery); @@ -1225,8 +1232,7 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS while (1) { GET_NEXT_TIMEWINDOW(pQuery, &nextWin); - if (/*pWindowResInfo->startTime > nextWin.skey ||*/ - (nextWin.skey > pQuery->window.ekey && QUERY_IS_ASC_QUERY(pQuery)) || + if ((nextWin.skey > pQuery->window.ekey && QUERY_IS_ASC_QUERY(pQuery)) || (nextWin.skey < pQuery->window.ekey && !QUERY_IS_ASC_QUERY(pQuery))) { break; } @@ -1484,6 +1490,8 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order goto _clean; } + qDebug("QInfo:%p setup runtime env1", GET_QINFO_ADDR(pRuntimeEnv)); + pRuntimeEnv->offset[0] = 0; for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { SSqlFuncMsg *pSqlFuncMsg = &pQuery->pSelectExpr[i].base; @@ -1528,6 +1536,8 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order } } + qDebug("QInfo:%p setup runtime env2", GET_QINFO_ADDR(pRuntimeEnv)); + // set the order information for top/bottom query int32_t functionId = pCtx->functionId; @@ -1548,17 +1558,25 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order } } + qDebug("QInfo:%p setup runtime env3", GET_QINFO_ADDR(pRuntimeEnv)); + char* buf = (char*) pRuntimeEnv->resultInfo + sizeof(SResultInfo) * pQuery->numOfOutput; // set the intermediate result output buffer setWindowResultInfo(pRuntimeEnv->resultInfo, pQuery, pRuntimeEnv->stableQuery, buf); + qDebug("QInfo:%p setup runtime env4", GET_QINFO_ADDR(pRuntimeEnv)); + // if it is group by normal column, do not set output buffer, the output buffer is pResult - if (!isGroupbyNormalCol(pQuery->pGroupbyExpr) && !pRuntimeEnv->stableQuery) { + if (!pRuntimeEnv->groupbyNormalCol && !pRuntimeEnv->stableQuery) { resetCtxOutputBuf(pRuntimeEnv); } + qDebug("QInfo:%p setup runtime env5", GET_QINFO_ADDR(pRuntimeEnv)); + setCtxTagColumnInfo(pRuntimeEnv, pRuntimeEnv->pCtx); + + qDebug("QInfo:%p init completed", GET_QINFO_ADDR(pRuntimeEnv)); return TSDB_CODE_SUCCESS; _clean: @@ -1910,9 +1928,20 @@ static int32_t getInitialPageNum(SQInfo *pQInfo) { return num; } -static FORCE_INLINE int32_t getNumOfRowsInResultPage(SQuery *pQuery, bool topBotQuery, bool isSTableQuery) { - int32_t rowSize = pQuery->rowSize * GET_ROW_PARAM_FOR_MULTIOUTPUT(pQuery, topBotQuery, isSTableQuery); - return (DEFAULT_INTERN_BUF_PAGE_SIZE - sizeof(tFilePage)) / rowSize; +static void getIntermediateBufInfo(SQueryRuntimeEnv* pRuntimeEnv, int32_t* ps, int32_t* rowsize) { + SQuery* pQuery = pRuntimeEnv->pQuery; + + *rowsize = pQuery->rowSize * GET_ROW_PARAM_FOR_MULTIOUTPUT(pQuery, pRuntimeEnv->topBotQuery, pRuntimeEnv->stableQuery); + int32_t overhead = sizeof(tFilePage); + + // one page contains at least two rows + *ps = DEFAULT_INTERN_BUF_PAGE_SIZE; + while(((*rowsize) * 2) > (*ps) - overhead) { + *ps = (*ps << 1u); + } + + pRuntimeEnv->numOfRowsPerPage = ((*ps) - sizeof(tFilePage)) / (*rowsize); + } #define IS_PREFILTER_TYPE(_t) ((_t) != TSDB_DATA_TYPE_BINARY && (_t) != TSDB_DATA_TYPE_NCHAR) @@ -2038,8 +2067,7 @@ int32_t loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, void* pQueryHandle, } else { // check if this data block is required to load // Calculate all time windows that are overlapping or contain current data block. - // If current data block is contained by all possible time window, loading current - // data block is not needed. + // If current data block is contained by all possible time window, do not load current data block. if (QUERY_IS_INTERVAL_QUERY(pQuery) && overlapWithTimeWindow(pQuery, pBlockInfo)) { status = BLK_DATA_ALL_NEEDED; } @@ -2424,7 +2452,7 @@ static void doMerge(SQueryRuntimeEnv *pRuntimeEnv, int64_t timestamp, SWindowRes pCtx[i].aOutputBuf = pCtx[i].aOutputBuf + pCtx[i].outputBytes; pCtx[i].currentStage = FIRST_STAGE_MERGE; - resetResultInfo(pCtx[i].resultInfo); + RESET_RESULT_INFO(pCtx[i].resultInfo); aAggs[functionId].init(&pCtx[i]); } @@ -2661,7 +2689,7 @@ void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) { int32_t size = taosArrayGetSize(list); for (int32_t i = 0; i < size; ++i) { int32_t* pgId = taosArrayGet(list, i); - tFilePage *pData = GET_RES_BUF_PAGE_BY_ID(pResultBuf, *pgId); + tFilePage *pData = getResBufPage(pResultBuf, *pgId); total += pData->num; } @@ -2670,7 +2698,7 @@ void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) { int32_t offset = 0; for (int32_t j = 0; j < size; ++j) { int32_t* pgId = taosArrayGet(list, j); - tFilePage *pData = GET_RES_BUF_PAGE_BY_ID(pResultBuf, *pgId); + tFilePage *pData = getResBufPage(pResultBuf, *pgId); for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { int32_t bytes = pRuntimeEnv->pCtx[i].outputBytes; @@ -2860,10 +2888,10 @@ int32_t flushFromResultBuf(SQInfo *pQInfo) { SQuery * pQuery = pRuntimeEnv->pQuery; SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf; - int32_t capacity = (DEFAULT_INTERN_BUF_PAGE_SIZE - sizeof(tFilePage)) / pQuery->rowSize; // the base value for group result, since the maximum number of table for each vnode will not exceed 100,000. int32_t pageId = -1; + int32_t capacity = pResultBuf->numOfRowsPerPage; int32_t remain = pQuery->sdata[0]->num; int32_t offset = 0; @@ -3033,7 +3061,7 @@ void resetCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv) { * set the output buffer information and intermediate buffer * not all queries require the interResultBuf, such as COUNT/TAGPRJ/PRJ/TAG etc. */ - resetResultInfo(&pRuntimeEnv->resultInfo[i]); + RESET_RESULT_INFO(&pRuntimeEnv->resultInfo[i]); pCtx->resultInfo = &pRuntimeEnv->resultInfo[i]; // set the timestamp output buffer for top/bottom/diff query @@ -3072,7 +3100,7 @@ void forwardCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, int64_t output) { pRuntimeEnv->pCtx[j].ptsOutputBuf += TSDB_KEYSIZE * output; } - resetResultInfo(pRuntimeEnv->pCtx[j].resultInfo); + RESET_RESULT_INFO(pRuntimeEnv->pCtx[j].resultInfo); } } @@ -3326,8 +3354,8 @@ void scanOneTableDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start) { if (pRuntimeEnv->pSecQueryHandle == NULL) { longjmp(pRuntimeEnv->env, terrno); } - pRuntimeEnv->windowResInfo.curIndex = qstatus.windowIndex; + pRuntimeEnv->windowResInfo.curIndex = qstatus.windowIndex; setQueryStatus(pQuery, QUERY_NOT_COMPLETED); pRuntimeEnv->scanFlag = REPEAT_SCAN; @@ -3462,7 +3490,6 @@ void setExecutionContext(SQInfo *pQInfo, int32_t groupIndex, TSKEY nextKey) { return; } - int32_t GROUPRESULTID = 1; SWindowResult *pWindowRes = doSetTimeWindowFromKey(pRuntimeEnv, pWindowResInfo, (char *)&groupIndex, sizeof(groupIndex), true); if (pWindowRes == NULL) { @@ -3474,7 +3501,7 @@ void setExecutionContext(SQInfo *pQInfo, int32_t groupIndex, TSKEY nextKey) { * all group belong to one result set, and each group result has different group id so set the id to be one */ if (pWindowRes->pos.pageId == -1) { - if (addNewWindowResultBuf(pWindowRes, pRuntimeEnv->pResultBuf, GROUPRESULTID, pRuntimeEnv->numOfRowsPerPage) != + if (addNewWindowResultBuf(pWindowRes, pRuntimeEnv->pResultBuf, groupIndex, pRuntimeEnv->numOfRowsPerPage) != TSDB_CODE_SUCCESS) { return; } @@ -4170,7 +4197,6 @@ static SFillColInfo* taosCreateFillColInfo(SQuery* pQuery) { } int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bool isSTableQuery) { - qDebug("QInfo:%p start to init qhandle", pQInfo); SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; int32_t code = TSDB_CODE_SUCCESS; @@ -4209,33 +4235,42 @@ int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bo return code; } - pRuntimeEnv->numOfRowsPerPage = getNumOfRowsInResultPage(pQuery, pRuntimeEnv->topBotQuery, isSTableQuery); + int32_t ps = DEFAULT_PAGE_SIZE; + int32_t rowsize = 0; + getIntermediateBufInfo(pRuntimeEnv, &ps, &rowsize); if (isSTableQuery && !onlyQueryTags(pRuntimeEnv->pQuery)) { - int32_t rows = getInitialPageNum(pQInfo); - code = createDiskbasedResultBuffer(&pRuntimeEnv->pResultBuf, rows, pQuery->rowSize, pQInfo); + int32_t numOfPages = getInitialPageNum(pQInfo); + code = createDiskbasedResultBuffer(&pRuntimeEnv->pResultBuf, numOfPages, rowsize, ps, numOfPages, pQInfo); if (code != TSDB_CODE_SUCCESS) { return code; } if (!QUERY_IS_INTERVAL_QUERY(pQuery)) { int16_t type = TSDB_DATA_TYPE_NULL; + int32_t threshold = 0; if (pRuntimeEnv->groupbyNormalCol) { // group by columns not tags; type = getGroupbyColumnType(pQuery, pQuery->pGroupbyExpr); + threshold = 4000; } else { type = TSDB_DATA_TYPE_INT; // group id + threshold = GET_NUM_OF_TABLEGROUP(pQInfo); + if (threshold < 8) { + threshold = 8; + } } - code = initWindowResInfo(&pRuntimeEnv->windowResInfo, pRuntimeEnv, 32, 4096, type); + code = initWindowResInfo(&pRuntimeEnv->windowResInfo, pRuntimeEnv, 8, threshold, type); if (code != TSDB_CODE_SUCCESS) { return code; } } - } else if (pRuntimeEnv->groupbyNormalCol || QUERY_IS_INTERVAL_QUERY(pQuery)) { - int32_t rows = getInitialPageNum(pQInfo); - code = createDiskbasedResultBuffer(&pRuntimeEnv->pResultBuf, rows, pQuery->rowSize, pQInfo); + int32_t numOfResultRows = getInitialPageNum(pQInfo); + getIntermediateBufInfo(pRuntimeEnv, &ps, &rowsize); + + code = createDiskbasedResultBuffer(&pRuntimeEnv->pResultBuf, numOfResultRows, rowsize, ps, numOfResultRows, pQInfo); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -4247,7 +4282,7 @@ int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bo type = TSDB_DATA_TYPE_TIMESTAMP; } - code = initWindowResInfo(&pRuntimeEnv->windowResInfo, pRuntimeEnv, rows, 4096, type); + code = initWindowResInfo(&pRuntimeEnv->windowResInfo, pRuntimeEnv, numOfResultRows, 4096, type); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -5705,13 +5740,18 @@ static void calResultBufSize(SQuery* pQuery) { const int32_t RESULT_MSG_MIN_ROWS = 8192; const float RESULT_THRESHOLD_RATIO = 0.85; - int32_t numOfRes = RESULT_MSG_MIN_SIZE / pQuery->rowSize; - if (numOfRes < RESULT_MSG_MIN_ROWS) { - numOfRes = RESULT_MSG_MIN_ROWS; - } + if (isProjQuery(pQuery)) { + int32_t numOfRes = RESULT_MSG_MIN_SIZE / pQuery->rowSize; + if (numOfRes < RESULT_MSG_MIN_ROWS) { + numOfRes = RESULT_MSG_MIN_ROWS; + } - pQuery->rec.capacity = numOfRes; - pQuery->rec.threshold = numOfRes * RESULT_THRESHOLD_RATIO; + pQuery->rec.capacity = numOfRes; + pQuery->rec.threshold = numOfRes * RESULT_THRESHOLD_RATIO; + } else { // in case of non-prj query, a smaller output buffer will be used. + pQuery->rec.capacity = 4096; + pQuery->rec.threshold = pQuery->rec.capacity * RESULT_THRESHOLD_RATIO; + } } static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SArray* pTableIdList, SSqlGroupbyExpr *pGroupbyExpr, SExprInfo *pExprs, @@ -5723,6 +5763,7 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SArray* pTableIdList, if (pQInfo == NULL) { goto _cleanup_qinfo; } + // to make sure third party won't overwrite this structure pQInfo->signature = pQInfo; pQInfo->tableGroupInfo = *pTableGroupInfo; @@ -6041,11 +6082,10 @@ static void freeQInfo(SQInfo *pQInfo) { tfree(pQuery->sdata); tfree(pQuery); + pQInfo->signature = 0; qDebug("QInfo:%p QInfo is freed", pQInfo); - // destroy signature, in order to avoid the query process pass the object safety check - memset(pQInfo, 0, sizeof(SQInfo)); tfree(pQInfo); } @@ -6209,7 +6249,7 @@ int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, vo } int64_t el = taosGetTimestampUs() - st; - qDebug("qmsg:%p tag filter completed, elapsed time:%"PRId64"us", pQueryMsg, el); + qDebug("qmsg:%p tag filter completed, numOfTables:%zu, elapsed time:%"PRId64"us", pQueryMsg, tableGroupInfo.numOfTables, el); } else { assert(0); } diff --git a/src/query/src/qFilterFunc.c b/src/query/src/qfilterfunc.c similarity index 99% rename from src/query/src/qFilterFunc.c rename to src/query/src/qfilterfunc.c index 1a95b9fe21..6f336434e1 100644 --- a/src/query/src/qFilterFunc.c +++ b/src/query/src/qfilterfunc.c @@ -16,7 +16,7 @@ #define _DEFAULT_SOURCE #include "os.h" -#include "qExecutor.h" +#include "qexecutor.h" #include "taosmsg.h" #include "tcompare.h" #include "tsqlfunction.h" @@ -554,5 +554,3 @@ __filter_func_t* getValueFilterFuncArray(int32_t type) { default: return NULL; } } - -bool supportPrefilter(int32_t type) { return type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR; } diff --git a/src/query/src/qresultBuf.c b/src/query/src/qresultBuf.c deleted file mode 100644 index ae1a95179b..0000000000 --- a/src/query/src/qresultBuf.c +++ /dev/null @@ -1,184 +0,0 @@ -#include "qresultBuf.h" -#include "hash.h" -#include "qextbuffer.h" -#include "taoserror.h" -#include "queryLog.h" - -int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t size, int32_t rowSize, void* handle) { - *pResultBuf = calloc(1, sizeof(SDiskbasedResultBuf)); - SDiskbasedResultBuf* pResBuf = *pResultBuf; - if (pResBuf == NULL) { - return TSDB_CODE_COM_OUT_OF_MEMORY; - } - - pResBuf->numOfRowsPerPage = (DEFAULT_INTERN_BUF_PAGE_SIZE - sizeof(tFilePage)) / rowSize; - pResBuf->numOfPages = size; - - pResBuf->totalBufSize = pResBuf->numOfPages * DEFAULT_INTERN_BUF_PAGE_SIZE; - pResBuf->incStep = 4; - - // init id hash table - pResBuf->idsTable = taosHashInit(size, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false); - pResBuf->list = taosArrayInit(size, POINTER_BYTES); - - char path[4096] = {0}; - getTmpfilePath("tsdb_qbuf", path); - pResBuf->path = strdup(path); - - pResBuf->fd = open(pResBuf->path, O_CREAT | O_RDWR, 0666); - if (!FD_VALID(pResBuf->fd)) { - qError("failed to create tmp file: %s on disk. %s", pResBuf->path, strerror(errno)); - return TAOS_SYSTEM_ERROR(errno); - } - - int32_t ret = ftruncate(pResBuf->fd, pResBuf->numOfPages * DEFAULT_INTERN_BUF_PAGE_SIZE); - if (ret != TSDB_CODE_SUCCESS) { - qError("failed to create tmp file: %s on disk. %s", pResBuf->path, strerror(errno)); - return TAOS_SYSTEM_ERROR(errno); - } - - pResBuf->pBuf = mmap(NULL, pResBuf->totalBufSize, PROT_READ | PROT_WRITE, MAP_SHARED, pResBuf->fd, 0); - if (pResBuf->pBuf == MAP_FAILED) { - qError("QInfo:%p failed to map temp file: %s. %s", handle, pResBuf->path, strerror(errno)); - return TAOS_SYSTEM_ERROR(errno); - } - - qDebug("QInfo:%p create tmp file for output result:%s, %" PRId64 "bytes", handle, pResBuf->path, - pResBuf->totalBufSize); - - return TSDB_CODE_SUCCESS; -} - -int32_t getNumOfResultBufGroupId(SDiskbasedResultBuf* pResultBuf) { return taosHashGetSize(pResultBuf->idsTable); } - -int32_t getResBufSize(SDiskbasedResultBuf* pResultBuf) { return pResultBuf->totalBufSize; } - -static int32_t extendDiskFileSize(SDiskbasedResultBuf* pResultBuf, int32_t numOfPages) { - assert(pResultBuf->numOfPages * DEFAULT_INTERN_BUF_PAGE_SIZE == pResultBuf->totalBufSize); - - int32_t ret = munmap(pResultBuf->pBuf, pResultBuf->totalBufSize); - pResultBuf->numOfPages += numOfPages; - - /* - * disk-based output buffer is exhausted, try to extend the disk-based buffer, the available disk space may - * be insufficient - */ - ret = ftruncate(pResultBuf->fd, pResultBuf->numOfPages * DEFAULT_INTERN_BUF_PAGE_SIZE); - if (ret != 0) { - // dError("QInfo:%p failed to create intermediate result output file:%s. %s", pQInfo, pSupporter->extBufFile, - // strerror(errno)); - return TSDB_CODE_QRY_NO_DISKSPACE; - } - - pResultBuf->totalBufSize = pResultBuf->numOfPages * DEFAULT_INTERN_BUF_PAGE_SIZE; - pResultBuf->pBuf = mmap(NULL, pResultBuf->totalBufSize, PROT_READ | PROT_WRITE, MAP_SHARED, pResultBuf->fd, 0); - - if (pResultBuf->pBuf == MAP_FAILED) { - // dError("QInfo:%p failed to map temp file: %s. %s", pQInfo, pSupporter->extBufFile, strerror(errno)); - return TSDB_CODE_QRY_OUT_OF_MEMORY; - } - - return TSDB_CODE_SUCCESS; -} - -static FORCE_INLINE bool noMoreAvailablePages(SDiskbasedResultBuf* pResultBuf) { - return (pResultBuf->allocateId == pResultBuf->numOfPages - 1); -} - -static FORCE_INLINE int32_t getGroupIndex(SDiskbasedResultBuf* pResultBuf, int32_t groupId) { - assert(pResultBuf != NULL); - - char* p = taosHashGet(pResultBuf->idsTable, (const char*)&groupId, sizeof(int32_t)); - if (p == NULL) { // it is a new group id - return -1; - } - - int32_t slot = GET_INT32_VAL(p); - assert(slot >= 0 && slot < taosHashGetSize(pResultBuf->idsTable)); - - return slot; -} - -static int32_t addNewGroupId(SDiskbasedResultBuf* pResultBuf, int32_t groupId) { - int32_t num = getNumOfResultBufGroupId(pResultBuf); // the num is the newest allocated group id slot - taosHashPut(pResultBuf->idsTable, (const char*)&groupId, sizeof(int32_t), &num, sizeof(int32_t)); - - SArray* pa = taosArrayInit(1, sizeof(int32_t)); - taosArrayPush(pResultBuf->list, &pa); - - assert(taosArrayGetSize(pResultBuf->list) == taosHashGetSize(pResultBuf->idsTable)); - return num; -} - -static void registerPageId(SDiskbasedResultBuf* pResultBuf, int32_t groupId, int32_t pageId) { - int32_t slot = getGroupIndex(pResultBuf, groupId); - if (slot < 0) { - slot = addNewGroupId(pResultBuf, groupId); - } - - SIDList pList = taosArrayGetP(pResultBuf->list, slot); - taosArrayPush(pList, &pageId); -} - -tFilePage* getNewDataBuf(SDiskbasedResultBuf* pResultBuf, int32_t groupId, int32_t* pageId) { - if (noMoreAvailablePages(pResultBuf)) { - if (extendDiskFileSize(pResultBuf, pResultBuf->incStep) != TSDB_CODE_SUCCESS) { - return NULL; - } - } - - // register new id in this group - *pageId = (pResultBuf->allocateId++); - registerPageId(pResultBuf, groupId, *pageId); - - tFilePage* page = GET_RES_BUF_PAGE_BY_ID(pResultBuf, *pageId); - - // clear memory for the new page - memset(page, 0, DEFAULT_INTERN_BUF_PAGE_SIZE); - - return page; -} - -int32_t getNumOfRowsPerPage(SDiskbasedResultBuf* pResultBuf) { return pResultBuf->numOfRowsPerPage; } - -SIDList getDataBufPagesIdList(SDiskbasedResultBuf* pResultBuf, int32_t groupId) { - int32_t slot = getGroupIndex(pResultBuf, groupId); - if (slot < 0) { - return taosArrayInit(1, sizeof(int32_t)); - } else { - return taosArrayGetP(pResultBuf->list, slot); - } -} - -void destroyResultBuf(SDiskbasedResultBuf* pResultBuf, void* handle) { - if (pResultBuf == NULL) { - return; - } - - if (FD_VALID(pResultBuf->fd)) { - close(pResultBuf->fd); - } - - qDebug("QInfo:%p disk-based output buffer closed, %" PRId64 " bytes, file:%s", handle, pResultBuf->totalBufSize, pResultBuf->path); - munmap(pResultBuf->pBuf, pResultBuf->totalBufSize); - unlink(pResultBuf->path); - - tfree(pResultBuf->path); - - size_t size = taosArrayGetSize(pResultBuf->list); - for (int32_t i = 0; i < size; ++i) { - SArray* pa = taosArrayGetP(pResultBuf->list, i); - taosArrayDestroy(pa); - } - - taosArrayDestroy(pResultBuf->list); - taosHashCleanup(pResultBuf->idsTable); - - tfree(pResultBuf); -} - -int32_t getLastPageId(SIDList pList) { - size_t size = taosArrayGetSize(pList); - return *(int32_t*) taosArrayGet(pList, size - 1); -} - diff --git a/src/query/src/qresultbuf.c b/src/query/src/qresultbuf.c new file mode 100644 index 0000000000..646d8b7986 --- /dev/null +++ b/src/query/src/qresultbuf.c @@ -0,0 +1,218 @@ +#include "qresultbuf.h" +#include "hash.h" +#include "qextbuffer.h" +#include "queryLog.h" +#include "taoserror.h" + +int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t numOfPages, int32_t rowSize, + int32_t pagesize, int32_t inMemPages, void* handle) { + + *pResultBuf = calloc(1, sizeof(SDiskbasedResultBuf)); + SDiskbasedResultBuf* pResBuf = *pResultBuf; + if (pResBuf == NULL) { + return TSDB_CODE_COM_OUT_OF_MEMORY; + } + + pResBuf->pageSize = pagesize; + pResBuf->numOfPages = inMemPages; // all pages are in buffer in the first place + pResBuf->inMemPages = inMemPages; + assert(inMemPages <= numOfPages); + + pResBuf->numOfRowsPerPage = (pagesize - sizeof(tFilePage)) / rowSize; + + pResBuf->totalBufSize = pResBuf->numOfPages * pagesize; + pResBuf->incStep = 4; + pResBuf->allocateId = -1; + + pResBuf->iBuf = calloc(pResBuf->inMemPages, pResBuf->pageSize); + + // init id hash table + pResBuf->idsTable = taosHashInit(numOfPages, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false); + pResBuf->list = taosArrayInit(numOfPages, POINTER_BYTES); + + char path[PATH_MAX] = {0}; + getTmpfilePath("tsdb_qbuf", path); + pResBuf->path = strdup(path); + + pResBuf->fd = FD_INITIALIZER; + pResBuf->pBuf = NULL; + + qDebug("QInfo:%p create resBuf for output, page size:%d, initial pages:%d, %" PRId64 "bytes", handle, + pResBuf->pageSize, pResBuf->numOfPages, pResBuf->totalBufSize); + + return TSDB_CODE_SUCCESS; +} + +int32_t getNumOfResultBufGroupId(SDiskbasedResultBuf* pResultBuf) { return taosHashGetSize(pResultBuf->idsTable); } + +int32_t getResBufSize(SDiskbasedResultBuf* pResultBuf) { return pResultBuf->totalBufSize; } + +#define NUM_OF_PAGES_ON_DISK(_r) ((_r)->numOfPages - (_r)->inMemPages) +#define FILE_SIZE_ON_DISK(_r) (NUM_OF_PAGES_ON_DISK(_r) * (_r)->pageSize) + +static int32_t createDiskResidesBuf(SDiskbasedResultBuf* pResultBuf) { + pResultBuf->fd = open(pResultBuf->path, O_CREAT | O_RDWR, 0666); + if (!FD_VALID(pResultBuf->fd)) { + qError("failed to create tmp file: %s on disk. %s", pResultBuf->path, strerror(errno)); + return TAOS_SYSTEM_ERROR(errno); + } + + assert(pResultBuf->numOfPages == pResultBuf->inMemPages); + pResultBuf->numOfPages += pResultBuf->incStep; + + int32_t ret = ftruncate(pResultBuf->fd, NUM_OF_PAGES_ON_DISK(pResultBuf) * pResultBuf->pageSize); + if (ret != TSDB_CODE_SUCCESS) { + qError("failed to create tmp file: %s on disk. %s", pResultBuf->path, strerror(errno)); + return TAOS_SYSTEM_ERROR(errno); + } + + pResultBuf->pBuf = mmap(NULL, FILE_SIZE_ON_DISK(pResultBuf), PROT_READ | PROT_WRITE, MAP_SHARED, pResultBuf->fd, 0); + if (pResultBuf->pBuf == MAP_FAILED) { + qError("QInfo:%p failed to map temp file: %s. %s", pResultBuf->handle, pResultBuf->path, strerror(errno)); + return TAOS_SYSTEM_ERROR(errno); + } + + pResultBuf->totalBufSize = pResultBuf->numOfPages * pResultBuf->pageSize; + return TSDB_CODE_SUCCESS; +} + +static int32_t extendDiskFileSize(SDiskbasedResultBuf* pResultBuf, int32_t incNumOfPages) { + assert(pResultBuf->numOfPages * pResultBuf->pageSize == pResultBuf->totalBufSize); + int32_t ret = TSDB_CODE_SUCCESS; + + if (pResultBuf->pBuf == NULL) { + assert(pResultBuf->fd == FD_INITIALIZER); + + if ((ret = createDiskResidesBuf(pResultBuf)) != TSDB_CODE_SUCCESS) { + return ret; + } + } else { + ret = munmap(pResultBuf->pBuf, FILE_SIZE_ON_DISK(pResultBuf)); + pResultBuf->numOfPages += incNumOfPages; + + /* + * disk-based output buffer is exhausted, try to extend the disk-based buffer, the available disk space may + * be insufficient + */ + ret = ftruncate(pResultBuf->fd, NUM_OF_PAGES_ON_DISK(pResultBuf) * pResultBuf->pageSize); + if (ret != TSDB_CODE_SUCCESS) { + // dError("QInfo:%p failed to create intermediate result output file:%s. %s", pQInfo, pSupporter->extBufFile, + // strerror(errno)); + return TSDB_CODE_QRY_NO_DISKSPACE; + } + + pResultBuf->totalBufSize = pResultBuf->numOfPages * pResultBuf->pageSize; + pResultBuf->pBuf = mmap(NULL, FILE_SIZE_ON_DISK(pResultBuf), PROT_READ | PROT_WRITE, MAP_SHARED, pResultBuf->fd, 0); + + if (pResultBuf->pBuf == MAP_FAILED) { + // dError("QInfo:%p failed to map temp file: %s. %s", pQInfo, pSupporter->extBufFile, strerror(errno)); + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } + } + + return TSDB_CODE_SUCCESS; +} + +#define NO_AVAILABLE_PAGES(_b) ((_b)->allocateId == (_b)->numOfPages - 1) + +static FORCE_INLINE int32_t getGroupIndex(SDiskbasedResultBuf* pResultBuf, int32_t groupId) { + assert(pResultBuf != NULL); + + char* p = taosHashGet(pResultBuf->idsTable, (const char*)&groupId, sizeof(int32_t)); + if (p == NULL) { // it is a new group id + return -1; + } + + int32_t slot = GET_INT32_VAL(p); + assert(slot >= 0 && slot < taosHashGetSize(pResultBuf->idsTable)); + + return slot; +} + +static int32_t addNewGroupId(SDiskbasedResultBuf* pResultBuf, int32_t groupId) { + int32_t num = getNumOfResultBufGroupId(pResultBuf); // the num is the newest allocated group id slot + taosHashPut(pResultBuf->idsTable, (const char*)&groupId, sizeof(int32_t), &num, sizeof(int32_t)); + + SArray* pa = taosArrayInit(1, sizeof(int32_t)); + taosArrayPush(pResultBuf->list, &pa); + + assert(taosArrayGetSize(pResultBuf->list) == taosHashGetSize(pResultBuf->idsTable)); + return num; +} + +static void registerPageId(SDiskbasedResultBuf* pResultBuf, int32_t groupId, int32_t pageId) { + int32_t slot = getGroupIndex(pResultBuf, groupId); + if (slot < 0) { + slot = addNewGroupId(pResultBuf, groupId); + } + + SIDList pList = taosArrayGetP(pResultBuf->list, slot); + taosArrayPush(pList, &pageId); +} + +tFilePage* getNewDataBuf(SDiskbasedResultBuf* pResultBuf, int32_t groupId, int32_t* pageId) { + if (NO_AVAILABLE_PAGES(pResultBuf)) { + if (extendDiskFileSize(pResultBuf, pResultBuf->incStep) != TSDB_CODE_SUCCESS) { + return NULL; + } + } + + // register new id in this group + *pageId = (++pResultBuf->allocateId); + registerPageId(pResultBuf, groupId, *pageId); + + // clear memory for the new page + tFilePage* page = getResBufPage(pResultBuf, *pageId); + memset(page, 0, pResultBuf->pageSize); + + return page; +} + +int32_t getNumOfRowsPerPage(SDiskbasedResultBuf* pResultBuf) { return pResultBuf->numOfRowsPerPage; } + +SIDList getDataBufPagesIdList(SDiskbasedResultBuf* pResultBuf, int32_t groupId) { + int32_t slot = getGroupIndex(pResultBuf, groupId); + if (slot < 0) { + return taosArrayInit(1, sizeof(int32_t)); + } else { + return taosArrayGetP(pResultBuf->list, slot); + } +} + +void destroyResultBuf(SDiskbasedResultBuf* pResultBuf, void* handle) { + if (pResultBuf == NULL) { + return; + } + + if (FD_VALID(pResultBuf->fd)) { + qDebug("QInfo:%p disk-based output buffer closed, total:%" PRId64 " bytes, file created:%s, file size:%d", handle, + pResultBuf->totalBufSize, pResultBuf->path, FILE_SIZE_ON_DISK(pResultBuf)); + + close(pResultBuf->fd); + munmap(pResultBuf->pBuf, FILE_SIZE_ON_DISK(pResultBuf)); + pResultBuf->pBuf = NULL; + } else { + qDebug("QInfo:%p disk-based output buffer closed, total:%" PRId64 " bytes, no file created", handle, + pResultBuf->totalBufSize); + } + + unlink(pResultBuf->path); + tfree(pResultBuf->path); + + size_t size = taosArrayGetSize(pResultBuf->list); + for (int32_t i = 0; i < size; ++i) { + SArray* pa = taosArrayGetP(pResultBuf->list, i); + taosArrayDestroy(pa); + } + + taosArrayDestroy(pResultBuf->list); + taosHashCleanup(pResultBuf->idsTable); + + tfree(pResultBuf); +} + +int32_t getLastPageId(SIDList pList) { + size_t size = taosArrayGetSize(pList); + return *(int32_t*) taosArrayGet(pList, size - 1); +} + diff --git a/src/query/src/qUtil.c b/src/query/src/qutil.c similarity index 98% rename from src/query/src/qUtil.c rename to src/query/src/qutil.c index 4e71de830f..4c3595b0be 100644 --- a/src/query/src/qUtil.c +++ b/src/query/src/qutil.c @@ -19,19 +19,17 @@ #include "taosmsg.h" #include "ttime.h" -#include "qExecutor.h" -#include "qUtil.h" +#include "qexecutor.h" +#include "qutil.h" int32_t getOutputInterResultBufSize(SQuery* pQuery) { int32_t size = 0; for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - assert(pQuery->pSelectExpr[i].interBytes <= DEFAULT_INTERN_BUF_PAGE_SIZE); size += pQuery->pSelectExpr[i].interBytes; } assert(size > 0); - return size; } @@ -243,7 +241,7 @@ void clearTimeWindowResBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pWindow size_t size = pRuntimeEnv->pQuery->pSelectExpr[i].bytes; memset(s, 0, size); - resetResultInfo(pResultInfo); + RESET_RESULT_INFO(pResultInfo); } pWindowRes->numOfRows = 0; diff --git a/src/query/tests/resultBufferTest.cpp b/src/query/tests/resultBufferTest.cpp index 9270c6e458..bf6f3bb50f 100644 --- a/src/query/tests/resultBufferTest.cpp +++ b/src/query/tests/resultBufferTest.cpp @@ -2,8 +2,8 @@ #include #include +#include "qresultbuf.h" #include "taos.h" -#include "qresultBuf.h" #include "tsdb.h" namespace { diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 8463e791db..5cb0dffd46 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -1376,7 +1376,7 @@ static int32_t createDataBlocksInfo(STsdbQueryHandle* pQueryHandle, int32_t numO * } */ - tsdbDebug("%p %d data blocks sort completed", pQueryHandle, cnt); + tsdbDebug("%p %d data blocks sort completed, %p", pQueryHandle, cnt, pQueryHandle->qinfo); cleanBlockOrderSupporter(&sup, numOfTables); free(pTree); diff --git a/src/util/src/tutil.c b/src/util/src/tutil.c index 3764df4afc..7b73b1e17c 100644 --- a/src/util/src/tutil.c +++ b/src/util/src/tutil.c @@ -522,7 +522,7 @@ int32_t taosFileRename(char *fullPath, char *suffix, char delimiter, char **dstP void getTmpfilePath(const char *fileNamePrefix, char *dstPath) { const char* tdengineTmpFileNamePrefix = "tdengine-"; - char tmpPath[PATH_MAX] = {0}; + char tmpPath[PATH_MAX]; #ifdef WINDOWS char *tmpDir = getenv("tmp"); From 3cf5b7697b50a9986df2d8c4be27a3a809c65342 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 20 Jul 2020 00:02:51 +0800 Subject: [PATCH 032/100] [td-225] source files rename. --- src/client/inc/tscLocalMerge.h | 4 ++-- src/client/inc/tscUtil.h | 6 +++--- src/client/inc/tsclient.h | 4 ++-- src/client/src/tscFunctionImpl.c | 14 +++++++------- src/client/src/tscLocal.c | 12 ++++++------ src/client/src/tscSQLParser.c | 2 +- src/client/src/tscSql.c | 2 +- src/client/src/tscSubquery.c | 4 ++-- src/client/src/tscUtil.c | 6 +++--- src/query/inc/{qast.h => qAst.h} | 0 src/query/inc/{qexecutor.h => qExecutor.h} | 6 +++--- src/query/inc/{qextbuffer.h => qExtbuffer.h} | 2 +- src/query/inc/{qfill.h => qFill.h} | 2 +- src/query/inc/{qhistogram.h => qHistogram.h} | 0 src/query/inc/{qpercentile.h => qPercentile.h} | 2 +- src/query/inc/{qresultbuf.h => qResultbuf.h} | 4 ++-- ...qsyntaxtreefunction.h => qSyntaxtreefunction.h} | 0 src/query/inc/{qtsbuf.h => qTsbuf.h} | 0 src/query/inc/{qutil.h => qUtil.h} | 0 src/query/src/{qast.c => qAst.c} | 8 ++++---- src/query/src/{qexecutor.c => qExecutor.c} | 12 ++++++------ src/query/src/{qextbuffer.c => qExtbuffer.c} | 7 +++---- src/query/src/{qfill.c => qFill.c} | 4 ++-- src/query/src/{qfilterfunc.c => qFilterfunc.c} | 2 +- src/query/src/{qhistogram.c => qHistogram.c} | 2 +- src/query/src/{qparserImpl.c => qParserImpl.c} | 0 src/query/src/{qpercentile.c => qPercentile.c} | 6 +++--- src/query/src/{qresultbuf.c => qResultbuf.c} | 4 ++-- ...qsyntaxtreefunction.c => qSyntaxtreefunction.c} | 2 +- src/query/src/{qtokenizer.c => qTokenizer.c} | 0 src/query/src/{qtsbuf.c => qTsbuf.c} | 4 ++-- src/query/src/{qutil.c => qUtil.c} | 4 ++-- src/query/tests/astTest.cpp | 2 +- src/query/tests/histogramTest.cpp | 2 +- src/query/tests/resultBufferTest.cpp | 2 +- src/query/tests/tsBufTest.cpp | 2 +- 36 files changed, 66 insertions(+), 67 deletions(-) rename src/query/inc/{qast.h => qAst.h} (100%) rename src/query/inc/{qexecutor.h => qExecutor.h} (99%) rename src/query/inc/{qextbuffer.h => qExtbuffer.h} (98%) rename src/query/inc/{qfill.h => qFill.h} (99%) rename src/query/inc/{qhistogram.h => qHistogram.h} (100%) rename src/query/inc/{qpercentile.h => qPercentile.h} (98%) rename src/query/inc/{qresultbuf.h => qResultbuf.h} (99%) rename src/query/inc/{qsyntaxtreefunction.h => qSyntaxtreefunction.h} (100%) rename src/query/inc/{qtsbuf.h => qTsbuf.h} (100%) rename src/query/inc/{qutil.h => qUtil.h} (100%) rename src/query/src/{qast.c => qAst.c} (99%) rename src/query/src/{qexecutor.c => qExecutor.c} (99%) rename src/query/src/{qextbuffer.c => qExtbuffer.c} (99%) rename src/query/src/{qfill.c => qFill.c} (99%) rename src/query/src/{qfilterfunc.c => qFilterfunc.c} (99%) rename src/query/src/{qhistogram.c => qHistogram.c} (99%) rename src/query/src/{qparserImpl.c => qParserImpl.c} (100%) rename src/query/src/{qpercentile.c => qPercentile.c} (99%) rename src/query/src/{qresultbuf.c => qResultbuf.c} (99%) rename src/query/src/{qsyntaxtreefunction.c => qSyntaxtreefunction.c} (99%) rename src/query/src/{qtokenizer.c => qTokenizer.c} (100%) rename src/query/src/{qtsbuf.c => qTsbuf.c} (99%) rename src/query/src/{qutil.c => qUtil.c} (99%) diff --git a/src/client/inc/tscLocalMerge.h b/src/client/inc/tscLocalMerge.h index d38885ab2e..397a60d140 100644 --- a/src/client/inc/tscLocalMerge.h +++ b/src/client/inc/tscLocalMerge.h @@ -20,8 +20,8 @@ extern "C" { #endif -#include "qextbuffer.h" -#include "qfill.h" +#include "qExtbuffer.h" +#include "qFill.h" #include "taosmsg.h" #include "tlosertree.h" #include "tsclient.h" diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index 786133a8f3..590f205e1d 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -23,11 +23,11 @@ extern "C" { /* * @date 2018/09/30 */ -#include "os.h" -#include "tbuffer.h" #include "exception.h" -#include "qextbuffer.h" +#include "os.h" +#include "qExtbuffer.h" #include "taosdef.h" +#include "tbuffer.h" #include "tscLocalMerge.h" #include "tsclient.h" diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index 3f548c18ca..3b19a475dc 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -30,9 +30,9 @@ extern "C" { #include "tsqlfunction.h" #include "tutil.h" -#include "qexecutor.h" +#include "qExecutor.h" +#include "qTsbuf.h" #include "qsqlparser.h" -#include "qtsbuf.h" #include "tcmdtype.h" // forward declaration diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c index fb15e4efd5..72ccd5adc6 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/client/src/tscFunctionImpl.c @@ -14,15 +14,15 @@ */ #include "os.h" -#include "qextbuffer.h" -#include "qfill.h" -#include "qhistogram.h" -#include "qpercentile.h" -#include "qsyntaxtreefunction.h" -#include "qtsbuf.h" +#include "qAst.h" +#include "qExtbuffer.h" +#include "qFill.h" +#include "qHistogram.h" +#include "qPercentile.h" +#include "qSyntaxtreefunction.h" +#include "qTsbuf.h" #include "taosdef.h" #include "taosmsg.h" -#include "qast.h" #include "tscLog.h" #include "tscSubquery.h" #include "tscompression.h" diff --git a/src/client/src/tscLocal.c b/src/client/src/tscLocal.c index 83700ce0a5..7f336daa91 100644 --- a/src/client/src/tscLocal.c +++ b/src/client/src/tscLocal.c @@ -16,14 +16,14 @@ #include "os.h" #include "taosmsg.h" -#include "tcache.h" -#include "tscUtil.h" -#include "tsclient.h" +#include "qExtbuffer.h" #include "taosdef.h" -#include "tscLog.h" -#include "qextbuffer.h" -#include "tschemautil.h" +#include "tcache.h" #include "tname.h" +#include "tscLog.h" +#include "tscUtil.h" +#include "tschemautil.h" +#include "tsclient.h" static void tscSetLocalQueryResult(SSqlObj *pSql, const char *val, const char *columnName, int16_t type, size_t valueLength); diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index b97e486449..09db65a105 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -18,9 +18,9 @@ #define _DEFAULT_SOURCE #include "os.h" +#include "qAst.h" #include "taos.h" #include "taosmsg.h" -#include "qast.h" #include "tcompare.h" #include "tname.h" #include "tscLog.h" diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index 6a14d3a65e..77584f168c 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -15,7 +15,7 @@ #include "hash.h" #include "os.h" -#include "qast.h" +#include "qAst.h" #include "tcache.h" #include "tnote.h" #include "trpc.h" diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 1dbc52efb0..5d26d09fae 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -14,8 +14,8 @@ */ #include "os.h" -#include "qtsbuf.h" -#include "qast.h" +#include "qAst.h" +#include "qTsbuf.h" #include "tcompare.h" #include "tscLog.h" #include "tscSubquery.h" diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 4e6133663e..27f6727339 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -13,11 +13,11 @@ * along with this program. If not, see . */ -#include "os.h" -#include "hash.h" #include "tscUtil.h" +#include "hash.h" +#include "os.h" +#include "qAst.h" #include "taosmsg.h" -#include "qast.h" #include "tcache.h" #include "tkey.h" #include "tmd5.h" diff --git a/src/query/inc/qast.h b/src/query/inc/qAst.h similarity index 100% rename from src/query/inc/qast.h rename to src/query/inc/qAst.h diff --git a/src/query/inc/qexecutor.h b/src/query/inc/qExecutor.h similarity index 99% rename from src/query/inc/qexecutor.h rename to src/query/inc/qExecutor.h index 97a7565de8..127c38a6f8 100644 --- a/src/query/inc/qexecutor.h +++ b/src/query/inc/qExecutor.h @@ -18,10 +18,10 @@ #include "os.h" #include "hash.h" -#include "qfill.h" -#include "qresultbuf.h" +#include "qFill.h" +#include "qResultbuf.h" +#include "qTsbuf.h" #include "qsqlparser.h" -#include "qtsbuf.h" #include "query.h" #include "taosdef.h" #include "tarray.h" diff --git a/src/query/inc/qextbuffer.h b/src/query/inc/qExtbuffer.h similarity index 98% rename from src/query/inc/qextbuffer.h rename to src/query/inc/qExtbuffer.h index 2cbef2b1be..b57c48933f 100644 --- a/src/query/inc/qextbuffer.h +++ b/src/query/inc/qExtbuffer.h @@ -28,7 +28,7 @@ extern "C" { #include "tdataformat.h" #include "talgo.h" -#define DEFAULT_PAGE_SIZE (1024L*64) // 16k larger than the SHistoInfo +#define DEFAULT_PAGE_SIZE (1024L*4) // 16k larger than the SHistoInfo #define MAX_TMPFILE_PATH_LENGTH PATH_MAX #define INITIAL_ALLOCATION_BUFFER_SIZE 64 diff --git a/src/query/inc/qfill.h b/src/query/inc/qFill.h similarity index 99% rename from src/query/inc/qfill.h rename to src/query/inc/qFill.h index ee5974708a..db6a69c2c5 100644 --- a/src/query/inc/qfill.h +++ b/src/query/inc/qFill.h @@ -21,8 +21,8 @@ extern "C" { #endif #include "os.h" +#include "qExtbuffer.h" #include "taosdef.h" -#include "qextbuffer.h" typedef struct { STColumn col; // column info diff --git a/src/query/inc/qhistogram.h b/src/query/inc/qHistogram.h similarity index 100% rename from src/query/inc/qhistogram.h rename to src/query/inc/qHistogram.h diff --git a/src/query/inc/qpercentile.h b/src/query/inc/qPercentile.h similarity index 98% rename from src/query/inc/qpercentile.h rename to src/query/inc/qPercentile.h index c1227dad77..52f666c338 100644 --- a/src/query/inc/qpercentile.h +++ b/src/query/inc/qPercentile.h @@ -16,7 +16,7 @@ #ifndef TDENGINE_QPERCENTILE_H #define TDENGINE_QPERCENTILE_H -#include "qextbuffer.h" +#include "qExtbuffer.h" typedef struct MinMaxEntry { union { diff --git a/src/query/inc/qresultbuf.h b/src/query/inc/qResultbuf.h similarity index 99% rename from src/query/inc/qresultbuf.h rename to src/query/inc/qResultbuf.h index f901474cc1..41fccd177d 100644 --- a/src/query/inc/qresultbuf.h +++ b/src/query/inc/qResultbuf.h @@ -20,9 +20,9 @@ extern "C" { #endif -#include "os.h" -#include "qextbuffer.h" #include "hash.h" +#include "os.h" +#include "qExtbuffer.h" typedef struct SArray* SIDList; diff --git a/src/query/inc/qsyntaxtreefunction.h b/src/query/inc/qSyntaxtreefunction.h similarity index 100% rename from src/query/inc/qsyntaxtreefunction.h rename to src/query/inc/qSyntaxtreefunction.h diff --git a/src/query/inc/qtsbuf.h b/src/query/inc/qTsbuf.h similarity index 100% rename from src/query/inc/qtsbuf.h rename to src/query/inc/qTsbuf.h diff --git a/src/query/inc/qutil.h b/src/query/inc/qUtil.h similarity index 100% rename from src/query/inc/qutil.h rename to src/query/inc/qUtil.h diff --git a/src/query/src/qast.c b/src/query/src/qAst.c similarity index 99% rename from src/query/src/qast.c rename to src/query/src/qAst.c index 42f9f214f5..c2578c15c0 100644 --- a/src/query/src/qast.c +++ b/src/query/src/qAst.c @@ -16,17 +16,17 @@ #include "os.h" -#include "tname.h" -#include "qast.h" -#include "tsdb.h" #include "exception.h" +#include "qAst.h" +#include "qSyntaxtreefunction.h" #include "qsqlparser.h" -#include "qsyntaxtreefunction.h" #include "taosdef.h" #include "taosmsg.h" #include "tarray.h" #include "tbuffer.h" #include "tcompare.h" +#include "tname.h" +#include "tsdb.h" #include "tskiplist.h" #include "tsqlfunction.h" #include "tstoken.h" diff --git a/src/query/src/qexecutor.c b/src/query/src/qExecutor.c similarity index 99% rename from src/query/src/qexecutor.c rename to src/query/src/qExecutor.c index 9348f702d0..b4f825a7c2 100644 --- a/src/query/src/qexecutor.c +++ b/src/query/src/qExecutor.c @@ -13,19 +13,19 @@ * along with this program. If not, see . */ #include "os.h" +#include "qFill.h" +#include "taosmsg.h" #include "tcache.h" #include "tglobal.h" -#include "qfill.h" -#include "taosmsg.h" #include "exception.h" #include "hash.h" -#include "qast.h" -#include "qexecutor.h" -#include "qresultbuf.h" +#include "qAst.h" +#include "qExecutor.h" +#include "qResultbuf.h" +#include "qUtil.h" #include "query.h" #include "queryLog.h" -#include "qutil.h" #include "tlosertree.h" #include "tscompression.h" #include "ttime.h" diff --git a/src/query/src/qextbuffer.c b/src/query/src/qExtbuffer.c similarity index 99% rename from src/query/src/qextbuffer.c rename to src/query/src/qExtbuffer.c index afcf902123..69c5f0e24f 100644 --- a/src/query/src/qextbuffer.c +++ b/src/query/src/qExtbuffer.c @@ -12,16 +12,15 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ +#include "qExtbuffer.h" #include "os.h" -#include "tulog.h" -#include "qextbuffer.h" +#include "queryLog.h" #include "taos.h" #include "taosdef.h" #include "taosmsg.h" #include "tsqlfunction.h" -#include "ttime.h" +#include "tulog.h" #include "tutil.h" -#include "queryLog.h" #define COLMODEL_GET_VAL(data, schema, allrow, rowId, colId) \ (data + (schema)->pFields[colId].offset * (allrow) + (rowId) * (schema)->pFields[colId].field.bytes) diff --git a/src/query/src/qfill.c b/src/query/src/qFill.c similarity index 99% rename from src/query/src/qfill.c rename to src/query/src/qFill.c index 65951a5b9e..9dec2598bc 100644 --- a/src/query/src/qfill.c +++ b/src/query/src/qFill.c @@ -13,9 +13,9 @@ * along with this program. If not, see . */ +#include "qFill.h" #include "os.h" -#include "qfill.h" -#include "qextbuffer.h" +#include "qExtbuffer.h" #include "taosdef.h" #include "taosmsg.h" #include "tsqlfunction.h" diff --git a/src/query/src/qfilterfunc.c b/src/query/src/qFilterfunc.c similarity index 99% rename from src/query/src/qfilterfunc.c rename to src/query/src/qFilterfunc.c index 6f336434e1..7e9f5c7da5 100644 --- a/src/query/src/qfilterfunc.c +++ b/src/query/src/qFilterfunc.c @@ -16,7 +16,7 @@ #define _DEFAULT_SOURCE #include "os.h" -#include "qexecutor.h" +#include "qExecutor.h" #include "taosmsg.h" #include "tcompare.h" #include "tsqlfunction.h" diff --git a/src/query/src/qhistogram.c b/src/query/src/qHistogram.c similarity index 99% rename from src/query/src/qhistogram.c rename to src/query/src/qHistogram.c index 26482e9f14..7835d82469 100644 --- a/src/query/src/qhistogram.c +++ b/src/query/src/qHistogram.c @@ -14,7 +14,7 @@ */ #include "os.h" -#include "qhistogram.h" +#include "qHistogram.h" #include "taosdef.h" #include "taosmsg.h" #include "tlosertree.h" diff --git a/src/query/src/qparserImpl.c b/src/query/src/qParserImpl.c similarity index 100% rename from src/query/src/qparserImpl.c rename to src/query/src/qParserImpl.c diff --git a/src/query/src/qpercentile.c b/src/query/src/qPercentile.c similarity index 99% rename from src/query/src/qpercentile.c rename to src/query/src/qPercentile.c index dc5ecb796a..c4490a01e7 100644 --- a/src/query/src/qpercentile.c +++ b/src/query/src/qPercentile.c @@ -13,12 +13,12 @@ * along with this program. If not, see . */ +#include "qPercentile.h" #include "os.h" -#include "tulog.h" -#include "qpercentile.h" +#include "queryLog.h" #include "taosdef.h" #include "taosmsg.h" -#include "queryLog.h" +#include "tulog.h" tExtMemBuffer *releaseBucketsExceptFor(tMemBucket *pMemBucket, int16_t segIdx, int16_t slotIdx) { tExtMemBuffer *pBuffer = NULL; diff --git a/src/query/src/qresultbuf.c b/src/query/src/qResultbuf.c similarity index 99% rename from src/query/src/qresultbuf.c rename to src/query/src/qResultbuf.c index 646d8b7986..f2194bbfb5 100644 --- a/src/query/src/qresultbuf.c +++ b/src/query/src/qResultbuf.c @@ -1,6 +1,6 @@ -#include "qresultbuf.h" +#include "qResultbuf.h" #include "hash.h" -#include "qextbuffer.h" +#include "qExtbuffer.h" #include "queryLog.h" #include "taoserror.h" diff --git a/src/query/src/qsyntaxtreefunction.c b/src/query/src/qSyntaxtreefunction.c similarity index 99% rename from src/query/src/qsyntaxtreefunction.c rename to src/query/src/qSyntaxtreefunction.c index 5719bb0188..2104edfd91 100644 --- a/src/query/src/qsyntaxtreefunction.c +++ b/src/query/src/qSyntaxtreefunction.c @@ -15,7 +15,7 @@ #include "os.h" -#include "qsyntaxtreefunction.h" +#include "qSyntaxtreefunction.h" #include "taosdef.h" #include "tutil.h" diff --git a/src/query/src/qtokenizer.c b/src/query/src/qTokenizer.c similarity index 100% rename from src/query/src/qtokenizer.c rename to src/query/src/qTokenizer.c diff --git a/src/query/src/qtsbuf.c b/src/query/src/qTsbuf.c similarity index 99% rename from src/query/src/qtsbuf.c rename to src/query/src/qTsbuf.c index b84fbded38..20b29107f5 100644 --- a/src/query/src/qtsbuf.c +++ b/src/query/src/qTsbuf.c @@ -1,7 +1,7 @@ -#include "qtsbuf.h" +#include "qTsbuf.h" +#include "taoserror.h" #include "tscompression.h" #include "tutil.h" -#include "taoserror.h" static int32_t getDataStartOffset(); static void TSBufUpdateVnodeInfo(STSBuf* pTSBuf, int32_t index, STSVnodeBlockInfo* pBlockInfo); diff --git a/src/query/src/qutil.c b/src/query/src/qUtil.c similarity index 99% rename from src/query/src/qutil.c rename to src/query/src/qUtil.c index 4c3595b0be..be84471493 100644 --- a/src/query/src/qutil.c +++ b/src/query/src/qUtil.c @@ -19,8 +19,8 @@ #include "taosmsg.h" #include "ttime.h" -#include "qexecutor.h" -#include "qutil.h" +#include "qExecutor.h" +#include "qUtil.h" int32_t getOutputInterResultBufSize(SQuery* pQuery) { int32_t size = 0; diff --git a/src/query/tests/astTest.cpp b/src/query/tests/astTest.cpp index df27087216..99f03a7ff8 100644 --- a/src/query/tests/astTest.cpp +++ b/src/query/tests/astTest.cpp @@ -3,8 +3,8 @@ #include #include +#include "qAst.h" #include "taosmsg.h" -#include "qast.h" #include "tsdb.h" #include "tskiplist.h" diff --git a/src/query/tests/histogramTest.cpp b/src/query/tests/histogramTest.cpp index c23f0f5924..4a5f7fbbbe 100644 --- a/src/query/tests/histogramTest.cpp +++ b/src/query/tests/histogramTest.cpp @@ -9,7 +9,7 @@ #include "tstoken.h" #include "tutil.h" -#include "qhistogram.h" +#include "qHistogram.h" /* test validate the names for table/database */ TEST(testCase, histogram_binary_search) { diff --git a/src/query/tests/resultBufferTest.cpp b/src/query/tests/resultBufferTest.cpp index bf6f3bb50f..f822deeb73 100644 --- a/src/query/tests/resultBufferTest.cpp +++ b/src/query/tests/resultBufferTest.cpp @@ -2,7 +2,7 @@ #include #include -#include "qresultbuf.h" +#include "qResultbuf.h" #include "taos.h" #include "tsdb.h" diff --git a/src/query/tests/tsBufTest.cpp b/src/query/tests/tsBufTest.cpp index f813261957..e9827518e1 100644 --- a/src/query/tests/tsBufTest.cpp +++ b/src/query/tests/tsBufTest.cpp @@ -5,10 +5,10 @@ #include "taos.h" #include "tsdb.h" +#include "qTsbuf.h" #include "tstoken.h" #include "ttime.h" #include "tutil.h" -#include "qtsbuf.h" namespace { /** From 731707608c9391392db8b95ab910f7f3fc6301d4 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 20 Jul 2020 00:04:21 +0800 Subject: [PATCH 033/100] [td-225] fix bugs in descending traverse over in-men data --- src/tsdb/src/tsdbRead.c | 79 ++++++++++++++++++++++++----------------- 1 file changed, 47 insertions(+), 32 deletions(-) diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 5cb0dffd46..5b1afe3da8 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -21,7 +21,7 @@ #include "tcompare.h" #include "exception.h" -#include "../../../query/inc/qast.h" // todo move to common module +#include "../../query/inc/qAst.h" // todo move to common module #include "tlosertree.h" #include "tsdb.h" #include "tsdbMain.h" @@ -331,7 +331,8 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh tsdbDebug("%p uid:%" PRId64", tid:%d check data in mem from skey:%" PRId64 ", order:%d, %p", pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, key, order, pHandle->qinfo); } else { - tsdbDebug("%p uid:%" PRId64 ", tid:%d no data in mem", pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid); + tsdbDebug("%p uid:%"PRId64", tid:%d no data in mem, %p", pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, + pHandle->qinfo); } if (!imemEmpty) { @@ -343,7 +344,8 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh tsdbDebug("%p uid:%" PRId64", tid:%d check data in imem from skey:%" PRId64 ", order:%d, %p", pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, key, order, pHandle->qinfo); } else { - tsdbDebug("%p uid:%"PRId64", tid:%d no data in imem", pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid); + tsdbDebug("%p uid:%"PRId64", tid:%d no data in imem, %p", pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, + pHandle->qinfo); } return true; @@ -354,7 +356,7 @@ static void destroyTableMemIterator(STableCheckInfo* pCheckInfo) { tSkipListDestroyIter(pCheckInfo->iiter); } -SDataRow getSDataRowInTableMem(STableCheckInfo* pCheckInfo) { +SDataRow getSDataRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order) { SDataRow rmem = NULL, rimem = NULL; if (pCheckInfo->iter) { SSkipListNode* node = tSkipListIterGet(pCheckInfo->iter); @@ -371,20 +373,35 @@ SDataRow getSDataRowInTableMem(STableCheckInfo* pCheckInfo) { } if (rmem != NULL && rimem != NULL) { - if (dataRowKey(rmem) < dataRowKey(rimem)) { - pCheckInfo->chosen = 0; - return rmem; - } else if (dataRowKey(rmem) == dataRowKey(rimem)) { - // data ts are duplicated, ignore the data in mem + TSKEY r1 = dataRowKey(rmem); + TSKEY r2 = dataRowKey(rimem); + + if (r1 == r2) { // data ts are duplicated, ignore the data in mem tSkipListIterNext(pCheckInfo->iter); pCheckInfo->chosen = 1; return rimem; } else { - pCheckInfo->chosen = 1; - return rimem; + if (ASCENDING_TRAVERSE(order)) { + if (r1 < r2) { + pCheckInfo->chosen = 0; + return rmem; + } else { + pCheckInfo->chosen = 1; + return rimem; + } + } else { + if (r1 < r2) { + pCheckInfo->chosen = 1; + return rimem; + } else { + pCheckInfo->chosen = 0; + return rmem; + } + } } } + // at least one (rmem or rimem) is absent here if (rmem != NULL) { pCheckInfo->chosen = 0; return rmem; @@ -398,7 +415,7 @@ SDataRow getSDataRowInTableMem(STableCheckInfo* pCheckInfo) { return NULL; } -static bool moveToNextRow(STableCheckInfo* pCheckInfo) { +static bool moveToNextRowInMem(STableCheckInfo* pCheckInfo) { bool hasNext = false; if (pCheckInfo->chosen == 0) { if (pCheckInfo->iter != NULL) { @@ -412,19 +429,17 @@ static bool moveToNextRow(STableCheckInfo* pCheckInfo) { if (pCheckInfo->iiter != NULL) { return tSkipListIterGet(pCheckInfo->iiter) != NULL; } - } else { - if (pCheckInfo->chosen == 1) { - if (pCheckInfo->iiter != NULL) { - hasNext = tSkipListIterNext(pCheckInfo->iiter); - } + } else { //pCheckInfo->chosen == 1 + if (pCheckInfo->iiter != NULL) { + hasNext = tSkipListIterNext(pCheckInfo->iiter); + } - if (hasNext) { - return hasNext; - } + if (hasNext) { + return hasNext; + } - if (pCheckInfo->iter != NULL) { - return tSkipListIterGet(pCheckInfo->iter) != NULL; - } + if (pCheckInfo->iter != NULL) { + return tSkipListIterGet(pCheckInfo->iter) != NULL; } } @@ -445,7 +460,7 @@ static bool hasMoreDataInCache(STsdbQueryHandle* pHandle) { initTableMemIterator(pHandle, pCheckInfo); } - SDataRow row = getSDataRowInTableMem(pCheckInfo); + SDataRow row = getSDataRowInTableMem(pCheckInfo, pHandle->order); if (row == NULL) { return false; } @@ -650,7 +665,7 @@ static void handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SCompBlock* SDataBlockInfo binfo = GET_FILE_DATA_BLOCK_INFO(pCheckInfo, pBlock); /*bool hasData = */ initTableMemIterator(pQueryHandle, pCheckInfo); - SDataRow row = getSDataRowInTableMem(pCheckInfo); + SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order); TSKEY key = (row != NULL)? dataRowKey(row):TSKEY_INITIAL_VAL; cur->pos = ASCENDING_TRAVERSE(pQueryHandle->order)? 0:(binfo.rows-1); @@ -1033,7 +1048,7 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* } else if (pCheckInfo->iter != NULL || pCheckInfo->iiter != NULL) { SSkipListNode* node = NULL; do { - SDataRow row = getSDataRowInTableMem(pCheckInfo); + SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order); if (row == NULL) { break; } @@ -1061,9 +1076,9 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* cur->lastKey = key + step; cur->mixBlock = true; - moveToNextRow(pCheckInfo); + moveToNextRowInMem(pCheckInfo); } else if (key == tsArray[pos]) { // data in buffer has the same timestamp of data in file block, ignore it - moveToNextRow(pCheckInfo); + moveToNextRowInMem(pCheckInfo); } else if ((key > tsArray[pos] && ASCENDING_TRAVERSE(pQueryHandle->order)) || (key < tsArray[pos] && !ASCENDING_TRAVERSE(pQueryHandle->order))) { if (cur->win.skey == TSKEY_INITIAL_VAL) { @@ -1072,7 +1087,7 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* int32_t end = doBinarySearchKey(pCols->cols[0].pData, pCols->numOfRows, key, order); if (tsArray[end] == key) { // the value of key in cache equals to the end timestamp value, ignore it - moveToNextRow(pCheckInfo); + moveToNextRowInMem(pCheckInfo); } int32_t start = -1; @@ -1754,7 +1769,7 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int STable* pTable = pCheckInfo->pTableObj; do { - SDataRow row = getSDataRowInTableMem(pCheckInfo); + SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order); if (row == NULL) { break; } @@ -1775,11 +1790,11 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int copyOneRowFromMem(pQueryHandle, maxRowsToRead, numOfRows, row, pMeta, numOfCols, pTable); if (++numOfRows >= maxRowsToRead) { - moveToNextRow(pCheckInfo); + moveToNextRowInMem(pCheckInfo); break; } - } while(moveToNextRow(pCheckInfo)); + } while(moveToNextRowInMem(pCheckInfo)); assert(numOfRows <= maxRowsToRead); From 31349ebbae85249454f293feff7568fdda361a0d Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 20 Jul 2020 01:12:37 +0800 Subject: [PATCH 034/100] [td-225] fix memory leaks; --- src/query/src/qResultbuf.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/query/src/qResultbuf.c b/src/query/src/qResultbuf.c index f2194bbfb5..0eefb106b9 100644 --- a/src/query/src/qResultbuf.c +++ b/src/query/src/qResultbuf.c @@ -207,7 +207,8 @@ void destroyResultBuf(SDiskbasedResultBuf* pResultBuf, void* handle) { taosArrayDestroy(pResultBuf->list); taosHashCleanup(pResultBuf->idsTable); - + + tfree(pResultBuf->iBuf); tfree(pResultBuf); } From 5a448cd9a3b8c3b53fe627f730b36c4f4dbe796f Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 20 Jul 2020 11:10:22 +0800 Subject: [PATCH 035/100] [td-225] fix bugs in test cases. --- src/query/inc/qResultbuf.h | 4 ++-- src/query/tests/resultBufferTest.cpp | 5 ++--- src/util/inc/tutil.h | 4 ++-- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/query/inc/qResultbuf.h b/src/query/inc/qResultbuf.h index 41fccd177d..7df39c00fd 100644 --- a/src/query/inc/qResultbuf.h +++ b/src/query/inc/qResultbuf.h @@ -90,9 +90,9 @@ SIDList getDataBufPagesIdList(SDiskbasedResultBuf* pResultBuf, int32_t groupId); //#define getResBufPage(buf, id) ((tFilePage*)((buf)->pBuf + (buf)->pageSize * (id))) static FORCE_INLINE tFilePage* getResBufPage(SDiskbasedResultBuf* pResultBuf, int32_t id) { if (id < pResultBuf->inMemPages) { - return pResultBuf->iBuf + id * pResultBuf->pageSize; + return (tFilePage*) ((char*) pResultBuf->iBuf + id * pResultBuf->pageSize); } else { - return pResultBuf->pBuf + (id - pResultBuf->inMemPages) * pResultBuf->pageSize; + return (tFilePage*) ((char*) pResultBuf->pBuf + (id - pResultBuf->inMemPages) * pResultBuf->pageSize); } } /** diff --git a/src/query/tests/resultBufferTest.cpp b/src/query/tests/resultBufferTest.cpp index f822deeb73..63ed89ab9f 100644 --- a/src/query/tests/resultBufferTest.cpp +++ b/src/query/tests/resultBufferTest.cpp @@ -10,7 +10,7 @@ namespace { // simple test void simpleTest() { SDiskbasedResultBuf* pResultBuf = NULL; - int32_t ret = createDiskbasedResultBuffer(&pResultBuf, 1000, 64, NULL); + int32_t ret = createDiskbasedResultBuffer(&pResultBuf, 1000, 64, 1024, 4, NULL); int32_t pageId = 0; int32_t groupId = 0; @@ -22,8 +22,7 @@ void simpleTest() { ASSERT_EQ(getResBufSize(pResultBuf), 1000*16384L); SIDList list = getDataBufPagesIdList(pResultBuf, groupId); - ASSERT_EQ(list.size, 1); - + ASSERT_EQ(taosArrayGetSize(list), 1); ASSERT_EQ(getNumOfResultBufGroupId(pResultBuf), 1); destroyResultBuf(pResultBuf, NULL); diff --git a/src/util/inc/tutil.h b/src/util/inc/tutil.h index f7c69e3973..34f35c3807 100644 --- a/src/util/inc/tutil.h +++ b/src/util/inc/tutil.h @@ -35,12 +35,12 @@ extern "C" { #define WCHAR wchar_t #define tfree(x) \ - { \ + do { \ if (x) { \ free((void *)(x)); \ x = 0; \ } \ - } + } while(0); #define tstrncpy(dst, src, size) \ do { \ From ac1d0a266b39c5d8dfc2e17f78d0da51b0da67d7 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 20 Jul 2020 11:36:12 +0800 Subject: [PATCH 036/100] fix query unittest compile error. [TD-964] --- src/query/tests/resultBufferTest.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/query/tests/resultBufferTest.cpp b/src/query/tests/resultBufferTest.cpp index 9270c6e458..aaea0836d1 100644 --- a/src/query/tests/resultBufferTest.cpp +++ b/src/query/tests/resultBufferTest.cpp @@ -22,7 +22,7 @@ void simpleTest() { ASSERT_EQ(getResBufSize(pResultBuf), 1000*16384L); SIDList list = getDataBufPagesIdList(pResultBuf, groupId); - ASSERT_EQ(list.size, 1); + ASSERT_EQ(taosArrayGetSize(list), 1); ASSERT_EQ(getNumOfResultBufGroupId(pResultBuf), 1); From 9213011d184c77ecdb6dc2035693726352000200 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 20 Jul 2020 12:32:58 +0800 Subject: [PATCH 037/100] [td-225] add the stop check during scan files --- src/query/inc/qResultbuf.h | 1 - src/tsdb/inc/tsdbMain.h | 1 + src/tsdb/src/tsdbMemTable.c | 3 +-- src/tsdb/src/tsdbRead.c | 38 +++++++++++++++++++++++-------------- 4 files changed, 26 insertions(+), 17 deletions(-) diff --git a/src/query/inc/qResultbuf.h b/src/query/inc/qResultbuf.h index 7df39c00fd..d76659cc91 100644 --- a/src/query/inc/qResultbuf.h +++ b/src/query/inc/qResultbuf.h @@ -87,7 +87,6 @@ SIDList getDataBufPagesIdList(SDiskbasedResultBuf* pResultBuf, int32_t groupId); * @param id * @return */ -//#define getResBufPage(buf, id) ((tFilePage*)((buf)->pBuf + (buf)->pageSize * (id))) static FORCE_INLINE tFilePage* getResBufPage(SDiskbasedResultBuf* pResultBuf, int32_t id) { if (id < pResultBuf->inMemPages) { return (tFilePage*) ((char*) pResultBuf->iBuf + id * pResultBuf->pageSize); diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h index 40f2dac660..210d95853c 100644 --- a/src/tsdb/inc/tsdbMain.h +++ b/src/tsdb/inc/tsdbMain.h @@ -426,6 +426,7 @@ int tsdbUpdateFileHeader(SFile* pFile, uint32_t version); int tsdbEncodeSFileInfo(void** buf, const STsdbFileInfo* pInfo); void* tsdbDecodeSFileInfo(void* buf, STsdbFileInfo* pInfo); void tsdbRemoveFileGroup(STsdbRepo* pRepo, SFileGroup* pFGroup); +void tsdbGetFidKeyRange(int daysPerFile, int8_t precision, int fileId, TSKEY *minKey, TSKEY *maxKey); // ------------------ tsdbRWHelper.c #define TSDB_HELPER_CLEAR_STATE 0x0 // Clear state diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index b29cec3cf9..1b7db635b3 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -31,7 +31,6 @@ static int tsdbCommitMeta(STsdbRepo *pRepo); static void tsdbEndCommit(STsdbRepo *pRepo); static int tsdbHasDataToCommit(SCommitIter *iters, int nIters, TSKEY minKey, TSKEY maxKey); static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHelper *pHelper, SDataCols *pDataCols); -static void tsdbGetFidKeyRange(int daysPerFile, int8_t precision, int fileId, TSKEY *minKey, TSKEY *maxKey); static SCommitIter *tsdbCreateCommitIters(STsdbRepo *pRepo); static void tsdbDestroyCommitIters(SCommitIter *iters, int maxTables); @@ -544,7 +543,7 @@ static int tsdbHasDataToCommit(SCommitIter *iters, int nIters, TSKEY minKey, TSK return 0; } -static void tsdbGetFidKeyRange(int daysPerFile, int8_t precision, int fileId, TSKEY *minKey, TSKEY *maxKey) { +void tsdbGetFidKeyRange(int daysPerFile, int8_t precision, int fileId, TSKEY *minKey, TSKEY *maxKey) { *minKey = fileId * daysPerFile * tsMsPerDay[precision]; *maxKey = *minKey + daysPerFile * tsMsPerDay[precision] - 1; } diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 5b1afe3da8..37784577c4 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -128,8 +128,7 @@ typedef struct STsdbQueryHandle { static void changeQueryHandleForLastrowQuery(TsdbQueryHandleT pqHandle); static void changeQueryHandleForInterpQuery(TsdbQueryHandleT pHandle); -static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SCompBlock* pBlock, - SArray* sa); +static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SCompBlock* pBlock); static int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order); static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int maxRowsToRead, STimeWindow* win, STsdbQueryHandle* pQueryHandle); @@ -695,7 +694,7 @@ static void handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SCompBlock* } doLoadFileDataBlock(pQueryHandle, pBlock, pCheckInfo); - doMergeTwoLevelData(pQueryHandle, pCheckInfo, pBlock, pQueryHandle->defaultLoadColumn); + doMergeTwoLevelData(pQueryHandle, pCheckInfo, pBlock); } else { /* * no data in cache, only load data from file @@ -711,6 +710,7 @@ static void handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SCompBlock* cur->mixBlock = false; cur->blockCompleted = true; cur->lastKey = binfo.window.ekey + (ASCENDING_TRAVERSE(pQueryHandle->order)? 1:-1); + pCheckInfo->lastKey = cur->lastKey; } } @@ -734,7 +734,7 @@ static bool loadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock cur->pos = 0; } - doMergeTwoLevelData(pQueryHandle, pCheckInfo, pBlock, pQueryHandle->defaultLoadColumn); + doMergeTwoLevelData(pQueryHandle, pCheckInfo, pBlock); } else { // the whole block is loaded in to buffer handleDataMergeIfNeeded(pQueryHandle, pBlock, pCheckInfo); } @@ -751,7 +751,7 @@ static bool loadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock cur->pos = pBlock->numOfRows - 1; } - doMergeTwoLevelData(pQueryHandle, pCheckInfo, pBlock, pQueryHandle->defaultLoadColumn); + doMergeTwoLevelData(pQueryHandle, pCheckInfo, pBlock); } else { handleDataMergeIfNeeded(pQueryHandle, pBlock, pCheckInfo); } @@ -907,12 +907,12 @@ static int32_t copyDataFromFileBlock(STsdbQueryHandle* pQueryHandle, int32_t cap pQueryHandle->cur.win.ekey = tsArray[end]; pQueryHandle->cur.lastKey = tsArray[end] + step; - + return numOfRows + num; } static void copyOneRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity, int32_t numOfRows, SDataRow row, - STsdbMeta *pMeta, int32_t numOfCols, STable* pTable) { + int32_t numOfCols, STable* pTable) { char* pData = NULL; // the schema version info is embeded in SDataRow @@ -973,8 +973,7 @@ static void copyOneRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity, // only return the qualified data to client in terms of query time window, data rows in the same block but do not // be included in the query time window will be discarded -static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SCompBlock* pBlock, - SArray* sa) { +static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SCompBlock* pBlock) { SQueryFilePos* cur = &pQueryHandle->cur; SDataBlockInfo blockInfo = GET_FILE_DATA_BLOCK_INFO(pCheckInfo, pBlock); @@ -987,7 +986,6 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1:-1; int32_t numOfCols = taosArrayGetSize(pQueryHandle->pColumns); - STsdbMeta* pMeta = tsdbGetMeta(pQueryHandle->pTsdb); STable* pTable = pCheckInfo->pTableObj; int32_t endPos = cur->pos; @@ -1066,7 +1064,7 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* if ((key < tsArray[pos] && ASCENDING_TRAVERSE(pQueryHandle->order)) || (key > tsArray[pos] && !ASCENDING_TRAVERSE(pQueryHandle->order))) { - copyOneRowFromMem(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, row, pMeta, numOfCols, pTable); + copyOneRowFromMem(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, row, numOfCols, pTable); numOfRows += 1; if (cur->win.skey == TSKEY_INITIAL_VAL) { cur->win.skey = key; @@ -1406,8 +1404,21 @@ static int32_t getDataBlocksInFilesImpl(STsdbQueryHandle* pQueryHandle, bool* ex int32_t numOfBlocks = 0; int32_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); - + + STsdbCfg* pCfg = &pQueryHandle->pTsdb->config; + STimeWindow win = TSWINDOW_INITIALIZER; + while ((pQueryHandle->pFileGroup = tsdbGetFileGroupNext(&pQueryHandle->fileIter)) != NULL) { + tsdbGetFidKeyRange(pCfg->daysPerFile, pCfg->precision, pQueryHandle->pFileGroup->fileId, &win.skey, &win.ekey); + + // current file are not overlapped with query time window, ignore remain files + if ((ASCENDING_TRAVERSE(pQueryHandle->order) && win.skey > pQueryHandle->window.ekey) || + (!ASCENDING_TRAVERSE(pQueryHandle->order) && win.ekey < pQueryHandle->window.ekey)) { + tsdbDebug("%p remain files are not qualified for qrange:%"PRId64"-%"PRId64", ignore, %p", pQueryHandle, pQueryHandle->window.skey, pQueryHandle->window.ekey, pQueryHandle->qinfo) + pQueryHandle->pFileGroup = NULL; + break; + } + if ((code = getFileCompInfo(pQueryHandle, &numOfBlocks)) != TSDB_CODE_SUCCESS) { break; } @@ -1765,7 +1776,6 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int win->skey = TSKEY_INITIAL_VAL; int64_t st = taosGetTimestampUs(); - STsdbMeta* pMeta = tsdbGetMeta(pQueryHandle->pTsdb); STable* pTable = pCheckInfo->pTableObj; do { @@ -1787,7 +1797,7 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int } win->ekey = key; - copyOneRowFromMem(pQueryHandle, maxRowsToRead, numOfRows, row, pMeta, numOfCols, pTable); + copyOneRowFromMem(pQueryHandle, maxRowsToRead, numOfRows, row, numOfCols, pTable); if (++numOfRows >= maxRowsToRead) { moveToNextRowInMem(pCheckInfo); From 31824d9ffa0c05b52d0d1d21ddd806b9bc372267 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 20 Jul 2020 15:01:37 +0800 Subject: [PATCH 038/100] get rid of maxTables in TSDB --- src/tsdb/inc/tsdbMain.h | 13 ++- src/tsdb/src/tsdbMain.c | 2 +- src/tsdb/src/tsdbRWHelper.c | 186 +++++++++++++++++++++++------------- src/tsdb/src/tsdbRead.c | 6 +- 4 files changed, 131 insertions(+), 76 deletions(-) diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h index da5f03a4fb..9346230e02 100644 --- a/src/tsdb/inc/tsdbMain.h +++ b/src/tsdb/inc/tsdbMain.h @@ -199,6 +199,7 @@ typedef struct { // ------------------ tsdbRWHelper.c typedef struct { + int32_t tid; uint32_t len; uint32_t offset; uint32_t hasLast : 2; @@ -262,9 +263,14 @@ typedef struct { typedef struct { uint64_t uid; int32_t tid; - int32_t sversion; } SHelperTable; +typedef struct { + SCompIdx* pIdxArray; + int numOfIdx; + int curIdx; +} SIdxH; + typedef struct { tsdb_rw_helper_t type; @@ -272,7 +278,9 @@ typedef struct { int8_t state; // For file set usage SHelperFile files; - SCompIdx* pCompIdx; + SIdxH idxH; + SCompIdx curCompIdx; + void* pWIdx; // For table set usage SHelperTable tableInfo; SCompInfo* pCompInfo; @@ -284,7 +292,6 @@ typedef struct { void* compBuffer; // Buffer for temperary compress/decompress purpose } SRWHelper; - // Operations // ------------------ tsdbMeta.c #define TABLE_TYPE(t) (t)->type diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index c8b31af4f0..82dbe5f498 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -793,7 +793,7 @@ static int tsdbRestoreInfo(STsdbRepo *pRepo) { for (int i = 1; i < pRepo->config.maxTables; i++) { STable *pTable = pMeta->tables[i]; if (pTable == NULL) continue; - SCompIdx *pIdx = &rhelper.pCompIdx[i]; + SCompIdx *pIdx = &(rhelper.curCompIdx); if (pIdx->offset > 0 && pTable->lastKey < pIdx->maxKey) pTable->lastKey = pIdx->maxKey; } diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index 79b4a8ef64..19cb791ce8 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -60,6 +60,7 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter int *blkIdx); static int tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIter *pCommitIter, SDataCols *pTarget, TSKEY maxKey, int maxRows); +static int tsdbCompareTidIdx(const void *key1, const void *key2); // ---------------------- INTERNAL FUNCTIONS ---------------------- int tsdbInitReadHelper(SRWHelper *pHelper, STsdbRepo *pRepo) { @@ -251,17 +252,35 @@ void tsdbSetHelperTable(SRWHelper *pHelper, STable *pTable, STsdbRepo *pRepo) { pHelper->tableInfo.tid = pTable->tableId.tid; pHelper->tableInfo.uid = pTable->tableId.uid; STSchema *pSchema = tsdbGetTableSchemaImpl(pTable, false, false, -1); - pHelper->tableInfo.sversion = schemaVersion(pSchema); tdInitDataCols(pHelper->pDataCols[0], pSchema); tdInitDataCols(pHelper->pDataCols[1], pSchema); - SCompIdx *pIdx = pHelper->pCompIdx + pTable->tableId.tid; - if (pIdx->offset > 0) { - if (pIdx->uid != TABLE_UID(pTable)) { - memset((void *)pIdx, 0, sizeof(SCompIdx)); + if (helperType(pHelper) == TSDB_WRITE_HELPER) { + if (pHelper->idxH.numOfIdx > 0) { + if (pHelper->idxH.curIdx >= pHelper->idxH.numOfIdx) { + memset(&(pHelper->curCompIdx), 0, sizeof(SCompIdx)); + } else { + SCompIdx *pIdx = &(pHelper->idxH.pIdxArray[pHelper->idxH.curIdx]); + if (pIdx->tid == TABLE_TID(pTable)) { + pHelper->curCompIdx = *pIdx; + pHelper->idxH.curIdx++; + } else { + ASSERT(pIdx->tid > TABLE_TID(pTable)); + memset(&(pHelper->curCompIdx), 0, sizeof(SCompIdx)); + } + } } else { - if (pIdx->hasLast) pHelper->hasOldLastBlock = true; + memset(&(pHelper->curCompIdx), 0, sizeof(SCompIdx)); + } + } else { + // TODO: make it more efficient + void *ptr = bsearch(&TABLE_TID(pTable), (void *)pHelper->idxH.pIdxArray, pHelper->idxH.numOfIdx, sizeof(SCompIdx), + tsdbCompareTidIdx); + if (ptr == NULL) { + memset(&(pHelper->curCompIdx), 0, sizeof(SCompIdx)); + } else { + pHelper->curCompIdx = *(SCompIdx *)ptr; } } @@ -272,8 +291,8 @@ void tsdbSetHelperTable(SRWHelper *pHelper, STable *pTable, STsdbRepo *pRepo) { int tsdbCommitTableData(SRWHelper *pHelper, SCommitIter *pCommitIter, SDataCols *pDataCols, TSKEY maxKey) { ASSERT(helperType(pHelper) == TSDB_WRITE_HELPER); - SCompIdx * pIdx = &(pHelper->pCompIdx[TABLE_TID(pCommitIter->pTable)]); - int blkIdx = 0; + SCompIdx *pIdx = &(pHelper->curCompIdx); + int blkIdx = 0; ASSERT(pIdx->offset == 0 || pIdx->uid == TABLE_UID(pCommitIter->pTable)); if (tsdbLoadCompInfo(pHelper, NULL) < 0) return -1; @@ -298,7 +317,7 @@ int tsdbMoveLastBlockIfNeccessary(SRWHelper *pHelper) { STsdbCfg *pCfg = &pHelper->pRepo->config; ASSERT(helperType(pHelper) == TSDB_WRITE_HELPER); - SCompIdx * pIdx = pHelper->pCompIdx + pHelper->tableInfo.tid; + SCompIdx * pIdx = &(pHelper->curCompIdx); SCompBlock compBlock = {0}; if (TSDB_NLAST_FILE_OPENED(pHelper) && (pHelper->hasOldLastBlock)) { if (tsdbLoadCompInfo(pHelper, NULL) < 0) return -1; @@ -344,10 +363,11 @@ int tsdbMoveLastBlockIfNeccessary(SRWHelper *pHelper) { } int tsdbWriteCompInfo(SRWHelper *pHelper) { - off_t offset = 0; - SCompIdx *pIdx = pHelper->pCompIdx + pHelper->tableInfo.tid; - if (!helperHasState(pHelper, TSDB_HELPER_INFO_LOAD)) { - if (pIdx->offset > 0) { + SCompIdx *pIdx = &(pHelper->curCompIdx); + off_t offset = 0; + + if (pIdx->len > 0) { + if (!helperHasState(pHelper, TSDB_HELPER_INFO_LOAD)) { offset = lseek(helperNewHeadF(pHelper)->fd, 0, SEEK_END); if (offset < 0) { tsdbError("vgId:%d failed to lseed file %s since %s", REPO_ID(pHelper->pRepo), helperNewHeadF(pHelper)->fname, @@ -357,6 +377,8 @@ int tsdbWriteCompInfo(SRWHelper *pHelper) { } pIdx->offset = offset; + pIdx->uid = pHelper->tableInfo.uid; + pIdx->tid = pHelper->tableInfo.tid; ASSERT(pIdx->offset >= TSDB_FILE_HEAD_SIZE); if (tsendfile(helperNewHeadF(pHelper)->fd, helperHeadF(pHelper)->fd, NULL, pIdx->len) < pIdx->len) { @@ -365,9 +387,7 @@ int tsdbWriteCompInfo(SRWHelper *pHelper) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } - } - } else { - if (pIdx->len > 0) { + } else { pHelper->pCompInfo->delimiter = TSDB_FILE_DELIMITER; pHelper->pCompInfo->uid = pHelper->tableInfo.uid; pHelper->pCompInfo->checksum = 0; @@ -383,6 +403,7 @@ int tsdbWriteCompInfo(SRWHelper *pHelper) { } pIdx->offset = offset; pIdx->uid = pHelper->tableInfo.uid; + pIdx->tid = pHelper->tableInfo.tid; ASSERT(pIdx->offset >= TSDB_FILE_HEAD_SIZE); if (twrite(helperNewHeadF(pHelper)->fd, (void *)(pHelper->pCompInfo), pIdx->len) < pIdx->len) { @@ -392,6 +413,17 @@ int tsdbWriteCompInfo(SRWHelper *pHelper) { return -1; } } + + if (tsizeof(pHelper->pWIdx) < helperNewIdxF(pHelper)->info.len + sizeof(SCompIdx) + 12) { + pHelper->pWIdx = trealloc(pHelper->pWIdx, tsizeof(pHelper->pWIdx) == 0 ? 1024 : tsizeof(pHelper->pWIdx) * 2); + if (pHelper->pWIdx == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return -1; + } + } + + void *pBuf = POINTER_SHIFT(pHelper->pWIdx, helperNewIdxF(pHelper)->info.len); + helperNewIdxF(pHelper)->info.len += tsdbEncodeSCompIdx(&pBuf, &(pHelper->curCompIdx)); } return 0; @@ -399,57 +431,43 @@ int tsdbWriteCompInfo(SRWHelper *pHelper) { int tsdbWriteCompIdx(SRWHelper *pHelper) { ASSERT(helperType(pHelper) == TSDB_WRITE_HELPER); - STsdbCfg *pCfg = &pHelper->pRepo->config; + // STsdbCfg *pCfg = &pHelper->pRepo->config; SFile *pFile = helperNewIdxF(pHelper); - void *buf = pHelper->pBuffer; - for (uint32_t i = 0; i < pCfg->maxTables; i++) { - SCompIdx *pCompIdx = pHelper->pCompIdx + i; - if (pCompIdx->offset > 0) { - int drift = POINTER_DISTANCE(buf, pHelper->pBuffer); - if (tsizeof(pHelper->pBuffer) - drift < 128) { - pHelper->pBuffer = trealloc(pHelper->pBuffer, tsizeof(pHelper->pBuffer) * 2); - if (pHelper->pBuffer == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - } - buf = POINTER_SHIFT(pHelper->pBuffer, drift); - taosEncodeVariantU32(&buf, i); - tsdbEncodeSCompIdx(&buf, pCompIdx); + pFile->info.len += sizeof(TSCKSUM); + if (tsizeof(pHelper->pWIdx) < pFile->info.len) { + pHelper->pWIdx = trealloc(pHelper->pWIdx, pFile->info.len); + if (pHelper->pWIdx == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return -1; } } + taosCalcChecksumAppend(0, (uint8_t *)pHelper->pWIdx, pFile->info.len); - int tsize = (char *)buf - (char *)pHelper->pBuffer + sizeof(TSCKSUM); - taosCalcChecksumAppend(0, (uint8_t *)pHelper->pBuffer, tsize); - - if (twrite(pFile->fd, (void *)pHelper->pBuffer, tsize) < tsize) { - tsdbError("vgId:%d failed to write %d bytes to file %s since %s", REPO_ID(pHelper->pRepo), tsize, pFile->fname, - strerror(errno)); + if (twrite(pFile->fd, (void *)pHelper->pWIdx, pFile->info.len) < pFile->info.len) { + tsdbError("vgId:%d failed to write %d bytes to file %s since %s", REPO_ID(pHelper->pRepo), pFile->info.len, + pFile->fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); return -1; } - pFile->info.len = tsize; + return 0; } int tsdbLoadCompIdx(SRWHelper *pHelper, void *target) { - STsdbCfg *pCfg = &(pHelper->pRepo->config); - ASSERT(pHelper->state == TSDB_HELPER_FILE_SET_AND_OPEN); + SFile *pFile = helperIdxF(pHelper); + int fd = pFile->fd; if (!helperHasState(pHelper, TSDB_HELPER_IDX_LOAD)) { // If not load from file, just load it in object - SFile *pFile = helperIdxF(pHelper); - int fd = pFile->fd; - - memset(pHelper->pCompIdx, 0, tsizeof(pHelper->pCompIdx)); if (pFile->info.len > 0) { if ((pHelper->pBuffer = trealloc(pHelper->pBuffer, pFile->info.len)) == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; return -1; } + if (lseek(fd, TSDB_FILE_HEAD_SIZE, SEEK_SET) < 0) { tsdbError("vgId:%d failed to lseek file %s since %s", REPO_ID(pHelper->pRepo), pFile->fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); @@ -462,6 +480,7 @@ int tsdbLoadCompIdx(SRWHelper *pHelper, void *target) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } + if (!taosCheckChecksumWhole((uint8_t *)(pHelper->pBuffer), pFile->info.len)) { tsdbError("vgId:%d file %s SCompIdx part is corrupted. len %u", REPO_ID(pHelper->pRepo), pFile->fname, pFile->info.len); @@ -470,27 +489,49 @@ int tsdbLoadCompIdx(SRWHelper *pHelper, void *target) { } // Decode it + pHelper->idxH.numOfIdx = 0; void *ptr = pHelper->pBuffer; while (POINTER_DISTANCE(ptr, pHelper->pBuffer) < (pFile->info.len - sizeof(TSCKSUM))) { - uint32_t tid = 0; - if ((ptr = taosDecodeVariantU32(ptr, &tid)) == NULL) return -1; - ASSERT(tid > 0 && tid < pCfg->maxTables); + size_t tlen = tsizeof(pHelper->idxH.pIdxArray); + pHelper->idxH.numOfIdx++; - if ((ptr = tsdbDecodeSCompIdx(ptr, pHelper->pCompIdx + tid)) == NULL) return -1; + if (tlen < pHelper->idxH.numOfIdx) { + pHelper->idxH.pIdxArray = (SCompIdx *)trealloc(pHelper->idxH.pIdxArray, (tlen == 0) ? 1024 : tlen * 2); + if (pHelper->idxH.pIdxArray == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return -1; + } + } + + ptr = tsdbDecodeSCompIdx(ptr, &(pHelper->idxH.pIdxArray[pHelper->idxH.numOfIdx - 1])); + if (ptr == NULL) { + tsdbError("vgId:%d file %s SCompIdx part is corrupted. len %u", REPO_ID(pHelper->pRepo), pFile->fname, + pFile->info.len); + terrno = TSDB_CODE_TDB_FILE_CORRUPTED; + return -1; + } + + ASSERT(pHelper->idxH.numOfIdx == 1 || pHelper->idxH.pIdxArray[pHelper->idxH.numOfIdx - 1].tid > + pHelper->idxH.pIdxArray[pHelper->idxH.numOfIdx - 2].tid); ASSERT(POINTER_DISTANCE(ptr, pHelper->pBuffer) <= pFile->info.len - sizeof(TSCKSUM)); } - if (lseek(fd, TSDB_FILE_HEAD_SIZE, SEEK_SET) < 0) { - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } + // if (lseek(fd, TSDB_FILE_HEAD_SIZE, SEEK_SET) < 0) { + // terrno = TAOS_SYSTEM_ERROR(errno); + // return -1; + // } } } helperSetState(pHelper, TSDB_HELPER_IDX_LOAD); + if (helperType(pHelper) == TSDB_WRITE_HELPER) { + pFile->info.len = 0; + } + // Copy the memory for outside usage - if (target) memcpy(target, pHelper->pCompIdx, tsizeof(pHelper->pCompIdx)); + if (target && pHelper->idxH.numOfIdx > 0) + memcpy(target, pHelper->idxH.pIdxArray, sizeof(SCompIdx) * pHelper->idxH.numOfIdx); return 0; } @@ -498,7 +539,7 @@ int tsdbLoadCompIdx(SRWHelper *pHelper, void *target) { int tsdbLoadCompInfo(SRWHelper *pHelper, void *target) { ASSERT(helperHasState(pHelper, TSDB_HELPER_TABLE_SET)); - SCompIdx *pIdx = pHelper->pCompIdx + pHelper->tableInfo.tid; + SCompIdx *pIdx = &(pHelper->curCompIdx); int fd = helperHeadF(pHelper)->fd; @@ -820,7 +861,7 @@ static int tsdbAdjustInfoSizeIfNeeded(SRWHelper *pHelper, size_t esize) { } static int tsdbInsertSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx) { - SCompIdx *pIdx = pHelper->pCompIdx + pHelper->tableInfo.tid; + SCompIdx *pIdx = &(pHelper->curCompIdx); ASSERT(blkIdx >= 0 && blkIdx <= pIdx->numOfBlocks); ASSERT(pCompBlock->numOfSubBlocks == 1); @@ -867,7 +908,7 @@ _err: static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx, int rowsAdded) { ASSERT(pCompBlock->numOfSubBlocks == 0); - SCompIdx *pIdx = pHelper->pCompIdx + pHelper->tableInfo.tid; + SCompIdx *pIdx = &(pHelper->curCompIdx); ASSERT(blkIdx >= 0 && blkIdx < pIdx->numOfBlocks); SCompBlock *pSCompBlock = pHelper->pCompInfo->blocks + blkIdx; @@ -951,7 +992,7 @@ _err: static int tsdbUpdateSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx) { ASSERT(pCompBlock->numOfSubBlocks == 1); - SCompIdx *pIdx = pHelper->pCompIdx + pHelper->tableInfo.tid; + SCompIdx *pIdx = &(pHelper->curCompIdx); ASSERT(blkIdx >= 0 && blkIdx < pIdx->numOfBlocks); @@ -987,6 +1028,8 @@ static int tsdbUpdateSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int } static void tsdbResetHelperFileImpl(SRWHelper *pHelper) { + pHelper->idxH.numOfIdx = 0; + pHelper->idxH.curIdx = 0; memset((void *)&pHelper->files, 0, sizeof(pHelper->files)); helperIdxF(pHelper)->fd = -1; helperHeadF(pHelper)->fd = -1; @@ -998,14 +1041,6 @@ static void tsdbResetHelperFileImpl(SRWHelper *pHelper) { } static int tsdbInitHelperFile(SRWHelper *pHelper) { - STsdbCfg *pCfg = &pHelper->pRepo->config; - size_t tsize = sizeof(SCompIdx) * pCfg->maxTables + sizeof(TSCKSUM); - pHelper->pCompIdx = (SCompIdx *)tmalloc(tsize); - if (pHelper->pCompIdx == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - tsdbResetHelperFileImpl(pHelper); return 0; } @@ -1013,7 +1048,8 @@ static int tsdbInitHelperFile(SRWHelper *pHelper) { static void tsdbDestroyHelperFile(SRWHelper *pHelper) { tsdbCloseHelperFile(pHelper, false); tsdbResetHelperFileImpl(pHelper); - tzfree(pHelper->pCompIdx); + tzfree(pHelper->idxH.pIdxArray); + tzfree(pHelper->pWIdx); } // ---------- Operations on Helper Table part @@ -1331,6 +1367,7 @@ _err: static int tsdbEncodeSCompIdx(void **buf, SCompIdx *pIdx) { int tlen = 0; + tlen += taosEncodeVariantI32(buf, pIdx->tid); tlen += taosEncodeVariantU32(buf, pIdx->len); tlen += taosEncodeVariantU32(buf, pIdx->offset); tlen += taosEncodeFixedU8(buf, pIdx->hasLast); @@ -1346,6 +1383,7 @@ static void *tsdbDecodeSCompIdx(void *buf, SCompIdx *pIdx) { uint32_t numOfBlocks = 0; uint64_t value = 0; + if ((buf = taosDecodeVariantI32(buf, &(pIdx->tid))) == NULL) return NULL; if ((buf = taosDecodeVariantU32(buf, &(pIdx->len))) == NULL) return NULL; if ((buf = taosDecodeVariantU32(buf, &(pIdx->offset))) == NULL) return NULL; if ((buf = taosDecodeFixedU8(buf, &(hasLast))) == NULL) return NULL; @@ -1363,7 +1401,7 @@ static void *tsdbDecodeSCompIdx(void *buf, SCompIdx *pIdx) { static int tsdbProcessAppendCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, SDataCols *pDataCols, TSKEY maxKey) { STsdbCfg * pCfg = &(pHelper->pRepo->config); STable * pTable = pCommitIter->pTable; - SCompIdx * pIdx = pHelper->pCompIdx + TABLE_TID(pTable); + SCompIdx * pIdx = &(pHelper->curCompIdx); TSKEY keyFirst = tsdbNextIterKey(pCommitIter->pIter); int defaultRowsInBlock = pCfg->maxRowsPerFileBlock * 4 / 5; SCompBlock compBlock = {0}; @@ -1410,7 +1448,7 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, int *blkIdx) { STsdbCfg * pCfg = &(pHelper->pRepo->config); STable * pTable = pCommitIter->pTable; - SCompIdx * pIdx = pHelper->pCompIdx + TABLE_TID(pTable); + SCompIdx * pIdx = &(pHelper->curCompIdx); SCompBlock compBlock = {0}; TSKEY keyFirst = tsdbNextIterKey(pCommitIter->pIter); int defaultRowsInBlock = pCfg->maxRowsPerFileBlock * 4 / 5; @@ -1605,4 +1643,14 @@ static int tsdbWriteBlockToProperFile(SRWHelper *pHelper, SDataCols *pDataCols, if (tsdbWriteBlockToFile(pHelper, pFile, pDataCols, pCompBlock, isLast, true) < 0) return -1; return 0; +} + +static int tsdbCompareTidIdx(const void *key1, const void *key2) { + if (*(int32_t *)key1 > ((SCompIdx *)key2)->tid) { + return 1; + } else if (*(int32_t *)key1 < ((SCompIdx *)key2)->tid) { + return -1; + } else { + return 0; + } } \ No newline at end of file diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 5b1afe3da8..0d76c7f7bf 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -555,7 +555,9 @@ static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlo STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i); pCheckInfo->numOfBlocks = 0; - SCompIdx* compIndex = &pQueryHandle->rhelper.pCompIdx[pCheckInfo->tableId.tid]; + tsdbSetHelperTable(&pQueryHandle->rhelper, pCheckInfo->pTableObj, pQueryHandle->pTsdb); + + SCompIdx* compIndex = &pQueryHandle->rhelper.curCompIdx; // no data block in this file, try next file if (compIndex->len == 0 || compIndex->numOfBlocks == 0 || compIndex->uid != pCheckInfo->tableId.uid) { @@ -572,8 +574,6 @@ static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlo pCheckInfo->compSize = compIndex->len; } - tsdbSetHelperTable(&pQueryHandle->rhelper, pCheckInfo->pTableObj, pQueryHandle->pTsdb); - tsdbLoadCompInfo(&(pQueryHandle->rhelper), (void *)(pCheckInfo->pCompInfo)); SCompInfo* pCompInfo = pCheckInfo->pCompInfo; From 9bb9d7a2a857eac311907bf81f23d1d8d5bbc8d5 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 20 Jul 2020 15:19:15 +0800 Subject: [PATCH 039/100] use malloc for more efficent --- src/common/src/tdataformat.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/src/tdataformat.c b/src/common/src/tdataformat.c index 719d80aa77..12ea4ad78d 100644 --- a/src/common/src/tdataformat.c +++ b/src/common/src/tdataformat.c @@ -318,7 +318,7 @@ SDataCols *tdNewDataCols(int maxRowSize, int maxCols, int maxRows) { pCols->maxPoints = maxRows; pCols->bufSize = maxRowSize * maxRows; - pCols->buf = calloc(1, pCols->bufSize); + pCols->buf = malloc(pCols->bufSize); if (pCols->buf == NULL) { free(pCols); return NULL; From e57e4a6d654d968b5588b620d3a867eaa4782f2a Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 20 Jul 2020 16:06:00 +0800 Subject: [PATCH 040/100] add gcov to support coverage test. --- src/query/tests/CMakeLists.txt | 4 ++-- src/util/tests/CMakeLists.txt | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/query/tests/CMakeLists.txt b/src/query/tests/CMakeLists.txt index 0ae8600756..1856223391 100644 --- a/src/query/tests/CMakeLists.txt +++ b/src/query/tests/CMakeLists.txt @@ -11,5 +11,5 @@ IF (HEADER_GTEST_INCLUDE_DIR AND LIB_GTEST_STATIC_DIR) AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) ADD_EXECUTABLE(queryTest ${SOURCE_LIST}) - TARGET_LINK_LIBRARIES(queryTest taos query gtest pthread) -ENDIF() \ No newline at end of file + TARGET_LINK_LIBRARIES(queryTest taos query gtest pthread gcov) +ENDIF() diff --git a/src/util/tests/CMakeLists.txt b/src/util/tests/CMakeLists.txt index 9f66eba37a..b0b5d3013b 100644 --- a/src/util/tests/CMakeLists.txt +++ b/src/util/tests/CMakeLists.txt @@ -11,5 +11,5 @@ IF (HEADER_GTEST_INCLUDE_DIR AND LIB_GTEST_STATIC_DIR) AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) ADD_EXECUTABLE(utilTest ${SOURCE_LIST}) - TARGET_LINK_LIBRARIES(utilTest tutil common gtest pthread) -ENDIF() \ No newline at end of file + TARGET_LINK_LIBRARIES(utilTest tutil common gtest pthread gcov) +ENDIF() From dd9f62cfe3fd52a70f27e5852b1a9cc07e1474e1 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 20 Jul 2020 16:14:55 +0800 Subject: [PATCH 041/100] remove idx file back --- src/tsdb/inc/tsdbMain.h | 11 +++++- src/tsdb/src/tsdbFile.c | 26 ++++++++------ src/tsdb/src/tsdbMemTable.c | 2 ++ src/tsdb/src/tsdbRWHelper.c | 71 ++++++++++++++++++++++++++----------- 4 files changed, 78 insertions(+), 32 deletions(-) diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h index 9346230e02..8ab89d72ed 100644 --- a/src/tsdb/inc/tsdbMain.h +++ b/src/tsdb/inc/tsdbMain.h @@ -132,12 +132,18 @@ typedef struct { // ------------------ tsdbFile.c extern const char* tsdbFileSuffix[]; typedef enum { +#ifdef TSDB_IDX TSDB_FILE_TYPE_IDX = 0, TSDB_FILE_TYPE_HEAD, +#else + TSDB_FILE_TYPE_HEAD = 0, +#endif TSDB_FILE_TYPE_DATA, TSDB_FILE_TYPE_LAST, TSDB_FILE_TYPE_MAX, +#ifdef TSDB_IDX TSDB_FILE_TYPE_NIDX, +#endif TSDB_FILE_TYPE_NHEAD, TSDB_FILE_TYPE_NLAST } TSDB_FILE_TYPE; @@ -147,6 +153,7 @@ typedef struct { uint32_t len; uint32_t totalBlocks; uint32_t totalSubBlocks; + uint32_t offset; uint64_t size; // total size of the file uint64_t tombSize; // unused file size } STsdbFileInfo; @@ -450,11 +457,13 @@ void tsdbRemoveFileGroup(STsdbRepo* pRepo, SFileGroup* pFGroup); #define helperState(h) (h)->state #define TSDB_NLAST_FILE_OPENED(h) ((h)->files.nLastF.fd > 0) #define helperFileId(h) ((h)->files.fGroup.fileId) +#ifdef TSDB_IDX #define helperIdxF(h) (&((h)->files.fGroup.files[TSDB_FILE_TYPE_IDX])) +#define helperNewIdxF(h) (&((h)->files.nIdxF)) +#endif #define helperHeadF(h) (&((h)->files.fGroup.files[TSDB_FILE_TYPE_HEAD])) #define helperDataF(h) (&((h)->files.fGroup.files[TSDB_FILE_TYPE_DATA])) #define helperLastF(h) (&((h)->files.fGroup.files[TSDB_FILE_TYPE_LAST])) -#define helperNewIdxF(h) (&((h)->files.nIdxF)) #define helperNewHeadF(h) (&((h)->files.nHeadF)) #define helperNewLastF(h) (&((h)->files.nLastF)) diff --git a/src/tsdb/src/tsdbFile.c b/src/tsdb/src/tsdbFile.c index 7b7788c4d9..0a18d15773 100644 --- a/src/tsdb/src/tsdbFile.c +++ b/src/tsdb/src/tsdbFile.c @@ -30,7 +30,11 @@ #include "ttime.h" #include "tfile.h" +#ifdef TSDB_IDX const char *tsdbFileSuffix[] = {".idx", ".head", ".data", ".last", "", ".i", ".h", ".l"}; +#else +const char *tsdbFileSuffix[] = {".head", ".data", ".last", "", ".h", ".l"}; +#endif static int tsdbInitFile(SFile *pFile, STsdbRepo *pRepo, int fid, int type); static void tsdbDestroyFile(SFile *pFile); @@ -108,7 +112,7 @@ int tsdbOpenFileH(STsdbRepo *pRepo) { memset((void *)(&fileGroup), 0, sizeof(SFileGroup)); fileGroup.fileId = fid; - for (int type = TSDB_FILE_TYPE_IDX; type < TSDB_FILE_TYPE_MAX; type++) { + for (int type = 0; type < TSDB_FILE_TYPE_MAX; type++) { if (tsdbInitFile(&fileGroup.files[type], pRepo, fid, type) < 0) { tsdbError("vgId:%d failed to init file fid %d type %d", REPO_ID(pRepo), fid, type); goto _err; @@ -126,7 +130,7 @@ int tsdbOpenFileH(STsdbRepo *pRepo) { return 0; _err: - for (int type = TSDB_FILE_TYPE_IDX; type < TSDB_FILE_TYPE_MAX; type++) tsdbDestroyFile(&fileGroup.files[type]); + for (int type = 0; type < TSDB_FILE_TYPE_MAX; type++) tsdbDestroyFile(&fileGroup.files[type]); tfree(tDataDir); if (dir != NULL) closedir(dir); @@ -139,7 +143,7 @@ void tsdbCloseFileH(STsdbRepo *pRepo) { for (int i = 0; i < pFileH->nFGroups; i++) { SFileGroup *pFGroup = pFileH->pFGroup + i; - for (int type = TSDB_FILE_TYPE_IDX; type < TSDB_FILE_TYPE_MAX; type++) { + for (int type = 0; type < TSDB_FILE_TYPE_MAX; type++) { tsdbDestroyFile(&pFGroup->files[type]); } } @@ -156,7 +160,7 @@ SFileGroup *tsdbCreateFGroupIfNeed(STsdbRepo *pRepo, char *dataDir, int fid, int SFileGroup *pGroup = tsdbSearchFGroup(pFileH, fid, TD_EQ); if (pGroup == NULL) { // if not exists, create one pFGroup->fileId = fid; - for (int type = TSDB_FILE_TYPE_IDX; type < TSDB_FILE_TYPE_MAX; type++) { + for (int type = 0; type < TSDB_FILE_TYPE_MAX; type++) { if (tsdbCreateFile(&pFGroup->files[type], pRepo, fid, type) < 0) goto _err; } @@ -169,7 +173,7 @@ SFileGroup *tsdbCreateFGroupIfNeed(STsdbRepo *pRepo, char *dataDir, int fid, int return pGroup; _err: - for (int type = TSDB_FILE_TYPE_IDX; type < TSDB_FILE_TYPE_MAX; type++) tsdbDestroyFile(&pGroup->files[type]); + for (int type = 0; type < TSDB_FILE_TYPE_MAX; type++) tsdbDestroyFile(&pGroup->files[type]); return NULL; } @@ -325,10 +329,11 @@ int tsdbEncodeSFileInfo(void **buf, const STsdbFileInfo *pInfo) { int tlen = 0; tlen += taosEncodeFixedU32(buf, pInfo->magic); tlen += taosEncodeFixedU32(buf, pInfo->len); - tlen += taosEncodeFixedU64(buf, pInfo->size); - tlen += taosEncodeFixedU64(buf, pInfo->tombSize); tlen += taosEncodeFixedU32(buf, pInfo->totalBlocks); tlen += taosEncodeFixedU32(buf, pInfo->totalSubBlocks); + tlen += taosEncodeFixedU32(buf, pInfo->offset); + tlen += taosEncodeFixedU64(buf, pInfo->size); + tlen += taosEncodeFixedU64(buf, pInfo->tombSize); return tlen; } @@ -336,10 +341,11 @@ int tsdbEncodeSFileInfo(void **buf, const STsdbFileInfo *pInfo) { void *tsdbDecodeSFileInfo(void *buf, STsdbFileInfo *pInfo) { buf = taosDecodeFixedU32(buf, &(pInfo->magic)); buf = taosDecodeFixedU32(buf, &(pInfo->len)); - buf = taosDecodeFixedU64(buf, &(pInfo->size)); - buf = taosDecodeFixedU64(buf, &(pInfo->tombSize)); buf = taosDecodeFixedU32(buf, &(pInfo->totalBlocks)); buf = taosDecodeFixedU32(buf, &(pInfo->totalSubBlocks)); + buf = taosDecodeFixedU32(buf, &(pInfo->offset)); + buf = taosDecodeFixedU64(buf, &(pInfo->size)); + buf = taosDecodeFixedU64(buf, &(pInfo->tombSize)); return buf; } @@ -358,7 +364,7 @@ void tsdbRemoveFileGroup(STsdbRepo *pRepo, SFileGroup *pFGroup) { pFileH->nFGroups--; ASSERT(pFileH->nFGroups >= 0); - for (int type = TSDB_FILE_TYPE_IDX; type < TSDB_FILE_TYPE_MAX; type++) { + for (int type = 0; type < TSDB_FILE_TYPE_MAX; type++) { if (remove(fileGroup.files[type].fname) < 0) { tsdbError("vgId:%d failed to remove file %s", REPO_ID(pRepo), fileGroup.files[type].fname); } diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index 69e6ab3c0f..03fa9541c9 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -628,7 +628,9 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHe tsdbCloseHelperFile(pHelper, 0); pthread_rwlock_wrlock(&(pFileH->fhlock)); +#ifdef TSDB_IDX pGroup->files[TSDB_FILE_TYPE_IDX] = *(helperIdxF(pHelper)); +#endif pGroup->files[TSDB_FILE_TYPE_HEAD] = *(helperHeadF(pHelper)); pGroup->files[TSDB_FILE_TYPE_DATA] = *(helperDataF(pHelper)); pGroup->files[TSDB_FILE_TYPE_LAST] = *(helperLastF(pHelper)); diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index 19cb791ce8..22182694de 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -109,21 +109,27 @@ int tsdbSetAndOpenHelperFile(SRWHelper *pHelper, SFileGroup *pGroup) { // Set the files pHelper->files.fGroup = *pGroup; if (helperType(pHelper) == TSDB_WRITE_HELPER) { +#ifdef TSDB_IDX tsdbGetDataFileName(pHelper->pRepo, pGroup->fileId, TSDB_FILE_TYPE_NIDX, helperNewIdxF(pHelper)->fname); +#endif tsdbGetDataFileName(pHelper->pRepo, pGroup->fileId, TSDB_FILE_TYPE_NHEAD, helperNewHeadF(pHelper)->fname); tsdbGetDataFileName(pHelper->pRepo, pGroup->fileId, TSDB_FILE_TYPE_NLAST, helperNewLastF(pHelper)->fname); } // Open the files +#ifdef TSDB_IDX if (tsdbOpenFile(helperIdxF(pHelper), O_RDONLY) < 0) goto _err; +#endif if (tsdbOpenFile(helperHeadF(pHelper), O_RDONLY) < 0) goto _err; if (helperType(pHelper) == TSDB_WRITE_HELPER) { if (tsdbOpenFile(helperDataF(pHelper), O_RDWR) < 0) goto _err; if (tsdbOpenFile(helperLastF(pHelper), O_RDWR) < 0) goto _err; +#ifdef TSDB_IDX // Create and open .i file if (tsdbOpenFile(helperNewIdxF(pHelper), O_WRONLY | O_CREAT) < 0) return -1; if (tsdbUpdateFileHeader(helperNewIdxF(pHelper), 0) < 0) return -1; +#endif // Create and open .h if (tsdbOpenFile(helperNewHeadF(pHelper), O_WRONLY | O_CREAT) < 0) return -1; @@ -150,11 +156,13 @@ _err: int tsdbCloseHelperFile(SRWHelper *pHelper, bool hasError) { SFile *pFile = NULL; +#ifdef TSDB_IDX pFile = helperIdxF(pHelper); if (pFile->fd > 0) { close(pFile->fd); pFile->fd = -1; } +#endif pFile = helperHeadF(pHelper); if (pFile->fd > 0) { @@ -182,6 +190,7 @@ int tsdbCloseHelperFile(SRWHelper *pHelper, bool hasError) { } if (helperType(pHelper) == TSDB_WRITE_HELPER) { +#ifdef TSDB_IDX pFile = helperNewIdxF(pHelper); if (pFile->fd > 0) { if (!hasError) tsdbUpdateFileHeader(pFile, 0); @@ -200,6 +209,7 @@ int tsdbCloseHelperFile(SRWHelper *pHelper, bool hasError) { helperIdxF(pHelper)->info = pFile->info; } } +#endif pFile = helperNewHeadF(pHelper); if (pFile->fd > 0) { @@ -365,12 +375,13 @@ int tsdbMoveLastBlockIfNeccessary(SRWHelper *pHelper) { int tsdbWriteCompInfo(SRWHelper *pHelper) { SCompIdx *pIdx = &(pHelper->curCompIdx); off_t offset = 0; + SFile * pFile = helperNewHeadF(pHelper); if (pIdx->len > 0) { if (!helperHasState(pHelper, TSDB_HELPER_INFO_LOAD)) { - offset = lseek(helperNewHeadF(pHelper)->fd, 0, SEEK_END); + offset = lseek(pFile->fd, 0, SEEK_END); if (offset < 0) { - tsdbError("vgId:%d failed to lseed file %s since %s", REPO_ID(pHelper->pRepo), helperNewHeadF(pHelper)->fname, + tsdbError("vgId:%d failed to lseed file %s since %s", REPO_ID(pHelper->pRepo), pFile->fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -381,9 +392,9 @@ int tsdbWriteCompInfo(SRWHelper *pHelper) { pIdx->tid = pHelper->tableInfo.tid; ASSERT(pIdx->offset >= TSDB_FILE_HEAD_SIZE); - if (tsendfile(helperNewHeadF(pHelper)->fd, helperHeadF(pHelper)->fd, NULL, pIdx->len) < pIdx->len) { + if (tsendfile(pFile->fd, helperHeadF(pHelper)->fd, NULL, pIdx->len) < pIdx->len) { tsdbError("vgId:%d failed to send %d bytes from file %s to %s since %s", REPO_ID(pHelper->pRepo), pIdx->len, - helperHeadF(pHelper)->fname, helperNewHeadF(pHelper)->fname, strerror(errno)); + helperHeadF(pHelper)->fname, pFile->fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -394,9 +405,9 @@ int tsdbWriteCompInfo(SRWHelper *pHelper) { ASSERT(pIdx->len > sizeof(SCompInfo) + sizeof(TSCKSUM) && (pIdx->len - sizeof(SCompInfo) - sizeof(TSCKSUM)) % sizeof(SCompBlock) == 0); taosCalcChecksumAppend(0, (uint8_t *)pHelper->pCompInfo, pIdx->len); - offset = lseek(helperNewHeadF(pHelper)->fd, 0, SEEK_END); + offset = lseek(pFile->fd, 0, SEEK_END); if (offset < 0) { - tsdbError("vgId:%d failed to lseek file %s since %s", REPO_ID(pHelper->pRepo), helperNewHeadF(pHelper)->fname, + tsdbError("vgId:%d failed to lseek file %s since %s", REPO_ID(pHelper->pRepo), pFile->fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -406,15 +417,19 @@ int tsdbWriteCompInfo(SRWHelper *pHelper) { pIdx->tid = pHelper->tableInfo.tid; ASSERT(pIdx->offset >= TSDB_FILE_HEAD_SIZE); - if (twrite(helperNewHeadF(pHelper)->fd, (void *)(pHelper->pCompInfo), pIdx->len) < pIdx->len) { + if (twrite(pFile->fd, (void *)(pHelper->pCompInfo), pIdx->len) < pIdx->len) { tsdbError("vgId:%d failed to write %d bytes to file %s since %s", REPO_ID(pHelper->pRepo), pIdx->len, - helperNewHeadF(pHelper)->fname, strerror(errno)); + pFile->fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); return -1; } } - if (tsizeof(pHelper->pWIdx) < helperNewIdxF(pHelper)->info.len + sizeof(SCompIdx) + 12) { +#ifdef TSDB_IDX + pFile = helperNewIdxF(pHelper); +#endif + + if (tsizeof(pHelper->pWIdx) < pFile->info.len + sizeof(SCompIdx) + 12) { pHelper->pWIdx = trealloc(pHelper->pWIdx, tsizeof(pHelper->pWIdx) == 0 ? 1024 : tsizeof(pHelper->pWIdx) * 2); if (pHelper->pWIdx == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; @@ -422,8 +437,8 @@ int tsdbWriteCompInfo(SRWHelper *pHelper) { } } - void *pBuf = POINTER_SHIFT(pHelper->pWIdx, helperNewIdxF(pHelper)->info.len); - helperNewIdxF(pHelper)->info.len += tsdbEncodeSCompIdx(&pBuf, &(pHelper->curCompIdx)); + void *pBuf = POINTER_SHIFT(pHelper->pWIdx, pFile->info.len); + pFile->info.len += tsdbEncodeSCompIdx(&pBuf, &(pHelper->curCompIdx)); } return 0; @@ -431,9 +446,13 @@ int tsdbWriteCompInfo(SRWHelper *pHelper) { int tsdbWriteCompIdx(SRWHelper *pHelper) { ASSERT(helperType(pHelper) == TSDB_WRITE_HELPER); - // STsdbCfg *pCfg = &pHelper->pRepo->config; + off_t offset = 0; +#ifdef TSDB_IDX SFile *pFile = helperNewIdxF(pHelper); +#else + SFile *pFile = helperNewHeadF(pHelper); +#endif pFile->info.len += sizeof(TSCKSUM); if (tsizeof(pHelper->pWIdx) < pFile->info.len) { @@ -445,6 +464,15 @@ int tsdbWriteCompIdx(SRWHelper *pHelper) { } taosCalcChecksumAppend(0, (uint8_t *)pHelper->pWIdx, pFile->info.len); + offset = lseek(pFile->fd, 0, SEEK_END); + if (offset < 0) { + tsdbError("vgId:%d failed to lseek file %s since %s", REPO_ID(pHelper->pRepo), pFile->fname, strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + pFile->info.offset = offset; + if (twrite(pFile->fd, (void *)pHelper->pWIdx, pFile->info.len) < pFile->info.len) { tsdbError("vgId:%d failed to write %d bytes to file %s since %s", REPO_ID(pHelper->pRepo), pFile->info.len, pFile->fname, strerror(errno)); @@ -457,8 +485,12 @@ int tsdbWriteCompIdx(SRWHelper *pHelper) { int tsdbLoadCompIdx(SRWHelper *pHelper, void *target) { ASSERT(pHelper->state == TSDB_HELPER_FILE_SET_AND_OPEN); +#ifdef TSDB_IDX SFile *pFile = helperIdxF(pHelper); - int fd = pFile->fd; +#else + SFile *pFile = helperHeadF(pHelper); +#endif + int fd = pFile->fd; if (!helperHasState(pHelper, TSDB_HELPER_IDX_LOAD)) { // If not load from file, just load it in object @@ -468,7 +500,7 @@ int tsdbLoadCompIdx(SRWHelper *pHelper, void *target) { return -1; } - if (lseek(fd, TSDB_FILE_HEAD_SIZE, SEEK_SET) < 0) { + if (lseek(fd, pFile->info.offset, SEEK_SET) < 0) { tsdbError("vgId:%d failed to lseek file %s since %s", REPO_ID(pHelper->pRepo), pFile->fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -516,11 +548,6 @@ int tsdbLoadCompIdx(SRWHelper *pHelper, void *target) { ASSERT(POINTER_DISTANCE(ptr, pHelper->pBuffer) <= pFile->info.len - sizeof(TSCKSUM)); } - - // if (lseek(fd, TSDB_FILE_HEAD_SIZE, SEEK_SET) < 0) { - // terrno = TAOS_SYSTEM_ERROR(errno); - // return -1; - // } } } helperSetState(pHelper, TSDB_HELPER_IDX_LOAD); @@ -1031,13 +1058,15 @@ static void tsdbResetHelperFileImpl(SRWHelper *pHelper) { pHelper->idxH.numOfIdx = 0; pHelper->idxH.curIdx = 0; memset((void *)&pHelper->files, 0, sizeof(pHelper->files)); - helperIdxF(pHelper)->fd = -1; helperHeadF(pHelper)->fd = -1; helperDataF(pHelper)->fd = -1; helperLastF(pHelper)->fd = -1; - helperNewIdxF(pHelper)->fd = -1; helperNewHeadF(pHelper)->fd = -1; helperNewLastF(pHelper)->fd = -1; +#ifdef TSDB_IDX + helperIdxF(pHelper)->fd = -1; + helperNewIdxF(pHelper)->fd = -1; +#endif } static int tsdbInitHelperFile(SRWHelper *pHelper) { From dfe2c4fc67756167c13cfebe0de4ca515e1514b8 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 20 Jul 2020 17:16:59 +0800 Subject: [PATCH 042/100] [td-225] fix memory leaks. --- src/query/inc/qResultbuf.h | 1 + src/query/src/qResultbuf.c | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/query/inc/qResultbuf.h b/src/query/inc/qResultbuf.h index d76659cc91..8c8afb0957 100644 --- a/src/query/inc/qResultbuf.h +++ b/src/query/inc/qResultbuf.h @@ -42,6 +42,7 @@ typedef struct SDiskbasedResultBuf { void* iBuf; // inmemory buf void* handle; // for debug purpose + void* emptyDummyIdList; // dummy id list } SDiskbasedResultBuf; #define DEFAULT_INTERN_BUF_PAGE_SIZE (1024L) diff --git a/src/query/src/qResultbuf.c b/src/query/src/qResultbuf.c index 0eefb106b9..de59676e59 100644 --- a/src/query/src/qResultbuf.c +++ b/src/query/src/qResultbuf.c @@ -36,6 +36,7 @@ int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t nu pResBuf->fd = FD_INITIALIZER; pResBuf->pBuf = NULL; + pResBuf->emptyDummyIdList = taosArrayInit(1, sizeof(int32_t)); qDebug("QInfo:%p create resBuf for output, page size:%d, initial pages:%d, %" PRId64 "bytes", handle, pResBuf->pageSize, pResBuf->numOfPages, pResBuf->totalBufSize); @@ -173,7 +174,7 @@ int32_t getNumOfRowsPerPage(SDiskbasedResultBuf* pResultBuf) { return pResultBuf SIDList getDataBufPagesIdList(SDiskbasedResultBuf* pResultBuf, int32_t groupId) { int32_t slot = getGroupIndex(pResultBuf, groupId); if (slot < 0) { - return taosArrayInit(1, sizeof(int32_t)); + return pResultBuf->emptyDummyIdList; } else { return taosArrayGetP(pResultBuf->list, slot); } @@ -206,6 +207,7 @@ void destroyResultBuf(SDiskbasedResultBuf* pResultBuf, void* handle) { } taosArrayDestroy(pResultBuf->list); + taosArrayDestroy(pResultBuf->emptyDummyIdList); taosHashCleanup(pResultBuf->idsTable); tfree(pResultBuf->iBuf); From 975f8c2c97f97652e1c4bb8f36ce640c3d5cdcdf Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 20 Jul 2020 18:32:26 +0800 Subject: [PATCH 043/100] update file magic with other method --- src/tsdb/inc/tsdbMain.h | 4 +- src/tsdb/src/tsdbFile.c | 1 + src/tsdb/src/tsdbMain.c | 16 +++--- src/tsdb/src/tsdbRWHelper.c | 97 ++++++++++++++++++++----------------- src/util/inc/tkvstore.h | 11 +++-- src/util/src/tkvstore.c | 8 ++- 6 files changed, 80 insertions(+), 57 deletions(-) diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h index 8ab89d72ed..156a408059 100644 --- a/src/tsdb/inc/tsdbMain.h +++ b/src/tsdb/inc/tsdbMain.h @@ -42,6 +42,7 @@ extern int tsdbDebugFlag; #define TSDB_MAX_TABLE_SCHEMAS 16 #define TSDB_FILE_HEAD_SIZE 512 #define TSDB_FILE_DELIMITER 0xF00AFA0F +#define TSDB_FILE_INIT_MAGIC 0xFFFFFFFF // Definitions // ------------------ tsdbMeta.c @@ -230,7 +231,7 @@ typedef struct { typedef struct { int32_t delimiter; // For recovery usage - int32_t checksum; // TODO: decide if checksum logic in this file or make it one API + int32_t tid; uint64_t uid; SCompBlock blocks[]; } SCompInfo; @@ -308,6 +309,7 @@ typedef struct { #define TABLE_TID(t) (t)->tableId.tid #define TABLE_SUID(t) (t)->suid #define TABLE_LASTKEY(t) (t)->lastKey +#define TSDB_META_FILE_MAGIC(m) KVSTORE_MAGIC((m)->pStore) STsdbMeta* tsdbNewMeta(STsdbCfg* pCfg); void tsdbFreeMeta(STsdbMeta* pMeta); diff --git a/src/tsdb/src/tsdbFile.c b/src/tsdb/src/tsdbFile.c index 0a18d15773..767fbc8252 100644 --- a/src/tsdb/src/tsdbFile.c +++ b/src/tsdb/src/tsdbFile.c @@ -264,6 +264,7 @@ int tsdbCreateFile(SFile *pFile, STsdbRepo *pRepo, int fid, int type) { } pFile->info.size = TSDB_FILE_HEAD_SIZE; + pFile->info.magic = TSDB_FILE_INIT_MAGIC; if (tsdbUpdateFileHeader(pFile, 0) < 0) { tsdbCloseFile(pFile); diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index 82dbe5f498..7fe69021bf 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -212,6 +212,8 @@ uint32_t tsdbGetFileInfo(TSDB_REPO_T *repo, char *name, uint32_t *index, uint32_ char *sdup = strdup(pRepo->rootDir); char *prefix = dirname(sdup); + int prefixLen = strlen(prefix); + tfree(sdup); if (name[0] == 0) { // get the file from index or after, but not larger than eindex int fid = (*index) / TSDB_FILE_TYPE_MAX; @@ -220,8 +222,8 @@ uint32_t tsdbGetFileInfo(TSDB_REPO_T *repo, char *name, uint32_t *index, uint32_ if (*index <= TSDB_META_FILE_INDEX && TSDB_META_FILE_INDEX <= eindex) { fname = tsdbGetMetaFileName(pRepo->rootDir); *index = TSDB_META_FILE_INDEX; + magic = TSDB_META_FILE_MAGIC(pRepo->tsdbMeta); } else { - tfree(sdup); return 0; } } else { @@ -229,42 +231,42 @@ uint32_t tsdbGetFileInfo(TSDB_REPO_T *repo, char *name, uint32_t *index, uint32_ taosbsearch(&fid, pFileH->pFGroup, pFileH->nFGroups, sizeof(SFileGroup), keyFGroupCompFunc, TD_GE); if (pFGroup->fileId == fid) { fname = strdup(pFGroup->files[(*index) % TSDB_FILE_TYPE_MAX].fname); + magic = pFGroup->files[(*index) % TSDB_FILE_TYPE_MAX].info.magic; } else { if ((pFGroup->fileId + 1) * TSDB_FILE_TYPE_MAX - 1 < eindex) { fname = strdup(pFGroup->files[0].fname); *index = pFGroup->fileId * TSDB_FILE_TYPE_MAX; + magic = pFGroup->files[0].info.magic; } else { - tfree(sdup); return 0; } } } - strcpy(name, fname + strlen(prefix)); + strcpy(name, fname + prefixLen); } else { // get the named file at the specified index. If not there, return 0 if (*index == TSDB_META_FILE_INDEX) { // get meta file fname = tsdbGetMetaFileName(pRepo->rootDir); + magic = TSDB_META_FILE_MAGIC(pRepo->tsdbMeta); } else { int fid = (*index) / TSDB_FILE_TYPE_MAX; SFileGroup *pFGroup = tsdbSearchFGroup(pFileH, fid, TD_EQ); if (pFGroup == NULL) { // not found - tfree(sdup); return 0; } SFile *pFile = &pFGroup->files[(*index) % TSDB_FILE_TYPE_MAX]; fname = strdup(pFile->fname); + magic = pFile->info.magic; } } if (stat(fname, &fState) < 0) { - tfree(sdup); tfree(fname); return 0; } - tfree(sdup); *size = fState.st_size; - magic = *size; + // magic = *size; tfree(fname); return magic; diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index 22182694de..0b8d1fec49 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -100,6 +100,7 @@ void tsdbResetHelper(SRWHelper *pHelper) { int tsdbSetAndOpenHelperFile(SRWHelper *pHelper, SFileGroup *pGroup) { ASSERT(pHelper != NULL && pGroup != NULL); + SFile *pFile = NULL; // Clear the helper object tsdbResetHelper(pHelper); @@ -127,18 +128,27 @@ int tsdbSetAndOpenHelperFile(SRWHelper *pHelper, SFileGroup *pGroup) { #ifdef TSDB_IDX // Create and open .i file - if (tsdbOpenFile(helperNewIdxF(pHelper), O_WRONLY | O_CREAT) < 0) return -1; - if (tsdbUpdateFileHeader(helperNewIdxF(pHelper), 0) < 0) return -1; + pFile = helperNewIdxF(pHelper); + if (tsdbOpenFile(pFile, O_WRONLY | O_CREAT) < 0) return -1; + pFile->info.size = TSDB_FILE_HEAD_SIZE; + pFile->info.magic = TSDB_FILE_INIT_MAGIC; + if (tsdbUpdateFileHeader(pFile, 0) < 0) return -1; #endif // Create and open .h - if (tsdbOpenFile(helperNewHeadF(pHelper), O_WRONLY | O_CREAT) < 0) return -1; - if (tsdbUpdateFileHeader(helperNewHeadF(pHelper), 0) < 0) return -1; + pFile = helperNewHeadF(pHelper); + if (tsdbOpenFile(pFile, O_WRONLY | O_CREAT) < 0) return -1; + pFile->info.size = TSDB_FILE_HEAD_SIZE; + pFile->info.magic = TSDB_FILE_INIT_MAGIC; + if (tsdbUpdateFileHeader(pFile, 0) < 0) return -1; // Create and open .l file if should if (tsdbShouldCreateNewLast(pHelper)) { - if (tsdbOpenFile(helperNewLastF(pHelper), O_WRONLY | O_CREAT) < 0) goto _err; - if (tsdbUpdateFileHeader(helperNewLastF(pHelper), 0) < 0) return -1; + pFile = helperNewLastF(pHelper); + if (tsdbOpenFile(pFile, O_WRONLY | O_CREAT) < 0) goto _err; + pFile->info.size = TSDB_FILE_HEAD_SIZE; + pFile->info.magic = TSDB_FILE_INIT_MAGIC; + if (tsdbUpdateFileHeader(pFile, 0) < 0) return -1; } } else { if (tsdbOpenFile(helperDataF(pHelper), O_RDONLY) < 0) goto _err; @@ -334,7 +344,15 @@ int tsdbMoveLastBlockIfNeccessary(SRWHelper *pHelper) { SCompBlock *pCompBlock = blockAtIdx(pHelper, pIdx->numOfBlocks - 1); ASSERT(pCompBlock->last); + if (tsdbLoadBlockData(pHelper, pCompBlock, NULL) < 0) return -1; + ASSERT(pHelper->pDataCols[0]->numOfRows == pCompBlock->numOfRows && + pHelper->pDataCols[0]->numOfRows < pCfg->minRowsPerFileBlock); + if (tsdbWriteBlockToFile(pHelper, helperNewLastF(pHelper), pHelper->pDataCols[0], &compBlock, true, true) < 0) + return -1; + if (tsdbUpdateSuperBlock(pHelper, &compBlock, pIdx->numOfBlocks - 1) < 0) return -1; + +#if 0 if (pCompBlock->numOfSubBlocks > 1) { if (tsdbLoadBlockData(pHelper, pCompBlock, NULL) < 0) return -1; ASSERT(pHelper->pDataCols[0]->numOfRows == pCompBlock->numOfRows && @@ -365,6 +383,7 @@ int tsdbMoveLastBlockIfNeccessary(SRWHelper *pHelper) { return -1; } } +#endif pHelper->hasOldLastBlock = false; } @@ -379,50 +398,34 @@ int tsdbWriteCompInfo(SRWHelper *pHelper) { if (pIdx->len > 0) { if (!helperHasState(pHelper, TSDB_HELPER_INFO_LOAD)) { - offset = lseek(pFile->fd, 0, SEEK_END); - if (offset < 0) { - tsdbError("vgId:%d failed to lseed file %s since %s", REPO_ID(pHelper->pRepo), pFile->fname, - strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - pIdx->offset = offset; - pIdx->uid = pHelper->tableInfo.uid; - pIdx->tid = pHelper->tableInfo.tid; - ASSERT(pIdx->offset >= TSDB_FILE_HEAD_SIZE); - - if (tsendfile(pFile->fd, helperHeadF(pHelper)->fd, NULL, pIdx->len) < pIdx->len) { - tsdbError("vgId:%d failed to send %d bytes from file %s to %s since %s", REPO_ID(pHelper->pRepo), pIdx->len, - helperHeadF(pHelper)->fname, pFile->fname, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } + if (tsdbLoadCompInfo(pHelper, NULL) < 0) return -1; } else { pHelper->pCompInfo->delimiter = TSDB_FILE_DELIMITER; pHelper->pCompInfo->uid = pHelper->tableInfo.uid; - pHelper->pCompInfo->checksum = 0; + pHelper->pCompInfo->tid = pHelper->tableInfo.tid; ASSERT(pIdx->len > sizeof(SCompInfo) + sizeof(TSCKSUM) && (pIdx->len - sizeof(SCompInfo) - sizeof(TSCKSUM)) % sizeof(SCompBlock) == 0); taosCalcChecksumAppend(0, (uint8_t *)pHelper->pCompInfo, pIdx->len); - offset = lseek(pFile->fd, 0, SEEK_END); - if (offset < 0) { - tsdbError("vgId:%d failed to lseek file %s since %s", REPO_ID(pHelper->pRepo), pFile->fname, - strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - pIdx->offset = offset; - pIdx->uid = pHelper->tableInfo.uid; - pIdx->tid = pHelper->tableInfo.tid; - ASSERT(pIdx->offset >= TSDB_FILE_HEAD_SIZE); + } - if (twrite(pFile->fd, (void *)(pHelper->pCompInfo), pIdx->len) < pIdx->len) { - tsdbError("vgId:%d failed to write %d bytes to file %s since %s", REPO_ID(pHelper->pRepo), pIdx->len, - pFile->fname, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } + pFile->info.magic = taosCalcChecksum( + pFile->info.magic, (uint8_t *)POINTER_SHIFT(pHelper->pCompInfo, pIdx->len - sizeof(TSCKSUM)), sizeof(TSCKSUM)); + offset = lseek(pFile->fd, 0, SEEK_END); + if (offset < 0) { + tsdbError("vgId:%d failed to lseek file %s since %s", REPO_ID(pHelper->pRepo), pFile->fname, strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + pIdx->offset = offset; + pIdx->uid = pHelper->tableInfo.uid; + pIdx->tid = pHelper->tableInfo.tid; + ASSERT(pIdx->offset >= TSDB_FILE_HEAD_SIZE); + + if (twrite(pFile->fd, (void *)(pHelper->pCompInfo), pIdx->len) < pIdx->len) { + tsdbError("vgId:%d failed to write %d bytes to file %s since %s", REPO_ID(pHelper->pRepo), pIdx->len, + pFile->fname, strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; } #ifdef TSDB_IDX @@ -463,6 +466,8 @@ int tsdbWriteCompIdx(SRWHelper *pHelper) { } } taosCalcChecksumAppend(0, (uint8_t *)pHelper->pWIdx, pFile->info.len); + pFile->info.magic = taosCalcChecksum( + pFile->info.magic, (uint8_t *)POINTER_SHIFT(pHelper->pWIdx, pFile->info.len - sizeof(TSCKSUM)), sizeof(TSCKSUM)); offset = lseek(pFile->fd, 0, SEEK_END); if (offset < 0) { @@ -594,7 +599,7 @@ int tsdbLoadCompInfo(SRWHelper *pHelper, void *target) { return -1; } - ASSERT(pIdx->uid == pHelper->pCompInfo->uid); + ASSERT(pIdx->uid == pHelper->pCompInfo->uid && pIdx->tid == pHelper->pCompInfo->tid); } helperSetState(pHelper, TSDB_HELPER_INFO_LOAD); @@ -813,6 +818,8 @@ static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDa ASSERT(flen > 0); flen += sizeof(TSCKSUM); taosCalcChecksumAppend(0, (uint8_t *)tptr, flen); + pFile->info.magic = + taosCalcChecksum(pFile->info.magic, (uint8_t *)POINTER_SHIFT(tptr, flen - sizeof(TSCKSUM)), sizeof(TSCKSUM)); if (ncol != 0) { pCompCol->offset = toffset; @@ -831,6 +838,8 @@ static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDa pCompData->numOfCols = nColsNotAllNull; taosCalcChecksumAppend(0, (uint8_t *)pCompData, tsize); + pFile->info.magic = taosCalcChecksum(pFile->info.magic, (uint8_t *)POINTER_SHIFT(pCompData, tsize - sizeof(TSCKSUM)), + sizeof(TSCKSUM)); // Write the whole block to file if (twrite(pFile->fd, (void *)pCompData, lsize) < lsize) { diff --git a/src/util/inc/tkvstore.h b/src/util/inc/tkvstore.h index 346e567c41..6d67607e24 100644 --- a/src/util/inc/tkvstore.h +++ b/src/util/inc/tkvstore.h @@ -25,10 +25,11 @@ typedef int (*iterFunc)(void *, void *cont, int contLen); typedef void (*afterFunc)(void *); typedef struct { - int64_t size; // including 512 bytes of header size - int64_t tombSize; - int64_t nRecords; - int64_t nDels; + int64_t size; // including 512 bytes of header size + int64_t tombSize; + int64_t nRecords; + int64_t nDels; + uint32_t magic; } SStoreInfo; typedef struct { @@ -45,6 +46,8 @@ typedef struct { SStoreInfo info; } SKVStore; +#define KVSTORE_MAGIC(s) (s)->info.magic + int tdCreateKVStore(char *fname); int tdDestroyKVStore(char *fname); SKVStore *tdOpenKVStore(char *fname, iterFunc iFunc, afterFunc aFunc, void *appH); diff --git a/src/util/src/tkvstore.c b/src/util/src/tkvstore.c index ab2aa738c6..a2022569bf 100644 --- a/src/util/src/tkvstore.c +++ b/src/util/src/tkvstore.c @@ -34,6 +34,7 @@ #define TD_KVSTORE_MAINOR_VERSION 0 #define TD_KVSTORE_SNAP_SUFFIX ".snap" #define TD_KVSTORE_NEW_SUFFIX ".new" +#define TD_KVSTORE_INIT_MAGIC 0xFFFFFFFF typedef struct { uint64_t uid; @@ -251,6 +252,8 @@ int tdUpdateKVStoreRecord(SKVStore *pStore, uint64_t uid, void *cont, int contLe return -1; } + pStore->info.magic = + taosCalcChecksum(pStore->info.magic, (uint8_t *)POINTER_SHIFT(cont, contLen - sizeof(TSCKSUM)), sizeof(TSCKSUM)); pStore->info.size += (sizeof(SKVRecord) + contLen); SKVRecord *pRecord = taosHashGet(pStore->map, (void *)&uid, sizeof(uid)); if (pRecord != NULL) { // just to insert @@ -288,6 +291,7 @@ int tdDropKVStoreRecord(SKVStore *pStore, uint64_t uid) { return -1; } + pStore->info.magic = taosCalcChecksum(pStore->info.magic, (uint8_t *)buf, POINTER_DISTANCE(pBuf, buf)); pStore->info.size += POINTER_DISTANCE(pBuf, buf); pStore->info.nDels++; pStore->info.nRecords--; @@ -371,7 +375,7 @@ static int tdUpdateKVStoreHeader(int fd, char *fname, SStoreInfo *pInfo) { } static int tdInitKVStoreHeader(int fd, char *fname) { - SStoreInfo info = {TD_KVSTORE_HEADER_SIZE, 0, 0, 0}; + SStoreInfo info = {TD_KVSTORE_HEADER_SIZE, 0, 0, 0, TD_KVSTORE_INIT_MAGIC}; return tdUpdateKVStoreHeader(fd, fname, &info); } @@ -382,6 +386,7 @@ static int tdEncodeStoreInfo(void **buf, SStoreInfo *pInfo) { tlen += taosEncodeVariantI64(buf, pInfo->tombSize); tlen += taosEncodeVariantI64(buf, pInfo->nRecords); tlen += taosEncodeVariantI64(buf, pInfo->nDels); + tlen += taosEncodeFixedU32(buf, pInfo->magic); return tlen; } @@ -391,6 +396,7 @@ static void *tdDecodeStoreInfo(void *buf, SStoreInfo *pInfo) { buf = taosDecodeVariantI64(buf, &(pInfo->tombSize)); buf = taosDecodeVariantI64(buf, &(pInfo->nRecords)); buf = taosDecodeVariantI64(buf, &(pInfo->nDels)); + buf = taosDecodeFixedU32(buf, &(pInfo->magic)); return buf; } From 76e36a8d4e2a1d8c4a594c04ba64a31d9add6786 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 21 Jul 2020 03:05:47 +0000 Subject: [PATCH 044/100] fix defect found in coverity scan --- src/mnode/src/mnodeDb.c | 18 +++++++++++++----- src/mnode/src/mnodeSdb.c | 2 +- src/mnode/src/mnodeVgroup.c | 17 ++++++++++++----- src/plugins/http/src/gcJson.c | 4 ++++ src/plugins/http/src/httpContext.c | 2 +- src/plugins/http/src/restJson.c | 4 ++++ src/plugins/mqtt/src/mqttSystem.c | 2 +- 7 files changed, 36 insertions(+), 13 deletions(-) diff --git a/src/mnode/src/mnodeDb.c b/src/mnode/src/mnodeDb.c index fb97d6f380..a159e98ed5 100644 --- a/src/mnode/src/mnodeDb.c +++ b/src/mnode/src/mnodeDb.c @@ -67,8 +67,11 @@ static int32_t mnodeDbActionInsert(SSdbOper *pOper) { SAcctObj *pAcct = mnodeGetAcct(pDb->acct); pthread_mutex_init(&pDb->mutex, NULL); + pthread_mutex_lock(&pDb->mutex); pDb->vgListSize = VG_LIST_SIZE; pDb->vgList = calloc(pDb->vgListSize, sizeof(SVgObj *)); + pthread_mutex_unlock(&pDb->mutex); + pDb->numOfVgroups = 0; pDb->numOfTables = 0; pDb->numOfSuperTables = 0; @@ -395,8 +398,8 @@ static int32_t mnodeCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate, void *pMs code = sdbInsertRow(&oper); if (code != TSDB_CODE_SUCCESS) { - mnodeDestroyDb(pDb); mLInfo("db:%s, failed to create, reason:%s", pDb->name, tstrerror(code)); + mnodeDestroyDb(pDb); return code; } else { return TSDB_CODE_MND_ACTION_IN_PROGRESS; @@ -605,7 +608,9 @@ static int32_t mnodeGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn static char *mnodeGetDbStr(char *src) { char *pos = strstr(src, TS_PATH_DELIMITER); - return ++pos; + if (pos != NULL) ++pos; + + return pos; } static int32_t mnodeRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void *pConn) { @@ -622,10 +627,13 @@ static int32_t mnodeRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void cols = 0; - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; char* name = mnodeGetDbStr(pDb->name); - STR_WITH_MAXSIZE_TO_VARSTR(pWrite, name, pShow->bytes[cols]); + if (name != NULL) { + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, name, pShow->bytes[cols]); + } else { + STR_TO_VARSTR(pWrite, "NULL"); + } cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index 42ded7ed06..dee82f53be 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -453,7 +453,7 @@ static int32_t sdbInsertHash(SSdbTable *pTable, SSdbOper *pOper) { keySize = strlen((char *)key); } - taosHashPut(pTable->iHandle, key, keySize, &pOper->pObj, sizeof(void **)); + taosHashPut(pTable->iHandle, key, keySize, &pOper->pObj, sizeof(int64_t)); sdbIncRef(pTable, pOper->pObj); atomic_add_fetch_32(&pTable->numOfRows, 1); diff --git a/src/mnode/src/mnodeVgroup.c b/src/mnode/src/mnodeVgroup.c index 1de591df7c..46255a6c8f 100644 --- a/src/mnode/src/mnodeVgroup.c +++ b/src/mnode/src/mnodeVgroup.c @@ -434,15 +434,22 @@ int32_t mnodeGetAvailableVgroup(SMnodeMsg *pMsg, SVgObj **ppVgroup, int32_t *pSi } if (pDb->numOfVgroups < maxVgroupsPerDb) { - mDebug("app:%p:%p, db:%s, try to create a new vgroup, numOfVgroups:%d maxVgroupsPerDb:%d", pMsg->rpcMsg.ahandle, pMsg, - pDb->name, pDb->numOfVgroups, maxVgroupsPerDb); + mDebug("app:%p:%p, db:%s, try to create a new vgroup, numOfVgroups:%d maxVgroupsPerDb:%d", pMsg->rpcMsg.ahandle, + pMsg, pDb->name, pDb->numOfVgroups, maxVgroupsPerDb); pthread_mutex_unlock(&pDb->mutex); int32_t code = mnodeCreateVgroup(pMsg); - if (code == TSDB_CODE_MND_ACTION_IN_PROGRESS) return code; + if (code == TSDB_CODE_MND_ACTION_IN_PROGRESS) { + return code; + } else { + pthread_mutex_lock(&pDb->mutex); + } } SVgObj *pVgroup = pDb->vgList[0]; - if (pVgroup == NULL) return TSDB_CODE_MND_NO_ENOUGH_DNODES; + if (pVgroup == NULL) { + pthread_mutex_unlock(&pDb->mutex); + return TSDB_CODE_MND_NO_ENOUGH_DNODES; + } int32_t code = mnodeAllocVgroupIdPool(pVgroup); if (code != TSDB_CODE_SUCCESS) { @@ -483,7 +490,7 @@ static int32_t mnodeCreateVgroupCb(SMnodeMsg *pMsg, int32_t code) { } else { pVgroup->status = TAOS_VG_STATUS_READY; SSdbOper desc = {.type = SDB_OPER_GLOBAL, .pObj = pVgroup, .table = tsVgroupSdb}; - sdbUpdateRow(&desc); + (void)sdbUpdateRow(&desc); } mInfo("app:%p:%p, vgId:%d, is created in mnode, db:%s replica:%d", pMsg->rpcMsg.ahandle, pMsg, pVgroup->vgId, diff --git a/src/plugins/http/src/gcJson.c b/src/plugins/http/src/gcJson.c index 544a11b5fc..94d53db6ef 100644 --- a/src/plugins/http/src/gcJson.c +++ b/src/plugins/http/src/gcJson.c @@ -121,6 +121,10 @@ bool gcBuildQueryJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, for (int k = 0; k < numOfRows; ++k) { TAOS_ROW row = taos_fetch_row(result); + if (row == NULL) { + cmd->numOfRows--; + continue; + } int32_t* length = taos_fetch_lengths(result); // for group by diff --git a/src/plugins/http/src/httpContext.c b/src/plugins/http/src/httpContext.c index cefcca7821..225977abae 100644 --- a/src/plugins/http/src/httpContext.c +++ b/src/plugins/http/src/httpContext.c @@ -108,7 +108,7 @@ HttpContext *httpCreateContext(int32_t fd) { pContext->lastAccessTime = taosGetTimestampSec(); pContext->state = HTTP_CONTEXT_STATE_READY; - HttpContext **ppContext = taosCachePut(tsHttpServer.contextCache, &pContext, sizeof(void *), &pContext, sizeof(void *), 3); + HttpContext **ppContext = taosCachePut(tsHttpServer.contextCache, &pContext, sizeof(int64_t), &pContext, sizeof(int64_t), 3); pContext->ppContext = ppContext; httpDebug("context:%p, fd:%d, is created, data:%p", pContext, fd, ppContext); diff --git a/src/plugins/http/src/restJson.c b/src/plugins/http/src/restJson.c index 53b0248149..7a73f6559f 100644 --- a/src/plugins/http/src/restJson.c +++ b/src/plugins/http/src/restJson.c @@ -94,6 +94,10 @@ bool restBuildSqlJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, for (int k = 0; k < numOfRows; ++k) { TAOS_ROW row = taos_fetch_row(result); + if (row == NULL) { + cmd->numOfRows--; + continue; + } int32_t* length = taos_fetch_lengths(result); // data row array begin diff --git a/src/plugins/mqtt/src/mqttSystem.c b/src/plugins/mqtt/src/mqttSystem.c index 2687106124..0259ea23eb 100644 --- a/src/plugins/mqtt/src/mqttSystem.c +++ b/src/plugins/mqtt/src/mqttSystem.c @@ -64,7 +64,7 @@ int32_t mqttInitSystem() { } char* _begin_hostname = strstr(url, recntStatus.hostname); - if (strstr(_begin_hostname, ":") != NULL) { + if (_begin_hostname != NULL && strstr(_begin_hostname, ":") != NULL) { recntStatus.port = strbetween(_begin_hostname, ":", "/"); } else { recntStatus.port = strbetween("'1883'", "'", "'"); From ce0ffb408090d3ae64398e8d9fb77e1f9e12e33f Mon Sep 17 00:00:00 2001 From: liu0x54 Date: Tue, 21 Jul 2020 03:36:34 +0000 Subject: [PATCH 045/100] [TD-970] fix taosdemo bugs --- src/kit/taosdemo/taosdemo.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 305302b71a..79cfbe2a37 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -361,7 +361,7 @@ int main(int argc, char *argv[]) { arguments.num_of_DPT = 100000; arguments.num_of_RPR = 1000; arguments.use_metric = true; - arguments.insert_only = true; + arguments.insert_only = false; // end change argp_parse(&argp, argc, argv, 0, 0, &arguments); @@ -954,13 +954,13 @@ void *readMetric(void *sarg) { for (int i = 1; i <= m; i++) { if (i == 1) { - sprintf(tempS, "index = %d", i); + sprintf(tempS, "areaid = %d", i); } else { - sprintf(tempS, " or index = %d ", i); + sprintf(tempS, " or areaid = %d ", i); } strcat(condition, tempS); - sprintf(command, "select %s from m1 where %s", aggreFunc[j], condition); + sprintf(command, "select %s from meters where %s", aggreFunc[j], condition); printf("Where condition: %s\n", condition); fprintf(fp, "%s\n", command); From a0ca29dcfa3635079e9f646f9b7cfd7d22c3a7f0 Mon Sep 17 00:00:00 2001 From: liu0x54 Date: Tue, 21 Jul 2020 04:02:53 +0000 Subject: [PATCH 046/100] [TD-971] change create db sql --- tests/examples/python/read_example.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/examples/python/read_example.py b/tests/examples/python/read_example.py index a96115dfb2..73c4b95214 100644 --- a/tests/examples/python/read_example.py +++ b/tests/examples/python/read_example.py @@ -29,7 +29,7 @@ if __name__ == '__main__': # Create a database named db try: - c1.execute('create database db') + c1.execute('create database if not exists db ') except Exception as err: conn.close() raise(err) From 88ba2f49359c6b5fbfed26e942d6d8b1944584fc Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 21 Jul 2020 13:43:46 +0800 Subject: [PATCH 047/100] add missed compile option --- src/tsdb/inc/tsdbMain.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h index ba5711a81c..762d2253e2 100644 --- a/src/tsdb/inc/tsdbMain.h +++ b/src/tsdb/inc/tsdbMain.h @@ -263,7 +263,9 @@ typedef struct { TSKEY minKey; TSKEY maxKey; SFileGroup fGroup; - SFile nIdxF; +#ifdef TSDB_IDX + SFile nIdxF; +#endif SFile nHeadF; SFile nLastF; } SHelperFile; From 36f4948b2598b8d5b049d853feca577b73e21e5b Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Tue, 21 Jul 2020 05:45:44 +0000 Subject: [PATCH 048/100] set link secured except code is NOT_READY --- src/rpc/src/rpcMain.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index 2325d12d92..4790c22668 100644 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -446,7 +446,10 @@ void rpcSendResponse(const SRpcMsg *pRsp) { // set the idle timer to monitor the activity taosTmrReset(rpcProcessIdleTimer, pRpc->idleTime, pConn, pRpc->tmrCtrl, &pConn->pIdleTimer); rpcSendMsgToPeer(pConn, msg, msgLen); - pConn->secured = 1; // connection shall be secured + + // if not set to secured, set it expcet NOT_READY case, since client wont treat it as secured + if (pConn->secured == 0 && pMsg->code != TSDB_CODE_RPC_NOT_READY) + pConn->secured = 1; // connection shall be secured if (pConn->pReqMsg) rpcFreeCont(pConn->pReqMsg); pConn->pReqMsg = NULL; From 08fe5fbfa66c3869223c3adab63ae0d1de92492f Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 21 Jul 2020 13:55:31 +0800 Subject: [PATCH 049/100] increase crash_gen step to 1000. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 4d7a809e29..f2cd83343e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -63,7 +63,7 @@ matrix: pkill -TERM -x taosd fuser -k -n tcp 6030 sleep 1 - ./crash_gen.sh -a -p -t 4 -s 25|| travis_terminate $? + ./crash_gen.sh -a -p -t 4 -s 1000|| travis_terminate $? sleep 1 cd ${TRAVIS_BUILD_DIR}/tests/pytest From fac8cf100bfd774dd30c5dbd3c7afcf25009668a Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 21 Jul 2020 14:17:05 +0800 Subject: [PATCH 050/100] refactor and debug --- src/tsdb/src/tsdbRWHelper.c | 44 ++++++++++++++----------------------- src/tsdb/src/tsdbRead.c | 24 +++++++++++++++----- 2 files changed, 34 insertions(+), 34 deletions(-) diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index 0b8d1fec49..cec3b0d36b 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -60,7 +60,6 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter int *blkIdx); static int tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIter *pCommitIter, SDataCols *pTarget, TSKEY maxKey, int maxRows); -static int tsdbCompareTidIdx(const void *key1, const void *key2); // ---------------------- INTERNAL FUNCTIONS ---------------------- int tsdbInitReadHelper(SRWHelper *pHelper, STsdbRepo *pRepo) { @@ -276,32 +275,31 @@ void tsdbSetHelperTable(SRWHelper *pHelper, STable *pTable, STsdbRepo *pRepo) { tdInitDataCols(pHelper->pDataCols[0], pSchema); tdInitDataCols(pHelper->pDataCols[1], pSchema); - if (helperType(pHelper) == TSDB_WRITE_HELPER) { - if (pHelper->idxH.numOfIdx > 0) { + if (pHelper->idxH.numOfIdx > 0) { + while (true) { if (pHelper->idxH.curIdx >= pHelper->idxH.numOfIdx) { memset(&(pHelper->curCompIdx), 0, sizeof(SCompIdx)); - } else { - SCompIdx *pIdx = &(pHelper->idxH.pIdxArray[pHelper->idxH.curIdx]); - if (pIdx->tid == TABLE_TID(pTable)) { + break; + } + + SCompIdx *pIdx = &(pHelper->idxH.pIdxArray[pHelper->idxH.curIdx]); + if (pIdx->tid == TABLE_TID(pTable)) { + if (pIdx->uid == TABLE_UID(pTable)) { pHelper->curCompIdx = *pIdx; - pHelper->idxH.curIdx++; } else { - ASSERT(pIdx->tid > TABLE_TID(pTable)); memset(&(pHelper->curCompIdx), 0, sizeof(SCompIdx)); } + pHelper->idxH.curIdx++; + break; + } else if (pIdx->tid > TABLE_TID(pTable)) { + memset(&(pHelper->curCompIdx), 0, sizeof(SCompIdx)); + break; + } else { + pHelper->idxH.curIdx++; } - } else { - memset(&(pHelper->curCompIdx), 0, sizeof(SCompIdx)); } } else { - // TODO: make it more efficient - void *ptr = bsearch(&TABLE_TID(pTable), (void *)pHelper->idxH.pIdxArray, pHelper->idxH.numOfIdx, sizeof(SCompIdx), - tsdbCompareTidIdx); - if (ptr == NULL) { - memset(&(pHelper->curCompIdx), 0, sizeof(SCompIdx)); - } else { - pHelper->curCompIdx = *(SCompIdx *)ptr; - } + memset(&(pHelper->curCompIdx), 0, sizeof(SCompIdx)); } helperSetState(pHelper, TSDB_HELPER_TABLE_SET); @@ -1681,14 +1679,4 @@ static int tsdbWriteBlockToProperFile(SRWHelper *pHelper, SDataCols *pDataCols, if (tsdbWriteBlockToFile(pHelper, pFile, pDataCols, pCompBlock, isLast, true) < 0) return -1; return 0; -} - -static int tsdbCompareTidIdx(const void *key1, const void *key2) { - if (*(int32_t *)key1 > ((SCompIdx *)key2)->tid) { - return 1; - } else if (*(int32_t *)key1 < ((SCompIdx *)key2)->tid) { - return -1; - } else { - return 0; - } } \ No newline at end of file diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index b2a3887bed..b2204948d0 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -126,12 +126,13 @@ typedef struct STsdbQueryHandle { SIOCostSummary cost; } STsdbQueryHandle; -static void changeQueryHandleForLastrowQuery(TsdbQueryHandleT pqHandle); -static void changeQueryHandleForInterpQuery(TsdbQueryHandleT pHandle); -static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SCompBlock* pBlock); +static void changeQueryHandleForLastrowQuery(TsdbQueryHandleT pqHandle); +static void changeQueryHandleForInterpQuery(TsdbQueryHandleT pHandle); +static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SCompBlock* pBlock); static int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order); -static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int maxRowsToRead, STimeWindow* win, - STsdbQueryHandle* pQueryHandle); +static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int maxRowsToRead, STimeWindow* win, + STsdbQueryHandle* pQueryHandle); +static int tsdbCheckInfoCompar(const void* key1, const void* key2); static void tsdbInitDataBlockLoadInfo(SDataBlockLoadInfo* pBlockLoadInfo) { pBlockLoadInfo->slot = -1; @@ -236,7 +237,8 @@ TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STab taosArrayPush(pQueryHandle->pTableCheckInfo, &info); } } - + + taosArraySort(pQueryHandle->pTableCheckInfo, tsdbCheckInfoCompar); pQueryHandle->defaultLoadColumn = getDefaultLoadColumns(pQueryHandle, true); tsdbDebug("%p total numOfTable:%zu in query, %p", pQueryHandle, taosArrayGetSize(pQueryHandle->pTableCheckInfo), pQueryHandle->qinfo); @@ -2431,3 +2433,13 @@ void tsdbDestroyTableGroup(STableGroupInfo *pGroupList) { taosArrayDestroy(pGroupList->pGroupList); } +static int tsdbCheckInfoCompar(const void* key1, const void* key2) { + if (((STableCheckInfo*)key1)->tableId.tid < ((STableCheckInfo*)key2)->tableId.tid) { + return -1; + } else if (((STableCheckInfo*)key1)->tableId.tid > ((STableCheckInfo*)key2)->tableId.tid) { + return 1; + } else { + ASSERT(false); + return 0; + } +} \ No newline at end of file From bfe15fe0927b975f1cdf0726ec7649336f73e9bf Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 21 Jul 2020 14:18:06 +0800 Subject: [PATCH 051/100] increase crash_gen step to 2000. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index f2cd83343e..6a4acce451 100644 --- a/.travis.yml +++ b/.travis.yml @@ -63,7 +63,7 @@ matrix: pkill -TERM -x taosd fuser -k -n tcp 6030 sleep 1 - ./crash_gen.sh -a -p -t 4 -s 1000|| travis_terminate $? + ./crash_gen.sh -a -p -t 4 -s 2000|| travis_terminate $? sleep 1 cd ${TRAVIS_BUILD_DIR}/tests/pytest From 51797a45d5ecb05943935b025e5b3570c2439921 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 21 Jul 2020 15:39:58 +0800 Subject: [PATCH 052/100] fix invalid write --- src/tsdb/src/tsdbRWHelper.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index cec3b0d36b..9d0efd0de2 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -530,7 +530,7 @@ int tsdbLoadCompIdx(SRWHelper *pHelper, void *target) { size_t tlen = tsizeof(pHelper->idxH.pIdxArray); pHelper->idxH.numOfIdx++; - if (tlen < pHelper->idxH.numOfIdx) { + if (tlen < pHelper->idxH.numOfIdx * sizoef(SCompIdx)) { pHelper->idxH.pIdxArray = (SCompIdx *)trealloc(pHelper->idxH.pIdxArray, (tlen == 0) ? 1024 : tlen * 2); if (pHelper->idxH.pIdxArray == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; From 8a3616898f1836d06b7330a3bb06d2661535fc21 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 21 Jul 2020 15:45:04 +0800 Subject: [PATCH 053/100] fix typo --- src/tsdb/src/tsdbRWHelper.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index 9d0efd0de2..f8792469cb 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -530,7 +530,7 @@ int tsdbLoadCompIdx(SRWHelper *pHelper, void *target) { size_t tlen = tsizeof(pHelper->idxH.pIdxArray); pHelper->idxH.numOfIdx++; - if (tlen < pHelper->idxH.numOfIdx * sizoef(SCompIdx)) { + if (tlen < pHelper->idxH.numOfIdx * sizeof(SCompIdx)) { pHelper->idxH.pIdxArray = (SCompIdx *)trealloc(pHelper->idxH.pIdxArray, (tlen == 0) ? 1024 : tlen * 2); if (pHelper->idxH.pIdxArray == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; From 43b2f41fbbcbf7ee8c1620354dd7ff3441ac3caa Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Tue, 21 Jul 2020 16:24:59 +0800 Subject: [PATCH 054/100] TD-819: increase test coverage --- tests/pytest/fulltest.sh | 20 +++ tests/pytest/functions/function_avg.py | 73 +++++++++ tests/pytest/functions/function_bottom.py | 98 ++++++++++++ tests/pytest/functions/function_count.py | 84 +++++++++++ tests/pytest/functions/function_diff.py | 103 +++++++++++++ tests/pytest/functions/function_first.py | 124 ++++++++++++++++ tests/pytest/functions/function_last.py | 124 ++++++++++++++++ tests/pytest/functions/function_last_row.py | 133 +++++++++++++++++ .../pytest/functions/function_leastsquares.py | 82 ++++++++++ tests/pytest/functions/function_max.py | 78 ++++++++++ tests/pytest/functions/function_min.py | 78 ++++++++++ tests/pytest/functions/function_operations.py | 86 +++++++++++ tests/pytest/functions/function_percentile.py | 140 ++++++++++++++++++ tests/pytest/functions/function_spread.py | 111 ++++++++++++++ tests/pytest/functions/function_stddev.py | 80 ++++++++++ tests/pytest/functions/function_sum.py | 70 +++++++++ tests/pytest/functions/function_top.py | 98 ++++++++++++ tests/pytest/functions/function_twa.py | 135 +++++++++++++++++ tests/pytest/query/filterOtherTypes.py | 24 ++- tests/pytest/regressiontest.sh | 21 +++ tests/pytest/util/sql.py | 20 ++- 21 files changed, 1768 insertions(+), 14 deletions(-) create mode 100644 tests/pytest/functions/function_avg.py create mode 100644 tests/pytest/functions/function_bottom.py create mode 100644 tests/pytest/functions/function_count.py create mode 100644 tests/pytest/functions/function_diff.py create mode 100644 tests/pytest/functions/function_first.py create mode 100644 tests/pytest/functions/function_last.py create mode 100644 tests/pytest/functions/function_last_row.py create mode 100644 tests/pytest/functions/function_leastsquares.py create mode 100644 tests/pytest/functions/function_max.py create mode 100644 tests/pytest/functions/function_min.py create mode 100644 tests/pytest/functions/function_operations.py create mode 100644 tests/pytest/functions/function_percentile.py create mode 100644 tests/pytest/functions/function_spread.py create mode 100644 tests/pytest/functions/function_stddev.py create mode 100644 tests/pytest/functions/function_sum.py create mode 100644 tests/pytest/functions/function_top.py create mode 100644 tests/pytest/functions/function_twa.py diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index 9d1aef0dc5..904e712fa4 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -144,6 +144,7 @@ python3 ./test.py -f query/querySort.py python3 ./test.py -f query/queryJoin.py python3 ./test.py -f query/select_last_crash.py python3 ./test.py -f query/queryNullValueTest.py +python3 ./test.py -f query/queryInsertValue.py #stream python3 ./test.py -f stream/metric_1.py @@ -161,3 +162,22 @@ python3 ./test.py -f client/client.py # Misc python3 testCompress.py python3 testNoCompress.py + +# functions +python3 ./test.py -f functions/function_avg.py +python3 ./test.py -f functions/function_bottom.py +python3 ./test.py -f functions/function_count.py +python3 ./test.py -f functions/function_diff.py +python3 ./test.py -f functions/function_first.py +python3 ./test.py -f functions/function_last.py +python3 ./test.py -f functions/function_last_row.py +python3 ./test.py -f functions/function_leastsquares.py +python3 ./test.py -f functions/function_max.py +python3 ./test.py -f functions/function_min.py +python3 ./test.py -f functions/function_operations.py +python3 ./test.py -f functions/function_percentile.py +python3 ./test.py -f functions/function_spread.py +python3 ./test.py -f functions/function_stddev.py +python3 ./test.py -f functions/function_sum.py +python3 ./test.py -f functions/function_top.py +python3 ./test.py -f functions/function_twa.py \ No newline at end of file diff --git a/tests/pytest/functions/function_avg.py b/tests/pytest/functions/function_avg.py new file mode 100644 index 0000000000..9481550ba3 --- /dev/null +++ b/tests/pytest/functions/function_avg.py @@ -0,0 +1,73 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +import numpy as np + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.rowNum = 10 + self.ts = 1537146000000 + + def run(self): + tdSql.prepare() + + intData = [] + floatData = [] + + tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''') + tdSql.execute("create table test1 using test tags('beijing')") + for i in range(self.rowNum): + tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) + intData.append(i + 1) + floatData.append(i + 0.1) + + # average verifacation + tdSql.error("select avg(ts) from test") + tdSql.error("select avg(ts) from test1") + tdSql.error("select avg(col7) from test") + tdSql.error("select avg(col7) from test1") + tdSql.error("select avg(col8) from test") + tdSql.error("select avg(col8) from test1") + tdSql.error("select avg(col9) from test") + tdSql.error("select avg(col9) from test1") + + tdSql.query("select avg(col1) from test") + tdSql.checkData(0, 0, np.average(intData)) + tdSql.query("select avg(col2) from test") + tdSql.checkData(0, 0, np.average(intData)) + tdSql.query("select avg(col3) from test") + tdSql.checkData(0, 0, np.average(intData)) + tdSql.query("select avg(col4) from test") + tdSql.checkData(0, 0, np.average(intData)) + tdSql.query("select avg(col5) from test") + tdSql.checkData(0, 0, np.average(floatData)) + tdSql.query("select avg(col6) from test") + tdSql.checkData(0, 0, np.average(floatData)) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/functions/function_bottom.py b/tests/pytest/functions/function_bottom.py new file mode 100644 index 0000000000..cb008d522a --- /dev/null +++ b/tests/pytest/functions/function_bottom.py @@ -0,0 +1,98 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +import numpy as np + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.rowNum = 10 + self.ts = 1537146000000 + + def run(self): + tdSql.prepare() + + intData = [] + floatData = [] + + tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''') + tdSql.execute("create table test1 using test tags('beijing')") + for i in range(self.rowNum): + tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) + intData.append(i + 1) + floatData.append(i + 0.1) + + # bottom verifacation + tdSql.error("select bottom(ts, 10) from test") + tdSql.error("select bottom(col1, 0) from test") + tdSql.error("select bottom(col1, 101) from test") + tdSql.error("select bottom(col2, 0) from test") + tdSql.error("select bottom(col2, 101) from test") + tdSql.error("select bottom(col3, 0) from test") + tdSql.error("select bottom(col3, 101) from test") + tdSql.error("select bottom(col4, 0) from test") + tdSql.error("select bottom(col4, 101) from test") + tdSql.error("select bottom(col5, 0) from test") + tdSql.error("select bottom(col5, 101) from test") + tdSql.error("select bottom(col6, 0) from test") + tdSql.error("select bottom(col6, 101) from test") + tdSql.error("select bottom(col7, 10) from test") + tdSql.error("select bottom(col8, 10) from test") + tdSql.error("select bottom(col9, 10) from test") + + tdSql.query("select bottom(col1, 2) from test") + tdSql.checkRows(2) + tdSql.checkData(0, 1, 1) + tdSql.checkData(1, 1, 2) + + tdSql.query("select bottom(col2, 2) from test") + tdSql.checkRows(2) + tdSql.checkData(0, 1, 1) + tdSql.checkData(1, 1, 2) + + tdSql.query("select bottom(col3, 2) from test") + tdSql.checkRows(2) + tdSql.checkData(0, 1, 1) + tdSql.checkData(1, 1, 2) + + tdSql.query("select bottom(col4, 2) from test") + tdSql.checkRows(2) + tdSql.checkData(0, 1, 1) + tdSql.checkData(1, 1, 2) + + tdSql.query("select bottom(col5, 2) from test") + tdSql.checkRows(2) + tdSql.checkData(0, 1, 0.1) + tdSql.checkData(1, 1, 1.1) + + tdSql.query("select bottom(col6, 2) from test") + tdSql.checkRows(2) + tdSql.checkData(0, 1, 0.1) + tdSql.checkData(1, 1, 1.1) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/functions/function_count.py b/tests/pytest/functions/function_count.py new file mode 100644 index 0000000000..e5ef71bf5a --- /dev/null +++ b/tests/pytest/functions/function_count.py @@ -0,0 +1,84 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +import numpy as np + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.rowNum = 10 + self.ts = 1537146000000 + + def run(self): + tdSql.prepare() + + intData = [] + floatData = [] + + tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''') + tdSql.execute("create table test1 using test tags('beijing')") + for i in range(self.rowNum): + tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) + intData.append(i + 1) + floatData.append(i + 0.1) + + # Count verifacation + tdSql.query("select count(*) from test") + tdSql.checkData(0, 0, 10) + + tdSql.query("select count(ts) from test") + tdSql.checkData(0, 0, 10) + tdSql.query("select count(col1) from test") + tdSql.checkData(0, 0, 10) + tdSql.query("select count(col2) from test") + tdSql.checkData(0, 0, 10) + tdSql.query("select count(col3) from test") + tdSql.checkData(0, 0, 10) + tdSql.query("select count(col4) from test") + tdSql.checkData(0, 0, 10) + tdSql.query("select count(col5) from test") + tdSql.checkData(0, 0, 10) + tdSql.query("select count(col6) from test") + tdSql.checkData(0, 0, 10) + tdSql.query("select count(col7) from test") + tdSql.checkData(0, 0, 10) + tdSql.query("select count(col8) from test") + tdSql.checkData(0, 0, 10) + tdSql.query("select count(col9) from test") + tdSql.checkData(0, 0, 10) + + tdSql.execute("alter table test add column col10 int") + tdSql.query("select count(col10) from test") + tdSql.checkRows(0) + + tdSql.execute("insert into test1 values(now, 1, 2, 3, 4, 1.1, 2.2, false, 'test', 'test' 1)") + tdSql.query("select count(col10) from test") + tdSql.checkData(0, 0, 1) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/functions/function_diff.py b/tests/pytest/functions/function_diff.py new file mode 100644 index 0000000000..62daa5fa82 --- /dev/null +++ b/tests/pytest/functions/function_diff.py @@ -0,0 +1,103 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +import numpy as np + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.rowNum = 10 + self.ts = 1537146000000 + + def run(self): + tdSql.prepare() + + intData = [] + floatData = [] + + tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''') + tdSql.execute("create table test1 using test tags('beijing')") + tdSql.execute("insert into test1 values(%d, 0, 0, 0, 0, 0.0, 0.0, False, ' ', ' ')" % (self.ts - 1)) + tdSql.query("select diff(col1) from test1") + tdSql.checkRows(0) + + tdSql.query("select diff(col2) from test1") + tdSql.checkRows(0) + + tdSql.query("select diff(col3) from test1") + tdSql.checkRows(0) + + tdSql.query("select diff(col4) from test1") + tdSql.checkRows(0) + + tdSql.query("select diff(col5) from test1") + tdSql.checkRows(0) + + tdSql.query("select diff(col6) from test1") + tdSql.checkRows(0) + + for i in range(self.rowNum): + tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) + intData.append(i + 1) + floatData.append(i + 0.1) + + # diff verifacation + tdSql.error("select diff(ts) from test") + tdSql.error("select diff(ts) from test1") + tdSql.error("select diff(col1) from test") + tdSql.error("select diff(col2) from test") + tdSql.error("select diff(col3) from test") + tdSql.error("select diff(col4) from test") + tdSql.error("select diff(col5) from test") + tdSql.error("select diff(col6) from test") + tdSql.error("select diff(col7) from test") + tdSql.error("select diff(col7) from test1") + tdSql.error("select diff(col8) from test") + tdSql.error("select diff(col8) from test1") + tdSql.error("select diff(col9) from test") + tdSql.error("select diff(col9) from test1") + + tdSql.query("select diff(col1) from test1") + tdSql.checkRows(10) + + tdSql.query("select diff(col2) from test1") + tdSql.checkRows(10) + + tdSql.query("select diff(col3) from test1") + tdSql.checkRows(10) + + tdSql.query("select diff(col4) from test1") + tdSql.checkRows(10) + + tdSql.query("select diff(col5) from test1") + tdSql.checkRows(10) + + tdSql.query("select diff(col6) from test1") + tdSql.checkRows(10) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/functions/function_first.py b/tests/pytest/functions/function_first.py new file mode 100644 index 0000000000..27ce09b309 --- /dev/null +++ b/tests/pytest/functions/function_first.py @@ -0,0 +1,124 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +import numpy as np + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.rowNum = 10 + self.ts = 1537146000000 + + def run(self): + tdSql.prepare() + + intData = [] + floatData = [] + + tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''') + tdSql.execute("create table test1 using test tags('beijing')") + tdSql.execute("insert into test1(ts) values(%d)" % (self.ts - 1)) + + # first verifacation + tdSql.query("select first(*) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 1, None) + + tdSql.query("select first(col1) from test1") + tdSql.checkRows(0) + + tdSql.query("select first(col2) from test1") + tdSql.checkRows(0) + + tdSql.query("select first(col3) from test1") + tdSql.checkRows(0) + + tdSql.query("select first(col4) from test1") + tdSql.checkRows(0) + + tdSql.query("select first(col5) from test1") + tdSql.checkRows(0) + + tdSql.query("select first(col6) from test1") + tdSql.checkRows(0) + + tdSql.query("select first(col7) from test1") + tdSql.checkRows(0) + + tdSql.query("select first(col8) from test1") + tdSql.checkRows(0) + + tdSql.query("select first(col9) from test1") + tdSql.checkRows(0) + + for i in range(self.rowNum): + tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) + intData.append(i + 1) + floatData.append(i + 0.1) + + tdSql.query("select first(*) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 1, 1) + + tdSql.query("select first(col1) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 1) + + tdSql.query("select first(col2) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 1) + + tdSql.query("select first(col3) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 1) + + tdSql.query("select first(col4) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 1) + + tdSql.query("select first(col5) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 0.1) + + tdSql.query("select first(col6) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 0.1) + + tdSql.query("select first(col7) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, False) + + tdSql.query("select first(col8) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 'taosdata1') + + tdSql.query("select first(col9) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '涛思数据1') + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/functions/function_last.py b/tests/pytest/functions/function_last.py new file mode 100644 index 0000000000..6412b7ce6a --- /dev/null +++ b/tests/pytest/functions/function_last.py @@ -0,0 +1,124 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +import numpy as np + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.rowNum = 10 + self.ts = 1537146000000 + + def run(self): + tdSql.prepare() + + intData = [] + floatData = [] + + tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''') + tdSql.execute("create table test1 using test tags('beijing')") + tdSql.execute("insert into test1(ts) values(%d)" % (self.ts - 1)) + + # first verifacation + tdSql.query("select last(*) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 1, None) + + tdSql.query("select last(col1) from test1") + tdSql.checkRows(0) + + tdSql.query("select last(col2) from test1") + tdSql.checkRows(0) + + tdSql.query("select last(col3) from test1") + tdSql.checkRows(0) + + tdSql.query("select last(col4) from test1") + tdSql.checkRows(0) + + tdSql.query("select last(col5) from test1") + tdSql.checkRows(0) + + tdSql.query("select last(col6) from test1") + tdSql.checkRows(0) + + tdSql.query("select last(col7) from test1") + tdSql.checkRows(0) + + tdSql.query("select last(col8) from test1") + tdSql.checkRows(0) + + tdSql.query("select last(col9) from test1") + tdSql.checkRows(0) + + for i in range(self.rowNum): + tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) + intData.append(i + 1) + floatData.append(i + 0.1) + + tdSql.query("select last(*) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 1, 10) + + tdSql.query("select last(col1) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + + tdSql.query("select last(col2) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + + tdSql.query("select last(col3) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + + tdSql.query("select last(col4) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + + tdSql.query("select last(col5) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 9.1) + + tdSql.query("select last(col6) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 9.1) + + tdSql.query("select last(col7) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, True) + + tdSql.query("select last(col8) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 'taosdata10') + + tdSql.query("select last(col9) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '涛思数据10') + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/functions/function_last_row.py b/tests/pytest/functions/function_last_row.py new file mode 100644 index 0000000000..19db865a3d --- /dev/null +++ b/tests/pytest/functions/function_last_row.py @@ -0,0 +1,133 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +import numpy as np + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.rowNum = 10 + self.ts = 1537146000000 + + def run(self): + tdSql.prepare() + + intData = [] + floatData = [] + + tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''') + tdSql.execute("create table test1 using test tags('beijing')") + tdSql.execute("insert into test1(ts) values(%d)" % (self.ts - 1)) + + # first verifacation + tdSql.query("select last_row(*) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 1, None) + + tdSql.query("select last_row(col1) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, None) + + tdSql.query("select last_row(col2) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, None) + + tdSql.query("select last_row(col3) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, None) + + tdSql.query("select last_row(col4) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, None) + + tdSql.query("select last_row(col5) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, None) + + tdSql.query("select last_row(col6) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, None) + + tdSql.query("select last_row(col7) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, None) + + tdSql.query("select last_row(col8) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, None) + + tdSql.query("select last_row(col9) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, None) + + for i in range(self.rowNum): + tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) + intData.append(i + 1) + floatData.append(i + 0.1) + + tdSql.query("select last_row(*) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 1, 10) + + tdSql.query("select last_row(col1) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + + tdSql.query("select last_row(col2) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + + tdSql.query("select last_row(col3) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + + tdSql.query("select last_row(col4) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + + tdSql.query("select last_row(col5) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 9.1) + + tdSql.query("select last_row(col6) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 9.1) + + tdSql.query("select last_row(col7) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, True) + + tdSql.query("select last_row(col8) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 'taosdata10') + + tdSql.query("select last_row(col9) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, '涛思数据10') + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/functions/function_leastsquares.py b/tests/pytest/functions/function_leastsquares.py new file mode 100644 index 0000000000..66187926f5 --- /dev/null +++ b/tests/pytest/functions/function_leastsquares.py @@ -0,0 +1,82 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +import numpy as np + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.rowNum = 10 + self.ts = 1537146000000 + + def run(self): + tdSql.prepare() + + intData = [] + floatData = [] + + tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''') + tdSql.execute("create table test1 using test tags('beijing')") + for i in range(self.rowNum): + tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) + intData.append(i + 1) + floatData.append(i + 0.1) + + # leastsquares verifacation + tdSql.error("select leastsquares(ts, 1, 1) from test1") + tdSql.error("select leastsquares(col1, 1, 1) from test") + tdSql.error("select leastsquares(col2, 1, 1) from test") + tdSql.error("select leastsquares(col3, 1, 1) from test") + tdSql.error("select leastsquares(col4, 1, 1) from test") + tdSql.error("select leastsquares(col5, 1, 1) from test") + tdSql.error("select leastsquares(col6, 1, 1) from test") + tdSql.error("select leastsquares(col7, 1, 1) from test1") + tdSql.error("select leastsquares(col8, 1, 1) from test1") + tdSql.error("select leastsquares(col9, 1, 1) from test1") + + tdSql.query("select leastsquares(col1, 1, 1) from test1") + tdSql.checkData(0, 0, '{slop:1.000000, intercept:0.000000}') + + tdSql.query("select leastsquares(col2, 1, 1) from test1") + tdSql.checkData(0, 0, '{slop:1.000000, intercept:0.000000}') + + tdSql.query("select leastsquares(col3, 1, 1) from test1") + tdSql.checkData(0, 0, '{slop:1.000000, intercept:0.000000}') + + tdSql.query("select leastsquares(col4, 1, 1) from test1") + tdSql.checkData(0, 0, '{slop:1.000000, intercept:0.000000}') + + tdSql.query("select leastsquares(col5, 1, 1) from test1") + tdSql.checkData(0, 0, '{slop:1.000000, intercept:-0.900000}') + + tdSql.query("select leastsquares(col6, 1, 1) from test1") + tdSql.checkData(0, 0, '{slop:1.000000, intercept:-0.900000}') + + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/functions/function_max.py b/tests/pytest/functions/function_max.py new file mode 100644 index 0000000000..02364e0fde --- /dev/null +++ b/tests/pytest/functions/function_max.py @@ -0,0 +1,78 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +import numpy as np + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.rowNum = 10 + self.ts = 1537146000000 + + def run(self): + tdSql.prepare() + + intData = [] + floatData = [] + + tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''') + tdSql.execute("create table test1 using test tags('beijing')") + for i in range(self.rowNum): + tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) + intData.append(i + 1) + floatData.append(i + 0.1) + + # min verifacation + tdSql.error("select max(ts) from test") + tdSql.error("select max(ts) from test1") + tdSql.error("select max(col7) from test") + tdSql.error("select max(col7) from test1") + tdSql.error("select max(col8) from test") + tdSql.error("select max(col8) from test1") + tdSql.error("select max(col9) from test") + tdSql.error("select max(col9) from test1") + + tdSql.query("select max(col1) from test1") + tdSql.checkData(0, 0, np.max(intData)) + + tdSql.query("select max(col2) from test1") + tdSql.checkData(0, 0, np.max(intData)) + + tdSql.query("select max(col3) from test1") + tdSql.checkData(0, 0, np.max(intData)) + + tdSql.query("select max(col4) from test1") + tdSql.checkData(0, 0, np.max(intData)) + + tdSql.query("select max(col5) from test1") + tdSql.checkData(0, 0, np.max(floatData)) + + tdSql.query("select max(col6) from test1") + tdSql.checkData(0, 0, np.max(floatData)) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/functions/function_min.py b/tests/pytest/functions/function_min.py new file mode 100644 index 0000000000..bc180bc224 --- /dev/null +++ b/tests/pytest/functions/function_min.py @@ -0,0 +1,78 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +import numpy as np + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.rowNum = 10 + self.ts = 1537146000000 + + def run(self): + tdSql.prepare() + + intData = [] + floatData = [] + + tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''') + tdSql.execute("create table test1 using test tags('beijing')") + for i in range(self.rowNum): + tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) + intData.append(i + 1) + floatData.append(i + 0.1) + + # min verifacation + tdSql.error("select min(ts) from test") + tdSql.error("select min(ts) from test1") + tdSql.error("select min(col7) from test") + tdSql.error("select min(col7) from test1") + tdSql.error("select min(col8) from test") + tdSql.error("select min(col8) from test1") + tdSql.error("select min(col9) from test") + tdSql.error("select min(col9) from test1") + + tdSql.query("select min(col1) from test1") + tdSql.checkData(0, 0, np.min(intData)) + + tdSql.query("select min(col2) from test1") + tdSql.checkData(0, 0, np.min(intData)) + + tdSql.query("select min(col3) from test1") + tdSql.checkData(0, 0, np.min(intData)) + + tdSql.query("select min(col4) from test1") + tdSql.checkData(0, 0, np.min(intData)) + + tdSql.query("select min(col5) from test1") + tdSql.checkData(0, 0, np.min(floatData)) + + tdSql.query("select min(col6) from test1") + tdSql.checkData(0, 0, np.min(floatData)) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/functions/function_operations.py b/tests/pytest/functions/function_operations.py new file mode 100644 index 0000000000..ec8da92c95 --- /dev/null +++ b/tests/pytest/functions/function_operations.py @@ -0,0 +1,86 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +import numpy as np + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.rowNum = 10 + self.ts = 1537146000000 + + def run(self): + tdSql.prepare() + + intData = [] + floatData = [] + + tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''') + tdSql.execute("create table test1 using test tags('beijing')") + for i in range(self.rowNum): + tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) + intData.append(i + 1) + floatData.append(i + 0.1) + + # min verifacation + tdSql.error("select ts + col1 from test") + tdSql.error("select ts + col1 from test1") + tdSql.error("select col1 + col7 from test") + tdSql.error("select col1 + col7 from test1") + tdSql.error("select col1 + col8 from test") + tdSql.error("select col1 + col8 from test1") + tdSql.error("select col1 + col9 from test") + tdSql.error("select col1 + col9 from test1") + + tdSql.query("select col1 + col2 from test1") + tdSql.checkRows(10) + tdSql.checkData(0, 0, 2.0) + + tdSql.query("select col1 + col2 * col3 from test1") + tdSql.checkRows(10) + tdSql.checkData(1, 0, 6.0) + + tdSql.query("select col1 + col2 * col3 + col3 / col4 + col5 + col6 from test1") + tdSql.checkRows(10) + tdSql.checkData(0, 0, 3.2) + + tdSql.execute("insert into test1(ts, col1) values(%d, 11)" % (self.ts + 11)) + tdSql.query("select col1 + col2 from test1") + tdSql.checkRows(11) + tdSql.checkData(10, 0, None) + + tdSql.query("select col1 + col2 * col3 from test1") + tdSql.checkRows(11) + tdSql.checkData(10, 0, None) + + tdSql.query("select col1 + col2 * col3 + col3 / col4 + col5 + col6 from test1") + tdSql.checkRows(11) + tdSql.checkData(10, 0, None) + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/functions/function_percentile.py b/tests/pytest/functions/function_percentile.py new file mode 100644 index 0000000000..aaeb94372e --- /dev/null +++ b/tests/pytest/functions/function_percentile.py @@ -0,0 +1,140 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +import numpy as np + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.rowNum = 10 + self.ts = 1537146000000 + + def run(self): + tdSql.prepare() + + intData = [] + floatData = [] + + tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + col7 bool, col8 binary(20), col9 nchar(20))''') + for i in range(self.rowNum): + tdSql.execute("insert into test values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) + intData.append(i + 1) + floatData.append(i + 0.1) + + # percentile verifacation + tdSql.error("select percentile(ts 20) from test") + tdSql.error("select apercentile(ts 20) from test") + tdSql.error("select percentile(col7 20) from test") + tdSql.error("select apercentile(col7 20) from test") + tdSql.error("select percentile(col8 20) from test") + tdSql.error("select apercentile(col8 20) from test") + tdSql.error("select percentile(col9 20) from test") + tdSql.error("select apercentile(col9 20) from test") + + tdSql.query("select percentile(col1, 0) from test") + tdSql.checkData(0, 0, np.percentile(intData, 0)) + tdSql.query("select apercentile(col1, 0) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + tdSql.query("select percentile(col1, 50) from test") + tdSql.checkData(0, 0, np.percentile(intData, 50)) + tdSql.query("select apercentile(col1, 50) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + tdSql.query("select percentile(col1, 100) from test") + tdSql.checkData(0, 0, np.percentile(intData, 100)) + tdSql.query("select apercentile(col1, 100) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + + tdSql.query("select percentile(col2, 0) from test") + tdSql.checkData(0, 0, np.percentile(intData, 0)) + tdSql.query("select apercentile(col2, 0) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + tdSql.query("select percentile(col2, 50) from test") + tdSql.checkData(0, 0, np.percentile(intData, 50)) + tdSql.query("select apercentile(col2, 50) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + tdSql.query("select percentile(col2, 100) from test") + tdSql.checkData(0, 0, np.percentile(intData, 100)) + tdSql.query("select apercentile(col2, 100) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + + tdSql.query("select percentile(col3, 0) from test") + tdSql.checkData(0, 0, np.percentile(intData, 0)) + tdSql.query("select apercentile(col3, 0) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + tdSql.query("select percentile(col3, 50) from test") + tdSql.checkData(0, 0, np.percentile(intData, 50)) + tdSql.query("select apercentile(col3, 50) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + tdSql.query("select percentile(col3, 100) from test") + tdSql.checkData(0, 0, np.percentile(intData, 100)) + tdSql.query("select apercentile(col3, 100) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + + tdSql.query("select percentile(col4, 0) from test") + tdSql.checkData(0, 0, np.percentile(intData, 0)) + tdSql.query("select apercentile(col4, 0) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + tdSql.query("select percentile(col4, 50) from test") + tdSql.checkData(0, 0, np.percentile(intData, 50)) + tdSql.query("select apercentile(col4, 50) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + tdSql.query("select percentile(col4, 100) from test") + tdSql.checkData(0, 0, np.percentile(intData, 100)) + tdSql.query("select apercentile(col4, 100) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + + tdSql.query("select percentile(col5, 0) from test") + print("query result: %s" % tdSql.getData(0, 0)) + print("array result: %s" % np.percentile(floatData, 0)) + tdSql.query("select apercentile(col5, 0) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + tdSql.query("select percentile(col5, 50) from test") + print("query result: %s" % tdSql.getData(0, 0)) + print("array result: %s" % np.percentile(floatData, 50)) + tdSql.query("select apercentile(col5, 50) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + tdSql.query("select percentile(col5, 100) from test") + print("query result: %s" % tdSql.getData(0, 0)) + print("array result: %s" % np.percentile(floatData, 100)) + tdSql.query("select apercentile(col5, 100) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + + tdSql.query("select percentile(col6, 0) from test") + tdSql.checkData(0, 0, np.percentile(floatData, 0)) + tdSql.query("select apercentile(col6, 0) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + tdSql.query("select percentile(col6, 50) from test") + tdSql.checkData(0, 0, np.percentile(floatData, 50)) + tdSql.query("select apercentile(col6, 50) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + tdSql.query("select percentile(col6, 100) from test") + tdSql.checkData(0, 0, np.percentile(floatData, 100)) + tdSql.query("select apercentile(col6, 100) from test") + print("apercentile result: %s" % tdSql.getData(0, 0)) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/functions/function_spread.py b/tests/pytest/functions/function_spread.py new file mode 100644 index 0000000000..62b604a279 --- /dev/null +++ b/tests/pytest/functions/function_spread.py @@ -0,0 +1,111 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +import numpy as np + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.rowNum = 10 + self.ts = 1537146000000 + + def run(self): + tdSql.prepare() + + intData = [] + floatData = [] + + tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''') + tdSql.execute("create table test1 using test tags('beijing')") + tdSql.execute("insert into test1 values(%d, 0, 0, 0, 0, 0.0, 0.0, False, ' ', ' ')" % (self.ts - 1)) + + # spread verifacation + tdSql.query("select spread(ts) from test1") + tdSql.checkRows(1) + + tdSql.query("select spread(col1) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 0) + + tdSql.query("select spread(col2) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 0) + + tdSql.query("select spread(col3) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 0) + + tdSql.query("select spread(col4) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 0) + + tdSql.query("select spread(col5) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 0) + + tdSql.query("select spread(col6) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 0) + + for i in range(self.rowNum): + tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) + intData.append(i + 1) + floatData.append(i + 0.1) + + tdSql.error("select spread(col7) from test") + tdSql.error("select spread(col7) from test1") + tdSql.error("select spread(col8) from test") + tdSql.error("select spread(col8) from test1") + tdSql.error("select spread(col9) from test") + tdSql.error("select spread(col9) from test1") + + tdSql.query("select spread(col1) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + + tdSql.query("select spread(col2) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + + tdSql.query("select spread(col3) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + + tdSql.query("select spread(col4) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 10) + + tdSql.query("select spread(col5) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 9.1) + + tdSql.query("select spread(col6) from test1") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 9.1) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/functions/function_stddev.py b/tests/pytest/functions/function_stddev.py new file mode 100644 index 0000000000..23df415aa3 --- /dev/null +++ b/tests/pytest/functions/function_stddev.py @@ -0,0 +1,80 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +import numpy as np + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.rowNum = 10 + self.ts = 1537146000000 + + def run(self): + tdSql.prepare() + + intData = [] + floatData = [] + + tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''') + tdSql.execute("create table test1 using test tags('beijing')") + for i in range(self.rowNum): + tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) + intData.append(i + 1) + floatData.append(i + 0.1) + + # stddev verifacation + tdSql.error("select stddev(ts) from test1") + tdSql.error("select stddev(col1) from test") + tdSql.error("select stddev(col2) from test") + tdSql.error("select stddev(col3) from test") + tdSql.error("select stddev(col4) from test") + tdSql.error("select stddev(col5) from test") + tdSql.error("select stddev(col6) from test") + tdSql.error("select stddev(col7) from test1") + tdSql.error("select stddev(col8) from test1") + tdSql.error("select stddev(col9) from test1") + + tdSql.query("select stddev(col1) from test1") + tdSql.checkData(0, 0, np.std(intData)) + + tdSql.query("select stddev(col2) from test1") + tdSql.checkData(0, 0, np.std(intData)) + + tdSql.query("select stddev(col3) from test1") + tdSql.checkData(0, 0, np.std(intData)) + + tdSql.query("select stddev(col4) from test1") + tdSql.checkData(0, 0, np.std(intData)) + + tdSql.query("select stddev(col5) from test1") + tdSql.checkData(0, 0, np.std(floatData)) + + tdSql.query("select stddev(col6) from test1") + tdSql.checkData(0, 0, np.std(floatData)) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/functions/function_sum.py b/tests/pytest/functions/function_sum.py new file mode 100644 index 0000000000..8e4e289c7e --- /dev/null +++ b/tests/pytest/functions/function_sum.py @@ -0,0 +1,70 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +import numpy as np + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.rowNum = 10 + self.ts = 1537146000000 + + def run(self): + tdSql.prepare() + + intData = [] + floatData = [] + + tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''') + tdSql.execute("create table test1 using test tags('beijing')") + for i in range(self.rowNum): + tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) + intData.append(i + 1) + floatData.append(i + 0.1) + + # sum verifacation + tdSql.error("select sum(ts) from test") + tdSql.error("select sum(col7) from test") + tdSql.error("select sum(col8) from test") + tdSql.error("select sum(col9) from test") + + tdSql.query("select sum(col1) from test") + tdSql.checkData(0, 0, np.sum(intData)) + tdSql.query("select sum(col2) from test") + tdSql.checkData(0, 0, np.sum(intData)) + tdSql.query("select sum(col3) from test") + tdSql.checkData(0, 0, np.sum(intData)) + tdSql.query("select sum(col4) from test") + tdSql.checkData(0, 0, np.sum(intData)) + tdSql.query("select sum(col5) from test") + print("query result for folat sum function: %f" % tdSql.getData(0, 0)) + print("calculation result for numpy sum function: %f" % np.sum(floatData)) + tdSql.query("select sum(col6) from test") + tdSql.checkData(0, 0, np.sum(floatData)) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/functions/function_top.py b/tests/pytest/functions/function_top.py new file mode 100644 index 0000000000..e24ff1cc53 --- /dev/null +++ b/tests/pytest/functions/function_top.py @@ -0,0 +1,98 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +import numpy as np + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.rowNum = 10 + self.ts = 1537146000000 + + def run(self): + tdSql.prepare() + + intData = [] + floatData = [] + + tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''') + tdSql.execute("create table test1 using test tags('beijing')") + for i in range(self.rowNum): + tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) + intData.append(i + 1) + floatData.append(i + 0.1) + + # top verifacation + tdSql.error("select top(ts, 10) from test") + tdSql.error("select top(col1, 0) from test") + tdSql.error("select top(col1, 101) from test") + tdSql.error("select top(col2, 0) from test") + tdSql.error("select top(col2, 101) from test") + tdSql.error("select top(col3, 0) from test") + tdSql.error("select top(col3, 101) from test") + tdSql.error("select top(col4, 0) from test") + tdSql.error("select top(col4, 101) from test") + tdSql.error("select top(col5, 0) from test") + tdSql.error("select top(col5, 101) from test") + tdSql.error("select top(col6, 0) from test") + tdSql.error("select top(col6, 101) from test") + tdSql.error("select top(col7, 10) from test") + tdSql.error("select top(col8, 10) from test") + tdSql.error("select top(col9, 10) from test") + + tdSql.query("select top(col1, 2) from test") + tdSql.checkRows(2) + tdSql.checkData(0, 1, 9) + tdSql.checkData(1, 1, 10) + + tdSql.query("select top(col2, 2) from test") + tdSql.checkRows(2) + tdSql.checkData(0, 1, 9) + tdSql.checkData(1, 1, 10) + + tdSql.query("select top(col3, 2) from test") + tdSql.checkRows(2) + tdSql.checkData(0, 1, 9) + tdSql.checkData(1, 1, 10) + + tdSql.query("select top(col4, 2) from test") + tdSql.checkRows(2) + tdSql.checkData(0, 1, 9) + tdSql.checkData(1, 1, 10) + + tdSql.query("select top(col5, 2) from test") + tdSql.checkRows(2) + tdSql.checkData(0, 1, 8.1) + tdSql.checkData(1, 1, 9.1) + + tdSql.query("select top(col6, 2) from test") + tdSql.checkRows(2) + tdSql.checkData(0, 1, 8.1) + tdSql.checkData(1, 1, 9.1) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/functions/function_twa.py b/tests/pytest/functions/function_twa.py new file mode 100644 index 0000000000..1ce4c99b60 --- /dev/null +++ b/tests/pytest/functions/function_twa.py @@ -0,0 +1,135 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +import numpy as np + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.rowNum = 10 + self.ts = 1537146000000 + + def run(self): + tdSql.prepare() + + intData = [] + floatData = [] + + tdSql.execute('''create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20))''') + tdSql.execute("create table test1 using test tags('beijing')") + for i in range(self.rowNum): + tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d')" + % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) + intData.append(i + 1) + floatData.append(i + 0.1) + + # twa verifacation + tdSql.error("select twa(ts) from test") + tdSql.error("select twa(ts) from test1") + + tdSql.error("select twa(col1) from test") + tdSql.error("select twa(col1) from test1") + + tdSql.error("select twa(col2) from test") + tdSql.error("select twa(col2) from test1") + + tdSql.error("select twa(col3) from test") + tdSql.error("select twa(col3) from test1") + + tdSql.error("select twa(col4) from test") + tdSql.error("select twa(col4) from test1") + + tdSql.error("select twa(col5) from test") + tdSql.error("select twa(col5) from test1") + + tdSql.error("select twa(col6) from test") + tdSql.error("select twa(col6) from test1") + + tdSql.error("select twa(col7) from test") + tdSql.error("select twa(col7) from test1") + + tdSql.error("select twa(col8) from test") + tdSql.error("select twa(col8) from test1") + + tdSql.error("select twa(col9) from test") + tdSql.error("select twa(col9) from test1") + + tdSql.error("select twa(col1) from test where ts > %d" % self.ts) + tdSql.error("select twa(col1) from test1 where ts > %d" % self.ts) + + tdSql.error("select twa(col2) from test where ts > %d" % self.ts) + tdSql.error("select twa(col2) from test1 where ts > %d" % self.ts) + + tdSql.error("select twa(col3) from test where ts > %d" % self.ts) + tdSql.error("select twa(col3) from test1 where ts > %d" % self.ts) + + tdSql.error("select twa(col4) from test where ts > %d" % self.ts) + tdSql.error("select twa(col4) from test1 where ts > %d" % self.ts) + + tdSql.error("select twa(col5) from test where ts > %d" % self.ts) + tdSql.error("select twa(col5) from test1 where ts > %d" % self.ts) + + tdSql.error("select twa(col6) from test where ts > %d" % self.ts) + tdSql.error("select twa(col6) from test1 where ts > %d" % self.ts) + + tdSql.error("select twa(col1) from test where ts < %d" % (self.ts + self.rowNum)) + tdSql.error("select twa(col1) from test1 where ts < %d" % (self.ts + self.rowNum)) + + tdSql.error("select twa(col2) from test where ts < %d" % (self.ts + self.rowNum)) + tdSql.error("select twa(col2) from test1 where ts < %d" % (self.ts + self.rowNum)) + + tdSql.error("select twa(col3) from test where ts < %d" % (self.ts + self.rowNum)) + tdSql.error("select twa(col3) from test1 where ts < %d" % (self.ts + self.rowNum)) + + tdSql.error("select twa(col4) from test where ts < %d" % (self.ts + self.rowNum)) + tdSql.error("select twa(col4) from test1 where ts < %d" % (self.ts + self.rowNum)) + + tdSql.error("select twa(col5) from test where ts < %d" % (self.ts + self.rowNum)) + tdSql.error("select twa(col5) from test1 where ts < %d" % (self.ts + self.rowNum)) + + tdSql.error("select twa(col6) from test where ts < %d" % (self.ts + self.rowNum)) + tdSql.error("select twa(col6) from test1 where ts < %d" % (self.ts + self.rowNum)) + + tdSql.query("select twa(col1) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + tdSql.query("select twa(col1) from test1 where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + + tdSql.query("select twa(col2) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + tdSql.query("select twa(col2) from test1 where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + + tdSql.query("select twa(col3) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + tdSql.query("select twa(col3) from test1 where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + + tdSql.query("select twa(col4) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + tdSql.query("select twa(col4) from test1 where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + + tdSql.query("select twa(col5) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + tdSql.query("select twa(col5) from test1 where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + + tdSql.query("select twa(col6) from test where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + tdSql.query("select twa(col6) from test1 where ts > %d and ts < %d" % (self.ts, self.ts + self.rowNum)) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/query/filterOtherTypes.py b/tests/pytest/query/filterOtherTypes.py index 1db5497604..5033ffdb48 100644 --- a/tests/pytest/query/filterOtherTypes.py +++ b/tests/pytest/query/filterOtherTypes.py @@ -231,10 +231,12 @@ class TDTestCase: tdSql.error("select * from st where tagcol1 like '____'") # > for nchar type on tag - tdSql.error("select * from st where tagcol2 > 'table'") + tdSql.query("select * from st where tagcol2 > 'table1'") + tdSql.checkRows(5) # >= for nchar type on tag - tdSql.error("select * from st where tagcol2 >= 'table'") + tdSql.query("select * from st where tagcol2 >= 'table1'") + tdSql.checkRows(10) # = for nchar type on tag tdSql.query("select * from st where tagcol2 = 'table1'") @@ -249,10 +251,12 @@ class TDTestCase: tdSql.checkRows(10) # > for nchar type on tag - tdSql.error("select * from st where tagcol2 < 'table'") + tdSql.query("select * from st where tagcol2 < 'table'") + tdSql.checkRows(0) # >= for nchar type on tag - tdSql.error("select * from st where tagcol2 <= 'table'") + tdSql.query("select * from st where tagcol2 <= 'table'") + tdSql.checkRows(0) # % for nchar type on tag case 1 tdSql.query("select * from st where tagcol2 like '%'") @@ -291,10 +295,12 @@ class TDTestCase: tdSql.checkRows(10) # > for binary type on tag - tdSql.error("select * from st where tagcol3 > '表'") + tdSql.query("select * from st where tagcol3 > '表'") + tdSql.checkRows(10) # >= for binary type on tag - tdSql.error("select * from st where tagcol3 >= '表'") + tdSql.query("select * from st where tagcol3 >= '表'") + tdSql.checkRows(10) # = for binary type on tag tdSql.query("select * from st where tagcol3 = '水表'") @@ -309,10 +315,12 @@ class TDTestCase: tdSql.checkRows(5) # > for binary type on tag - tdSql.error("select * from st where tagcol3 < '水表'") + tdSql.query("select * from st where tagcol3 < '水表'") + tdSql.checkRows(0) # >= for binary type on tag - tdSql.error("select * from st where tagcol3 <= '水表'") + tdSql.query("select * from st where tagcol3 <= '水表'") + tdSql.checkRows(5) # % for binary type on tag case 1 tdSql.query("select * from st where tagcol3 like '%'") diff --git a/tests/pytest/regressiontest.sh b/tests/pytest/regressiontest.sh index 24cd93f0fc..61ec491f5d 100755 --- a/tests/pytest/regressiontest.sh +++ b/tests/pytest/regressiontest.sh @@ -141,6 +141,7 @@ python3 ./test.py -f query/filterCombo.py python3 ./test.py -f query/queryNormal.py python3 ./test.py -f query/select_last_crash.py python3 ./test.py -f query/queryNullValueTest.py +python3 ./test.py -f query/queryInsertValue.py #stream python3 ./test.py -f stream/stream1.py @@ -155,3 +156,23 @@ python3 ./test.py -f client/client.py # Misc python3 testCompress.py python3 testNoCompress.py + + +# functions +python3 ./test.py -f functions/function_avg.py +python3 ./test.py -f functions/function_bottom.py +python3 ./test.py -f functions/function_count.py +python3 ./test.py -f functions/function_diff.py +python3 ./test.py -f functions/function_first.py +python3 ./test.py -f functions/function_last.py +python3 ./test.py -f functions/function_last_row.py +python3 ./test.py -f functions/function_leastsquares.py +python3 ./test.py -f functions/function_max.py +python3 ./test.py -f functions/function_min.py +python3 ./test.py -f functions/function_operations.py +python3 ./test.py -f functions/function_percentile.py +python3 ./test.py -f functions/function_spread.py +python3 ./test.py -f functions/function_stddev.py +python3 ./test.py -f functions/function_sum.py +python3 ./test.py -f functions/function_top.py +python3 ./test.py -f functions/function_twa.py diff --git a/tests/pytest/util/sql.py b/tests/pytest/util/sql.py index ec39ab61b9..1e1d02959f 100644 --- a/tests/pytest/util/sql.py +++ b/tests/pytest/util/sql.py @@ -122,11 +122,16 @@ class TDSql: return self.cursor.istype(col, dataType) def checkData(self, row, col, data): - self.checkRowCol(row, col) - if self.queryResult[row][col] != data: - caller = inspect.getframeinfo(inspect.stack()[1][0]) - args = (caller.filename, caller.lineno, self.sql, row, col, self.queryResult[row][col], data) - tdLog.exit("%s(%d) failed: sql:%s row:%d col:%d data:%s != expect:%s" % args) + self.checkRowCol(row, col) + if self.queryResult[row][col] != data: + if isinstance(data, float) and abs(self.queryResult[row][col] - data) <= 0.000001: + tdLog.info("sql:%s, row:%d col:%d data:%f == expect:%f" % + (self.sql, row, col, self.queryResult[row][col], data)) + return + else: + caller = inspect.getframeinfo(inspect.stack()[1][0]) + args = (caller.filename, caller.lineno, self.sql, row, col, self.queryResult[row][col], data) + tdLog.exit("%s(%d) failed: sql:%s row:%d col:%d data:%s != expect:%s" % args) if data is None: tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%s" % @@ -135,9 +140,12 @@ class TDSql: tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%s" % (self.sql, row, col, self.queryResult[row][col], data)) elif isinstance(data, datetime.date): + tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%s" % + (self.sql, row, col, self.queryResult[row][col], data)) + elif isinstance(data, float): tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%s" % (self.sql, row, col, self.queryResult[row][col], data)) - else: + else: tdLog.info("sql:%s, row:%d col:%d data:%s == expect:%d" % (self.sql, row, col, self.queryResult[row][col], data)) From 2867f3082942b3b27fa5e573e530ad66a8e1c2d8 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 21 Jul 2020 17:29:31 +0800 Subject: [PATCH 055/100] fix sh/exec signal for coverage test. [TD-976] --- tests/script/fullGeneralSuite.sim | 4 ---- tests/script/sh/exec-no-random-fail.sh | 12 +++++++----- tests/script/sh/exec-random-fail.sh | 15 +++++++++------ tests/script/sh/exec.sh | 8 ++++---- 4 files changed, 20 insertions(+), 19 deletions(-) diff --git a/tests/script/fullGeneralSuite.sim b/tests/script/fullGeneralSuite.sim index 2f45ab077c..15cc2954e8 100644 --- a/tests/script/fullGeneralSuite.sim +++ b/tests/script/fullGeneralSuite.sim @@ -135,7 +135,6 @@ run general/parser/set_tag_vals.sim #unsupport run general/parser/repeatAlter.sim #unsupport run general/parser/slimit_alter_tags.sim #unsupport run general/parser/stream_on_sys.sim -run general/parser/stream.sim #unsupport run general/parser/repeatStream.sim run general/stable/disk.sim run general/stable/dnode3.sim @@ -212,12 +211,9 @@ run general/vector/table_mix.sim run general/vector/table_query.sim run general/vector/table_time.sim run general/stream/restart_stream.sim -run general/stream/stream_1.sim -run general/stream/stream_2.sim run general/stream/stream_3.sim run general/stream/stream_restart.sim run general/stream/table_1.sim -run general/stream/metrics_1.sim run general/stream/table_n.sim run general/stream/metrics_n.sim run general/stream/table_del.sim diff --git a/tests/script/sh/exec-no-random-fail.sh b/tests/script/sh/exec-no-random-fail.sh index 04a663bc5a..2bd0a64923 100755 --- a/tests/script/sh/exec-no-random-fail.sh +++ b/tests/script/sh/exec-no-random-fail.sh @@ -88,7 +88,9 @@ if [ "$EXEC_OPTON" = "start" ]; then echo "ExcuteCmd:" $EXE_DIR/taosd -c $CFG_DIR if [ "$SHELL_OPTION" = "true" ]; then - nohup valgrind --log-file=${LOG_DIR}/valgrind.log --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes $EXE_DIR/taosd -c $CFG_DIR > /dev/null 2>&1 & + TT=`date +%s` + mkdir ${LOG_DIR}/${TT} + nohup valgrind --log-file=${LOG_DIR}/${TT}/valgrind.log --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes $EXE_DIR/taosd -c $CFG_DIR > /dev/null 2>&1 & else nohup $EXE_DIR/taosd -c $CFG_DIR --random-file-fail-factor 0 > /dev/null 2>&1 & fi @@ -99,12 +101,12 @@ else PID=`ps -ef|grep taosd | grep $RCFG_DIR | grep -v grep | awk '{print $2}'` while [ -n "$PID" ] do - if [ "$SIGNAL" = "SIGINT" ]; then - echo try to kill by signal SIGINT - kill -SIGINT $PID - else + if [ "$SIGNAL" = "SIGKILL" ]; then echo try to kill by signal SIGKILL kill -9 $PID + else + echo try to kill by signal SIGINT + kill -SIGINT $PID fi sleep 1 PID=`ps -ef|grep taosd | grep $RCFG_DIR | grep -v grep | awk '{print $2}'` diff --git a/tests/script/sh/exec-random-fail.sh b/tests/script/sh/exec-random-fail.sh index a354021684..3761498859 100755 --- a/tests/script/sh/exec-random-fail.sh +++ b/tests/script/sh/exec-random-fail.sh @@ -88,9 +88,12 @@ if [ "$EXEC_OPTON" = "start" ]; then echo "ExcuteCmd:" $EXE_DIR/taosd -c $CFG_DIR if [ "$SHELL_OPTION" = "true" ]; then - nohup valgrind --log-file=${LOG_DIR}/valgrind.log --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes $EXE_DIR/taosd -c $CFG_DIR > /dev/null 2>&1 & + TT=`date +%s` + mkdir ${LOG_DIR}/${TT} + nohup valgrind --log-file=${LOG_DIR}/${TT}/valgrind.log --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes $EXE_DIR/taosd -c $CFG_DIR > /dev/null 2>&1 & else - nohup $EXE_DIR/taosd -c $CFG_DIR --alloc-random-fail --random-file-fail-factor 5 > /dev/null 2>&1 & + nohup $EXE_DIR/taosd -c $CFG_DIR --alloc-random-fail \ + --random-file-fail-factor 5 > /dev/null 2>&1 & fi else @@ -99,12 +102,12 @@ else PID=`ps -ef|grep taosd | grep $RCFG_DIR | grep -v grep | awk '{print $2}'` while [ -n "$PID" ] do - if [ "$SIGNAL" = "SIGINT" ]; then - echo try to kill by signal SIGINT - kill -SIGINT $PID - else + if [ "$SIGNAL" = "SIGKILL" ]; then echo try to kill by signal SIGKILL kill -9 $PID + else + echo try to kill by signal SIGINT + kill -SIGINT $PID fi sleep 1 PID=`ps -ef|grep taosd | grep $RCFG_DIR | grep -v grep | awk '{print $2}'` diff --git a/tests/script/sh/exec.sh b/tests/script/sh/exec.sh index 2f294075a1..1c84a6b10e 100755 --- a/tests/script/sh/exec.sh +++ b/tests/script/sh/exec.sh @@ -101,12 +101,12 @@ else PID=`ps -ef|grep taosd | grep $RCFG_DIR | grep -v grep | awk '{print $2}'` while [ -n "$PID" ] do - if [ "$SIGNAL" = "SIGINT" ]; then - echo try to kill by signal SIGINT - kill -SIGINT $PID - else + if [ "$SIGNAL" = "SIGKILL" ]; then echo try to kill by signal SIGKILL kill -9 $PID + else + echo try to kill by signal SIGINT + kill -SIGINT $PID fi sleep 1 PID=`ps -ef|grep taosd | grep $RCFG_DIR | grep -v grep | awk '{print $2}'` From 5e8b11415a8e3241668bc875638346027f25d1b2 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 21 Jul 2020 18:08:46 +0800 Subject: [PATCH 056/100] need to restore magic number when kvstore open --- src/util/src/tkvstore.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/util/src/tkvstore.c b/src/util/src/tkvstore.c index a2022569bf..0704285da5 100644 --- a/src/util/src/tkvstore.c +++ b/src/util/src/tkvstore.c @@ -141,6 +141,7 @@ SKVStore *tdOpenKVStore(char *fname, iterFunc iFunc, afterFunc aFunc, void *appH if (tdLoadKVStoreHeader(pStore->fd, pStore->fname, &info) < 0) goto _err; pStore->info.size = TD_KVSTORE_HEADER_SIZE; + pStore->info.magic = info.magic; if (tdRestoreKVStore(pStore) < 0) goto _err; From 52fb827f0a5f42f33f952c313b8bcd02009ec928 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 21 Jul 2020 20:40:19 +0800 Subject: [PATCH 057/100] [TD-972] --- src/mnode/inc/mnodeDef.h | 32 +++++++++++++++----------------- src/mnode/src/mnodeSdb.c | 14 ++++++++++---- 2 files changed, 25 insertions(+), 21 deletions(-) diff --git a/src/mnode/inc/mnodeDef.h b/src/mnode/inc/mnodeDef.h index 3154a441ca..a454f413f0 100644 --- a/src/mnode/inc/mnodeDef.h +++ b/src/mnode/inc/mnodeDef.h @@ -50,8 +50,8 @@ typedef struct SDnodeObj { int8_t alternativeRole; // from dnode status msg, 0-any, 1-mgmt, 2-dnode int8_t status; // set in balance function int8_t isMgmt; - int8_t reserve1[14]; - int8_t updateEnd[1]; + int8_t reserve1[11]; + int8_t updateEnd[4]; int32_t refCount; uint32_t moduleStatus; uint32_t lastReboot; // time stamp for last reboot @@ -68,8 +68,8 @@ typedef struct SMnodeObj { int32_t mnodeId; int8_t reserved0[4]; int64_t createdTime; - int8_t reserved1[7]; - int8_t updateEnd[1]; + int8_t reserved1[4]; + int8_t updateEnd[4]; int32_t refCount; int8_t role; int8_t reserved2[3]; @@ -90,8 +90,7 @@ typedef struct SSuperTableObj { int32_t tversion; int32_t numOfColumns; int32_t numOfTags; - int8_t reserved1[3]; - int8_t updateEnd[1]; + int8_t updateEnd[4]; int32_t refCount; int32_t numOfTables; SSchema * schema; @@ -111,8 +110,7 @@ typedef struct { int32_t sid; int32_t vgId; int32_t sqlLen; - int8_t updateEnd[1]; - int8_t reserved1[1]; + int8_t updateEnd[4]; int32_t refCount; char* sql; //used by normal table SSchema* schema; //used by normal table @@ -138,8 +136,8 @@ typedef struct SVgObj { int8_t status; int8_t reserved0[4]; SVnodeGid vnodeGid[TSDB_MAX_REPLICA]; - int8_t reserved1[7]; - int8_t updateEnd[1]; + int8_t reserved1[4]; + int8_t updateEnd[4]; int32_t refCount; int32_t numOfTables; int64_t totalStorage; @@ -176,8 +174,8 @@ typedef struct SDbObj { int32_t cfgVersion; SDbCfg cfg; int8_t status; - int8_t reserved1[14]; - int8_t updateEnd[1]; + int8_t reserved1[11]; + int8_t updateEnd[4]; int32_t refCount; int32_t numOfVgroups; int32_t numOfTables; @@ -196,8 +194,8 @@ typedef struct SUserObj { int64_t createdTime; int8_t superAuth; int8_t writeAuth; - int8_t reserved[13]; - int8_t updateEnd[1]; + int8_t reserved[10]; + int8_t updateEnd[4]; int32_t refCount; struct SAcctObj * pAcct; } SUserObj; @@ -228,11 +226,11 @@ typedef struct SAcctObj { int64_t createdTime; int32_t acctId; int8_t status; - int8_t reserved0[10]; - int8_t updateEnd[1]; - SAcctInfo acctInfo; + int8_t reserved0[7]; + int8_t updateEnd[4]; int32_t refCount; int8_t reserved1[4]; + SAcctInfo acctInfo; pthread_mutex_t mutex; } SAcctObj; diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index dee82f53be..26b0f60fcd 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -406,7 +406,7 @@ void sdbDecRef(void *handle, void *pObj) { int32_t refCount = atomic_sub_fetch_32(pRefCount, 1); sdbTrace("def ref of table:%s record:%p:%s:%d", pTable->tableName, pObj, sdbGetKeyStrFromObj(pTable, pObj), *pRefCount); - int8_t *updateEnd = pObj + pTable->refCountPos - 1; + int32_t *updateEnd = pObj + pTable->refCountPos - 4; if (refCount <= 0 && *updateEnd) { sdbTrace("table:%s, record:%p:%s:%d is destroyed", pTable->tableName, pObj, sdbGetKeyStrFromObj(pTable, pObj), *pRefCount); SSdbOper oper = {.pObj = pObj}; @@ -472,6 +472,14 @@ static int32_t sdbInsertHash(SSdbTable *pTable, SSdbOper *pOper) { } static int32_t sdbDeleteHash(SSdbTable *pTable, SSdbOper *pOper) { + int32_t *updateEnd = pOper->pObj + pTable->refCountPos - 4; + bool set = atomic_val_compare_exchange_32(updateEnd, 0, 1) == 0; + if (!set) { + sdbError("table:%s, failed to delete record:%s from hash, for it already removed", pTable->tableName, + sdbGetKeyStrFromObj(pTable, pOper->pObj)); + return TSDB_CODE_MND_SDB_OBJ_NOT_THERE; + } + (*pTable->deleteFp)(pOper); void * key = sdbGetObjKey(pTable, pOper->pObj); @@ -486,8 +494,6 @@ static int32_t sdbDeleteHash(SSdbTable *pTable, SSdbOper *pOper) { sdbDebug("table:%s, delete record:%s from hash, numOfRows:%" PRId64 ", msg:%p", pTable->tableName, sdbGetKeyStrFromObj(pTable, pOper->pObj), pTable->numOfRows, pOper->pMsg); - int8_t *updateEnd = pOper->pObj + pTable->refCountPos - 1; - *updateEnd = 1; sdbDecRef(pTable, pOper->pObj); return TSDB_CODE_SUCCESS; @@ -654,7 +660,7 @@ bool sdbCheckRowDeleted(void *pTableInput, void *pRow) { SSdbTable *pTable = pTableInput; if (pTable == NULL) return false; - int8_t *updateEnd = pRow + pTable->refCountPos - 1; + int32_t *updateEnd = pRow + pTable->refCountPos - 4; return (*updateEnd == 1); } From 946f6cb87f111936e8d54a742453ed1700ea0d59 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 22 Jul 2020 10:07:57 +0800 Subject: [PATCH 058/100] fix restart bug --- src/tsdb/src/tsdbMain.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index 7fe69021bf..772bcf48d6 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -795,6 +795,7 @@ static int tsdbRestoreInfo(STsdbRepo *pRepo) { for (int i = 1; i < pRepo->config.maxTables; i++) { STable *pTable = pMeta->tables[i]; if (pTable == NULL) continue; + tsdbSetHelperTable(&rhelper, pTable, pRepo); SCompIdx *pIdx = &(rhelper.curCompIdx); if (pIdx->offset > 0 && pTable->lastKey < pIdx->maxKey) pTable->lastKey = pIdx->maxKey; From 2104d465863922c9352bb9778f88cc3f66cb1fdb Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 22 Jul 2020 10:26:25 +0800 Subject: [PATCH 059/100] change kill to term signal for coverage flush. --- tests/pytest/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pytest/test.py b/tests/pytest/test.py index a9da8e5671..678bd87336 100644 --- a/tests/pytest/test.py +++ b/tests/pytest/test.py @@ -96,7 +96,7 @@ if __name__ == "__main__": processID = subprocess.check_output(usePortPID, shell=True) if processID: - killCmd = "kill -9 %s" % processID + killCmd = "kill -TERM %s" % processID os.system(killCmd) fuserCmd = "fuser -k -n tcp %d" % port os.system(fuserCmd) From 9463982cecc1c00c2c47b3dc3448d8d148114c62 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 22 Jul 2020 11:27:02 +0800 Subject: [PATCH 060/100] [TD-972] drop table before rsp --- src/mnode/src/mnodeSdb.c | 3 +- src/mnode/src/mnodeTable.c | 68 +++++++++++++++++++++++--------------- 2 files changed, 44 insertions(+), 27 deletions(-) diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index 26b0f60fcd..4b2945152b 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -661,7 +661,8 @@ bool sdbCheckRowDeleted(void *pTableInput, void *pRow) { if (pTable == NULL) return false; int32_t *updateEnd = pRow + pTable->refCountPos - 4; - return (*updateEnd == 1); + return atomic_val_compare_exchange_32(updateEnd, 1, 1) == 1; + // return (*updateEnd == 1); } int32_t sdbDeleteRow(SSdbOper *pOper) { diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index 7478d7cd78..12ab58d949 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -72,7 +72,7 @@ static void mnodeProcessCreateChildTableRsp(SRpcMsg *rpcMsg); static int32_t mnodeProcessDropTableMsg(SMnodeMsg *mnodeMsg); static int32_t mnodeProcessDropSuperTableMsg(SMnodeMsg *pMsg); static void mnodeProcessDropSuperTableRsp(SRpcMsg *rpcMsg); -static int32_t mnodeProcessDropChildTableMsg(SMnodeMsg *pMsg, bool needReturn); +static int32_t mnodeProcessDropChildTableMsg(SMnodeMsg *pMsg); static void mnodeProcessDropChildTableRsp(SRpcMsg *rpcMsg); static int32_t mnodeProcessSuperTableVgroupMsg(SMnodeMsg *mnodeMsg); @@ -759,7 +759,7 @@ static int32_t mnodeProcessDropTableMsg(SMnodeMsg *pMsg) { SChildTableObj *pCTable = (SChildTableObj *)pMsg->pTable; mInfo("app:%p:%p, table:%s, start to drop ctable, vgId:%d sid:%d uid:%" PRIu64, pMsg->rpcMsg.ahandle, pMsg, pDrop->tableId, pCTable->vgId, pCTable->sid, pCTable->uid); - return mnodeProcessDropChildTableMsg(pMsg, true); + return mnodeProcessDropChildTableMsg(pMsg); } } @@ -882,7 +882,7 @@ static int32_t mnodeProcessCreateSuperTableMsg(SMnodeMsg *pMsg) { static int32_t mnodeDropSuperTableCb(SMnodeMsg *pMsg, int32_t code) { SSuperTableObj *pTable = (SSuperTableObj *)pMsg->pTable; if (code != TSDB_CODE_SUCCESS) { - mError("app:%p:%p, table:%s, failed to drop, sdb error", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId); + mError("app:%p:%p, stable:%s, failed to drop, sdb error", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId); } else { mLInfo("app:%p:%p, stable:%s, is dropped from sdb", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId); } @@ -1765,18 +1765,13 @@ static int32_t mnodeProcessCreateChildTableMsg(SMnodeMsg *pMsg) { } } -static int32_t mnodeProcessDropChildTableMsg(SMnodeMsg *pMsg, bool needReturn) { +static int32_t mnodeSendDropChildTableMsg(SMnodeMsg *pMsg, bool needReturn) { SChildTableObj *pTable = (SChildTableObj *)pMsg->pTable; - if (pMsg->pVgroup == NULL) pMsg->pVgroup = mnodeGetVgroup(pTable->vgId); - if (pMsg->pVgroup == NULL) { - mError("app:%p:%p, table:%s, failed to drop ctable, vgroup not exist", pMsg->rpcMsg.ahandle, pMsg, - pTable->info.tableId); - return TSDB_CODE_MND_APP_ERROR; - } + mLInfo("app:%p:%p, ctable:%s, is dropped from sdb", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId); SMDDropTableMsg *pDrop = rpcMallocCont(sizeof(SMDDropTableMsg)); if (pDrop == NULL) { - mError("app:%p:%p, table:%s, failed to drop ctable, no enough memory", pMsg->rpcMsg.ahandle, pMsg, + mError("app:%p:%p, ctable:%s, failed to drop ctable, no enough memory", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId); return TSDB_CODE_MND_OUT_OF_MEMORY; } @@ -1789,7 +1784,7 @@ static int32_t mnodeProcessDropChildTableMsg(SMnodeMsg *pMsg, bool needReturn) { SRpcEpSet epSet = mnodeGetEpSetFromVgroup(pMsg->pVgroup); - mInfo("app:%p:%p, table:%s, send drop ctable msg, vgId:%d sid:%d uid:%" PRIu64, pMsg->rpcMsg.ahandle, pMsg, + mInfo("app:%p:%p, ctable:%s, send drop ctable msg, vgId:%d sid:%d uid:%" PRIu64, pMsg->rpcMsg.ahandle, pMsg, pDrop->tableId, pTable->vgId, pTable->sid, pTable->uid); SRpcMsg rpcMsg = { @@ -1807,6 +1802,40 @@ static int32_t mnodeProcessDropChildTableMsg(SMnodeMsg *pMsg, bool needReturn) { return TSDB_CODE_MND_ACTION_IN_PROGRESS; } +static int32_t mnodeDropChildTableCb(SMnodeMsg *pMsg, int32_t code) { + if (code != TSDB_CODE_SUCCESS) { + SChildTableObj *pTable = (SChildTableObj *)pMsg->pTable; + mError("app:%p:%p, ctable:%s, failed to drop, sdb error", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId); + return code; + } + + return mnodeSendDropChildTableMsg(pMsg, true); +} + +static int32_t mnodeProcessDropChildTableMsg(SMnodeMsg *pMsg) { + SChildTableObj *pTable = (SChildTableObj *)pMsg->pTable; + if (pMsg->pVgroup == NULL) pMsg->pVgroup = mnodeGetVgroup(pTable->vgId); + if (pMsg->pVgroup == NULL) { + mError("app:%p:%p, table:%s, failed to drop ctable, vgroup not exist", pMsg->rpcMsg.ahandle, pMsg, + pTable->info.tableId); + return TSDB_CODE_MND_APP_ERROR; + } + + SSdbOper oper = { + .type = SDB_OPER_GLOBAL, + .table = tsChildTableSdb, + .pObj = pTable, + .pMsg = pMsg, + .cb = mnodeDropChildTableCb + }; + + int32_t code = sdbDeleteRow(&oper); + if (code == TSDB_CODE_SUCCESS) { + return TSDB_CODE_MND_ACTION_IN_PROGRESS; + } + return code; +} + static int32_t mnodeFindNormalTableColumnIndex(SChildTableObj *pTable, char *colName) { SSchema *schema = (SSchema *) pTable->schema; for (int32_t col = 0; col < pTable->numOfColumns; col++) { @@ -2220,19 +2249,6 @@ static void mnodeProcessDropChildTableRsp(SRpcMsg *rpcMsg) { return; } - SSdbOper oper = { - .type = SDB_OPER_GLOBAL, - .table = tsChildTableSdb, - .pObj = pTable - }; - - int32_t code = sdbDeleteRow(&oper); - if (code != TSDB_CODE_SUCCESS) { - mError("app:%p:%p, table:%s, update ctables sdb error", mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId); - dnodeSendRpcMnodeWriteRsp(mnodeMsg, TSDB_CODE_MND_SDB_ERROR); - return; - } - if (mnodeMsg->pVgroup->numOfTables <= 0) { mInfo("app:%p:%p, vgId:%d, all tables is dropped, drop vgroup", mnodeMsg->rpcMsg.ahandle, mnodeMsg, mnodeMsg->pVgroup->vgId); @@ -2259,7 +2275,7 @@ static void mnodeProcessCreateChildTableRsp(SRpcMsg *rpcMsg) { if (sdbCheckRowDeleted(tsChildTableSdb, pTable)) { mDebug("app:%p:%p, table:%s, create table rsp received, but a deleting opertion incoming, vgId:%d sid:%d uid:%" PRIu64, mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId, pTable->vgId, pTable->sid, pTable->uid); - mnodeProcessDropChildTableMsg(mnodeMsg, false); + mnodeSendDropChildTableMsg(mnodeMsg, false); rpcMsg->code = TSDB_CODE_SUCCESS; } From 9397bbe48a4bd18c5dceff6c0bc244dae011981b Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 22 Jul 2020 11:47:02 +0800 Subject: [PATCH 061/100] [TD-972] jump or move depends on uninitialised value --- src/mnode/src/mnodeShow.c | 4 ++-- src/mnode/src/mnodeUser.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/mnode/src/mnodeShow.c b/src/mnode/src/mnodeShow.c index e3d5b41be3..1d85a8cacd 100644 --- a/src/mnode/src/mnodeShow.c +++ b/src/mnode/src/mnodeShow.c @@ -236,7 +236,7 @@ static int32_t mnodeProcessHeartBeatMsg(SMnodeMsg *pMsg) { } SCMHeartBeatMsg *pHBMsg = pMsg->rpcMsg.pCont; - SRpcConnInfo connInfo; + SRpcConnInfo connInfo = {0}; rpcGetConnInfo(pMsg->rpcMsg.handle, &connInfo); int32_t connId = htonl(pHBMsg->connId); @@ -284,7 +284,7 @@ static int32_t mnodeProcessConnectMsg(SMnodeMsg *pMsg) { SCMConnectRsp *pConnectRsp = NULL; int32_t code = TSDB_CODE_SUCCESS; - SRpcConnInfo connInfo; + SRpcConnInfo connInfo = {0}; if (rpcGetConnInfo(pMsg->rpcMsg.handle, &connInfo) != 0) { mError("thandle:%p is already released while process connect msg", pMsg->rpcMsg.handle); code = TSDB_CODE_MND_INVALID_CONNECTION; diff --git a/src/mnode/src/mnodeUser.c b/src/mnode/src/mnodeUser.c index 84f5d6aa58..8c783eebaf 100644 --- a/src/mnode/src/mnodeUser.c +++ b/src/mnode/src/mnodeUser.c @@ -358,7 +358,7 @@ static int32_t mnodeRetrieveUsers(SShowObj *pShow, char *data, int32_t rows, voi } SUserObj *mnodeGetUserFromConn(void *pConn) { - SRpcConnInfo connInfo; + SRpcConnInfo connInfo = {0}; if (rpcGetConnInfo(pConn, &connInfo) == 0) { return mnodeGetUser(connInfo.user); } else { From b6fb79dc0dd43518442ef0a02ca71e5e905b495a Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Wed, 22 Jul 2020 14:32:36 +0800 Subject: [PATCH 062/100] fix td-900: support both config dir & file --- src/util/src/tconfig.c | 53 ++++++++++++++++++++++++------------------ 1 file changed, 31 insertions(+), 22 deletions(-) diff --git a/src/util/src/tconfig.c b/src/util/src/tconfig.c index 74acb7ce35..d561e8ba5f 100644 --- a/src/util/src/tconfig.c +++ b/src/util/src/tconfig.c @@ -308,38 +308,47 @@ bool taosReadGlobalCfg() { sprintf(fileName, "%s/taos.cfg", configDir); FILE* fp = fopen(fileName, "r"); + if (fp == NULL) { + struct stat s; + if (stat(configDir, &s) != 0 || (!S_ISREG(s.st_mode) && !S_ISLNK(s.st_mode))) { + //return true to follow behavior before file support + return true; + } + fp = fopen(configDir, "r"); + if (fp == NULL) { + return false; + } + } size_t len = 1024; line = calloc(1, len); - if (fp != NULL) { - while (!feof(fp)) { - memset(line, 0, len); + while (!feof(fp)) { + memset(line, 0, len); - option = value = NULL; - olen = vlen = 0; + option = value = NULL; + olen = vlen = 0; - getline(&line, &len, fp); - line[len - 1] = 0; - - paGetToken(line, &option, &olen); - if (olen == 0) continue; - option[olen] = 0; + getline(&line, &len, fp); + line[len - 1] = 0; + + paGetToken(line, &option, &olen); + if (olen == 0) continue; + option[olen] = 0; - paGetToken(option + olen + 1, &value, &vlen); - if (vlen == 0) continue; - value[vlen] = 0; + paGetToken(option + olen + 1, &value, &vlen); + if (vlen == 0) continue; + value[vlen] = 0; - // For dataDir, the format is: - // dataDir /mnt/disk1 0 - paGetToken(value + vlen + 1, &value1, &vlen1); - - taosReadConfigOption(option, value); - } - - fclose(fp); + // For dataDir, the format is: + // dataDir /mnt/disk1 0 + paGetToken(value + vlen + 1, &value1, &vlen1); + + taosReadConfigOption(option, value); } + fclose(fp); + tfree(line); return true; From ec47ae9ccab2d0a0695a66dd9b6cd8b7ad67cb5d Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Wed, 22 Jul 2020 14:38:06 +0800 Subject: [PATCH 063/100] TD-978: adjust kill command --- .../java/com/taosdata/jdbc/utils/TDNode.java | 51 ++++++++++++++++--- .../java/com/taosdata/jdbc/utils/TDNodes.java | 32 ++---------- .../test/java/com/taosdata/jdbc/BaseTest.java | 28 +++++----- 3 files changed, 62 insertions(+), 49 deletions(-) diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TDNode.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TDNode.java index df25076a95..800265868d 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TDNode.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TDNode.java @@ -1,6 +1,8 @@ package com.taosdata.jdbc.utils; +import java.io.BufferedReader; import java.io.File; +import java.io.InputStreamReader; import java.util.*; import java.util.concurrent.TimeUnit; @@ -31,6 +33,10 @@ public class TDNode { this.testCluster = testCluster; } + public void setRunning(int running) { + this.running = running; + } + public void searchTaosd(File dir, ArrayList taosdPath) { File[] fileList = dir.listFiles(); @@ -102,15 +108,46 @@ public class TDNode { this.running = 1; } - public void stop() { - String toBeKilled = "taosd"; + public Integer getTaosdPid() { + String cmd = "ps -ef|grep -w taosd| grep -v grep | awk '{print $2}'"; + String[] cmds = {"sh", "-c", cmd}; + try { + Process process = Runtime.getRuntime().exec(cmds); + BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream())); + String line = null; + Integer res = null; + while((line = reader.readLine()) != null) { + if(!line.isEmpty()) { + res = Integer.valueOf(line); + break; + } + } + + return res; + } catch (Exception e) { + e.printStackTrace(); + } + return null; + } + + public void stop() { if (this.running != 0) { - String killCmd = "pkill -kill -x " + toBeKilled; - String[] killCmds = {"sh", "-c", killCmd}; - try { - Runtime.getRuntime().exec(killCmds).waitFor(); + Integer pid = null; + while((pid = getTaosdPid()) != null) { + + String killCmd = "kill -term " + pid; + String[] killCmds = {"sh", "-c", killCmd}; + try { + Runtime.getRuntime().exec(killCmds).waitFor(); + TimeUnit.SECONDS.sleep(2); + } catch (Exception e) { + e.printStackTrace(); + } + } + + try { for(int port = 6030; port < 6041; port ++) { String fuserCmd = "fuser -k -n tcp " + port; Runtime.getRuntime().exec(fuserCmd).waitFor(); @@ -120,7 +157,7 @@ public class TDNode { } this.running = 0; - System.out.println("dnode:" + this.index + " is stopped by pkill"); + System.out.println("dnode:" + this.index + " is stopped by kill -term"); } } diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TDNodes.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TDNodes.java index ea15ae9863..efc4c53e28 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TDNodes.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TDNodes.java @@ -14,33 +14,6 @@ public class TDNodes { } } - public void setPath(String path) { - try { - String killCmd = "pkill -kill -x taosd"; - String[] killCmds = {"sh", "-c", killCmd}; - Runtime.getRuntime().exec(killCmds).waitFor(); - - String binPath = System.getProperty("user.dir"); - binPath += "/../../../debug"; - System.out.println("binPath: " + binPath); - - File file = new File(path); - binPath = file.getCanonicalPath(); - System.out.println("binPath real path: " + binPath); - - if(path.isEmpty()){ - file = new File(path + "/../../"); - path = file.getCanonicalPath(); - } - - for(int i = 0; i < tdNodes.size(); i++) { - tdNodes.get(i).setPath(path); - } - } catch (Exception e) { - e.printStackTrace(); - } - } - public void setTestCluster(boolean testCluster) { this.testCluster = testCluster; } @@ -70,6 +43,11 @@ public class TDNodes { check(index); tdNodes.get(index - 1).setCfgConfig(option, value); } + + public TDNode getTDNode(int index) { + check(index); + return tdNodes.get(index - 1); + } public void start(int index) { check(index); diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/BaseTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/BaseTest.java index 5f105fb782..6c3437186f 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/BaseTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/BaseTest.java @@ -1,6 +1,5 @@ package com.taosdata.jdbc; -import java.io.File; import com.taosdata.jdbc.utils.TDNodes; import org.junit.AfterClass; @@ -8,31 +7,30 @@ import org.junit.BeforeClass; public class BaseTest { - private static boolean testCluster = false; - private static String deployPath = System.getProperty("user.dir"); - private static TDNodes tdNodes = new TDNodes(); + private static boolean testCluster = false; + private static TDNodes nodes = new TDNodes(); @BeforeClass public static void setupEnv() { - try{ - File file = new File(deployPath + "/../../../"); - String rootPath = file.getCanonicalPath(); - - tdNodes.setPath(rootPath); - tdNodes.setTestCluster(testCluster); + try{ + if (nodes.getTDNode(1).getTaosdPid() != null) { + System.out.println("Kill taosd before running JDBC test"); + nodes.getTDNode(1).setRunning(1); + nodes.stop(1); + } - tdNodes.deploy(1); - tdNodes.start(1); + nodes.setTestCluster(testCluster); + nodes.deploy(1); + nodes.start(1); } catch (Exception e) { - e.printStackTrace(); - System.out.println("Base Test Exception"); + e.printStackTrace(); } } @AfterClass public static void cleanUpEnv() { - tdNodes.stop(1); + nodes.stop(1); } } \ No newline at end of file From 01e348cc0182111863b2d819700719255c721726 Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Wed, 22 Jul 2020 16:23:52 +0800 Subject: [PATCH 064/100] fix td-980 --- src/query/src/qExecutor.c | 6 ++++- src/tsdb/src/tsdbMeta.c | 5 +++- src/tsdb/src/tsdbRead.c | 53 ++++++++++++++++++++++++++------------- 3 files changed, 45 insertions(+), 19 deletions(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 906d0cfe67..1220c5ca31 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -6419,8 +6419,12 @@ int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp **pRsp, int32_t *co size += sizeof(STableIdInfo) * taosArrayGetSize(pQInfo->arrTableIdInfo); *contLen = size + sizeof(SRetrieveTableRsp); - // todo handle failed to allocate memory + // todo proper handle failed to allocate memory, + // current solution only avoid crash, but cannot return error code to client *pRsp = (SRetrieveTableRsp *)rpcMallocCont(*contLen); + if (*pRsp == NULL) { + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } (*pRsp)->numOfRows = htonl(pQuery->rec.rows); int32_t code = pQInfo->code; diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index c1923f5235..b25e734694 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -123,7 +123,10 @@ int tsdbCreateTable(TSDB_REPO_T *repo, STableCfg *pCfg) { int tlen2 = tsdbGetTableEncodeSize(TSDB_UPDATE_META, table); int tlen = tlen1 + tlen2; void *buf = tsdbAllocBytes(pRepo, tlen); - ASSERT(buf != NULL); + if (buf == NULL) { + goto _err; + } + if (newSuper) { void *pBuf = tsdbInsertTableAct(pRepo, TSDB_UPDATE_META, buf, super); ASSERT(POINTER_DISTANCE(pBuf, buf) == tlen1); diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index b2204948d0..fc67c64e2e 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -201,18 +201,34 @@ TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STab int32_t numOfCols = pCond->numOfCols; pQueryHandle->statis = calloc(numOfCols, sizeof(SDataStatis)); + if (pQueryHandle->statis == NULL) { + tsdbCleanupQueryHandle(pQueryHandle); + return NULL; + } pQueryHandle->pColumns = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); // todo: use list instead of array? + if (pQueryHandle->pColumns == NULL) { + tsdbCleanupQueryHandle(pQueryHandle); + return NULL; + } for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData colInfo = {{0}, 0}; colInfo.info = pCond->colList[i]; colInfo.pData = calloc(1, EXTRA_BYTES + pQueryHandle->outputCapacity * pCond->colList[i].bytes); + if (colInfo.pData == NULL) { + tsdbCleanupQueryHandle(pQueryHandle); + return NULL; + } taosArrayPush(pQueryHandle->pColumns, &colInfo); pQueryHandle->statis[i].colId = colInfo.info.colId; } pQueryHandle->pTableCheckInfo = taosArrayInit(groupList->numOfTables, sizeof(STableCheckInfo)); + if (pQueryHandle->pTableCheckInfo == NULL) { + tsdbCleanupQueryHandle(pQueryHandle); + return NULL; + } STsdbMeta* pMeta = tsdbGetMeta(tsdb); assert(pMeta != NULL); @@ -2372,28 +2388,31 @@ void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle) { return; } - size_t size = taosArrayGetSize(pQueryHandle->pTableCheckInfo); - for (int32_t i = 0; i < size; ++i) { - STableCheckInfo* pTableCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i); - destroyTableMemIterator(pTableCheckInfo); + if (pQueryHandle->pTableCheckInfo != NULL) { + size_t size = taosArrayGetSize(pQueryHandle->pTableCheckInfo); + for (int32_t i = 0; i < size; ++i) { + STableCheckInfo* pTableCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i); + destroyTableMemIterator(pTableCheckInfo); - if (pTableCheckInfo->pDataCols != NULL) { - tfree(pTableCheckInfo->pDataCols->buf); + if (pTableCheckInfo->pDataCols != NULL) { + tfree(pTableCheckInfo->pDataCols->buf); + } + + tfree(pTableCheckInfo->pDataCols); + tfree(pTableCheckInfo->pCompInfo); } - - tfree(pTableCheckInfo->pDataCols); - tfree(pTableCheckInfo->pCompInfo); + taosArrayDestroy(pQueryHandle->pTableCheckInfo); } - taosArrayDestroy(pQueryHandle->pTableCheckInfo); + if (pQueryHandle->pColumns != NULL) { + size_t cols = taosArrayGetSize(pQueryHandle->pColumns); + for (int32_t i = 0; i < cols; ++i) { + SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); + tfree(pColInfo->pData); + } + taosArrayDestroy(pQueryHandle->pColumns); + } - size_t cols = taosArrayGetSize(pQueryHandle->pColumns); - for (int32_t i = 0; i < cols; ++i) { - SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); - tfree(pColInfo->pData); - } - - taosArrayDestroy(pQueryHandle->pColumns); taosArrayDestroy(pQueryHandle->defaultLoadColumn); tfree(pQueryHandle->pDataBlockInfo); tfree(pQueryHandle->statis); From 41c76edcc3c7e356e181dac66e64c40a8428746e Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Wed, 22 Jul 2020 17:06:53 +0800 Subject: [PATCH 065/100] fix crash in executing compress2.sim --- src/tsdb/src/tsdbRead.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index fc67c64e2e..c82d1f905a 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -188,8 +188,7 @@ TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STab pQueryHandle->allocSize = 0; if (tsdbInitReadHelper(&pQueryHandle->rhelper, (STsdbRepo*) tsdb) != 0) { - free(pQueryHandle); - return NULL; + goto out_of_memory; } tsdbTakeMemSnapshot(pQueryHandle->pTsdb, &pQueryHandle->mem, &pQueryHandle->imem); @@ -202,13 +201,11 @@ TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STab pQueryHandle->statis = calloc(numOfCols, sizeof(SDataStatis)); if (pQueryHandle->statis == NULL) { - tsdbCleanupQueryHandle(pQueryHandle); - return NULL; + goto out_of_memory; } pQueryHandle->pColumns = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); // todo: use list instead of array? if (pQueryHandle->pColumns == NULL) { - tsdbCleanupQueryHandle(pQueryHandle); - return NULL; + goto out_of_memory; } for (int32_t i = 0; i < numOfCols; ++i) { @@ -217,8 +214,7 @@ TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STab colInfo.info = pCond->colList[i]; colInfo.pData = calloc(1, EXTRA_BYTES + pQueryHandle->outputCapacity * pCond->colList[i].bytes); if (colInfo.pData == NULL) { - tsdbCleanupQueryHandle(pQueryHandle); - return NULL; + goto out_of_memory; } taosArrayPush(pQueryHandle->pColumns, &colInfo); pQueryHandle->statis[i].colId = colInfo.info.colId; @@ -226,8 +222,7 @@ TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STab pQueryHandle->pTableCheckInfo = taosArrayInit(groupList->numOfTables, sizeof(STableCheckInfo)); if (pQueryHandle->pTableCheckInfo == NULL) { - tsdbCleanupQueryHandle(pQueryHandle); - return NULL; + goto out_of_memory; } STsdbMeta* pMeta = tsdbGetMeta(tsdb); assert(pMeta != NULL); @@ -263,6 +258,11 @@ TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STab tsdbInitCompBlockLoadInfo(&pQueryHandle->compBlockLoadInfo); return (TsdbQueryHandleT) pQueryHandle; + +out_of_memory: + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + tsdbCleanupQueryHandle(pQueryHandle); + return NULL; } TsdbQueryHandleT tsdbQueryLastRow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, void* qinfo) { From 8428a1afbc85afc734dd4b18fb155f942bb9fe49 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 22 Jul 2020 09:56:08 +0000 Subject: [PATCH 066/100] [TD-926] refactor alter dnode func --- src/client/src/tscSQLParser.c | 44 ++++++++++++++------- src/common/inc/tglobal.h | 1 + src/common/src/tglobal.c | 41 +++++++++++++++++++- src/inc/taoserror.h | 1 + src/inc/tbalance.h | 1 + src/mnode/src/mnodeBalance.c | 1 + src/mnode/src/mnodeDnode.c | 56 +++++++++++++-------------- src/plugins/monitor/src/monitorMain.c | 43 ++++++-------------- 8 files changed, 113 insertions(+), 75 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 09db65a105..32d44dfb81 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -358,7 +358,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { } case TSDB_SQL_CFG_DNODE: { - const char* msg2 = "invalid configure options or values"; + const char* msg2 = "invalid configure options or values, such as resetlog / debugFlag 135 / balance 'vnode:1-dnode:2' / monitor 1 "; const char* msg3 = "invalid dnode ep"; /* validate the ip address */ @@ -4674,26 +4674,42 @@ int32_t validateDNodeConfig(tDCLSQL* pOptions) { return TSDB_CODE_TSC_INVALID_SQL; } - const int DNODE_DYNAMIC_CFG_OPTIONS_SIZE = 19; - const SDNodeDynConfOption DNODE_DYNAMIC_CFG_OPTIONS[] = { - {"resetLog", 8}, {"resetQueryCache", 15}, {"debugFlag", 9}, {"mDebugFlag", 10}, - {"dDebugFlag", 10}, {"sdbDebugFlag", 12}, {"vDebugFlag", 10}, {"cDebugFlag", 10}, - {"httpDebugFlag", 13}, {"monitorDebugFlag", 16}, {"rpcDebugFlag", 12}, {"uDebugFlag", 10}, - {"tmrDebugFlag", 12}, {"qDebugflag", 10}, {"sDebugflag", 10}, {"tsdbDebugFlag", 13}, - {"mqttDebugFlag", 13}, {"wDebugFlag", 10}, {"monitor", 7}}; + const int tokenLogEnd = 2; + const int tokenBalance = 2; + const int tokenMonitor = 3; + const int tokenDebugFlag = 4; + const int tokenDebugFlagEnd = 20; + const SDNodeDynConfOption cfgOptions[] = { + {"resetLog", 8}, {"resetQueryCache", 15}, {"balance", 7}, {"monitor", 7}, + {"debugFlag", 9}, {"monitorDebugFlag", 16}, {"vDebugFlag", 10}, {"mDebugFlag", 10}, + {"cDebugFlag", 10}, {"httpDebugFlag", 13}, {"qDebugflag", 10}, {"sdbDebugFlag", 12}, + {"uDebugFlag", 10}, {"tsdbDebugFlag", 13}, {"sDebugflag", 10}, {"rpcDebugFlag", 12}, + {"dDebugFlag", 10}, {"mqttDebugFlag", 13}, {"wDebugFlag", 10}, {"tmrDebugFlag", 12}, + }; SSQLToken* pOptionToken = &pOptions->a[1]; if (pOptions->nTokens == 2) { // reset log and reset query cache does not need value - for (int32_t i = 0; i < 2; ++i) { - const SDNodeDynConfOption* pOption = &DNODE_DYNAMIC_CFG_OPTIONS[i]; + for (int32_t i = 0; i < tokenLogEnd; ++i) { + const SDNodeDynConfOption* pOption = &cfgOptions[i]; if ((strncasecmp(pOption->name, pOptionToken->z, pOptionToken->n) == 0) && (pOption->len == pOptionToken->n)) { return TSDB_CODE_SUCCESS; } } - } else if ((strncasecmp(DNODE_DYNAMIC_CFG_OPTIONS[DNODE_DYNAMIC_CFG_OPTIONS_SIZE - 1].name, pOptionToken->z, pOptionToken->n) == 0) && - (DNODE_DYNAMIC_CFG_OPTIONS[DNODE_DYNAMIC_CFG_OPTIONS_SIZE - 1].len == pOptionToken->n)) { + } else if ((strncasecmp(cfgOptions[tokenBalance].name, pOptionToken->z, pOptionToken->n) == 0) && + (cfgOptions[tokenBalance].len == pOptionToken->n)) { + SSQLToken* pValToken = &pOptions->a[2]; + int32_t vnodeIndex = 0; + int32_t dnodeIndex = 0; + strdequote(pValToken->z); + bool parseOk = taosCheckBalanceCfgOptions(pValToken->z, &vnodeIndex, &dnodeIndex); + if (!parseOk) { + return TSDB_CODE_TSC_INVALID_SQL; // options value is invalid + } + return TSDB_CODE_SUCCESS; + } else if ((strncasecmp(cfgOptions[tokenMonitor].name, pOptionToken->z, pOptionToken->n) == 0) && + (cfgOptions[tokenMonitor].len == pOptionToken->n)) { SSQLToken* pValToken = &pOptions->a[2]; int32_t val = strtol(pValToken->z, NULL, 10); if (val != 0 && val != 1) { @@ -4709,8 +4725,8 @@ int32_t validateDNodeConfig(tDCLSQL* pOptions) { return TSDB_CODE_TSC_INVALID_SQL; } - for (int32_t i = 2; i < DNODE_DYNAMIC_CFG_OPTIONS_SIZE - 1; ++i) { - const SDNodeDynConfOption* pOption = &DNODE_DYNAMIC_CFG_OPTIONS[i]; + for (int32_t i = tokenDebugFlag; i < tokenDebugFlagEnd; ++i) { + const SDNodeDynConfOption* pOption = &cfgOptions[i]; if ((strncasecmp(pOption->name, pOptionToken->z, pOptionToken->n) == 0) && (pOption->len == pOptionToken->n)) { /* options is valid */ diff --git a/src/common/inc/tglobal.h b/src/common/inc/tglobal.h index 6506707457..7ba96ceb60 100644 --- a/src/common/inc/tglobal.h +++ b/src/common/inc/tglobal.h @@ -174,6 +174,7 @@ bool taosCheckGlobalCfg(); void taosSetAllDebugFlag(); bool taosCfgDynamicOptions(char *msg); int taosGetFqdnPortFromEp(const char *ep, char *fqdn, uint16_t *port); +bool taosCheckBalanceCfgOptions(const char *option, int32_t *vnodeIndex, int32_t *dnodeIndex); #ifdef __cplusplus } diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index 56c63ee49d..c79b016b93 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -198,6 +198,7 @@ int32_t tsdbDebugFlag = 131; int32_t (*monitorStartSystemFp)() = NULL; void (*monitorStopSystemFp)() = NULL; +void (*monitorExecuteSQLFp)(char *sql) = NULL; static pthread_once_t tsInitGlobalCfgOnce = PTHREAD_ONCE_INIT; @@ -252,11 +253,15 @@ bool taosCfgDynamicOptions(char *msg) { if (monitorStartSystemFp) { (*monitorStartSystemFp)(); uInfo("monitor is enabled"); + } else { + uError("monitor can't be updated, for monitor not initialized"); } } else { if (monitorStopSystemFp) { (*monitorStopSystemFp)(); uInfo("monitor is disabled"); + } else { + uError("monitor can't be updated, for monitor not initialized"); } } return true; @@ -276,7 +281,12 @@ bool taosCfgDynamicOptions(char *msg) { } if (strncasecmp(option, "resetQueryCache", 15) == 0) { - uError("reset query cache can't be executed, for monitor not initialized"); + if (monitorExecuteSQLFp) { + (*monitorExecuteSQLFp)("resetQueryCache"); + uInfo("resetquerycache is executed"); + } else { + uError("resetquerycache can't be executed, for monitor not started"); + } } return false; @@ -1300,3 +1310,32 @@ int taosGetFqdnPortFromEp(const char *ep, char *fqdn, uint16_t *port) { return 0; } + +/* + * alter dnode 1 balance "vnode:1-dnode:2" + */ + +bool taosCheckBalanceCfgOptions(const char *option, int32_t *vnodeIndex, int32_t *dnodeIndex) { + int len = strlen(option); + if (strncasecmp(option, "vnode:", 6) != 0) { + return false; + } + + int pos = 0; + for (; pos < len; ++pos) { + if (option[pos] == '-') break; + } + + if (++pos >= len) return false; + if (strncasecmp(option + pos, "dnode:", 6) != 0) { + return false; + } + + *vnodeIndex = strtol(option + 6, NULL, 10); + *dnodeIndex = strtol(option + pos + 6, NULL, 10); + if (*vnodeIndex <= 1 || *dnodeIndex <= 0) { + return false; + } + + return true; +} \ No newline at end of file diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index 3503e39d31..59b2c0220b 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -227,6 +227,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_CPU_LIMITED, 0, 0x080B, "grant cpu // sync TAOS_DEFINE_ERROR(TSDB_CODE_SYN_INVALID_CONFIG, 0, 0x0900, "sync invalid configuration") +TAOS_DEFINE_ERROR(TSDB_CODE_SYN_NOT_ENABLED, 0, 0x0901, "sync module not enabled") // wal TAOS_DEFINE_ERROR(TSDB_CODE_WAL_APP_ERROR, 0, 0x1000, "wal app error") diff --git a/src/inc/tbalance.h b/src/inc/tbalance.h index c52f5afaaa..9ee8d73189 100644 --- a/src/inc/tbalance.h +++ b/src/inc/tbalance.h @@ -29,6 +29,7 @@ void balanceAsyncNotify(); void balanceSyncNotify(); void balanceReset(); int32_t balanceAllocVnodes(struct SVgObj *pVgroup); +int32_t balanceCfgDnode(struct SDnodeObj *pDnode, const char *option); int32_t balanceDropDnode(struct SDnodeObj *pDnode); #ifdef __cplusplus diff --git a/src/mnode/src/mnodeBalance.c b/src/mnode/src/mnodeBalance.c index 23fc10d0bd..d2ec6dd36e 100644 --- a/src/mnode/src/mnodeBalance.c +++ b/src/mnode/src/mnodeBalance.c @@ -28,6 +28,7 @@ void balanceCleanUp() {} void balanceAsyncNotify() {} void balanceSyncNotify() {} void balanceReset() {} +int32_t balanceCfgDnode(struct SDnodeObj *pDnode, const char *option) { return TSDB_CODE_SYN_NOT_ENABLED; } int32_t balanceAllocVnodes(SVgObj *pVgroup) { void * pIter = NULL; diff --git a/src/mnode/src/mnodeDnode.c b/src/mnode/src/mnodeDnode.c index 7edba8662e..26c4b7a3ea 100644 --- a/src/mnode/src/mnodeDnode.c +++ b/src/mnode/src/mnodeDnode.c @@ -277,45 +277,45 @@ static int32_t mnodeProcessCfgDnodeMsg(SMnodeMsg *pMsg) { SCMCfgDnodeMsg *pCmCfgDnode = pMsg->rpcMsg.pCont; if (pCmCfgDnode->ep[0] == 0) { tstrncpy(pCmCfgDnode->ep, tsLocalEp, TSDB_EP_LEN); - } - - int32_t dnodeId = 0; - char* pos = strchr(pCmCfgDnode->ep, ':'); - if (NULL == pos) { - dnodeId = strtol(pCmCfgDnode->ep, NULL, 10); - if (dnodeId <= 0 || dnodeId > 65536) { - mError("failed to cfg dnode, invalid dnodeId:%s", pCmCfgDnode->ep); - return TSDB_CODE_MND_DNODE_NOT_EXIST; - } } - SRpcEpSet epSet = mnodeGetEpSetFromIp(pCmCfgDnode->ep); - if (dnodeId != 0) { - SDnodeObj *pDnode = mnodeGetDnode(dnodeId); + SDnodeObj *pDnode = mnodeGetDnodeByEp(pCmCfgDnode->ep); + if (pDnode == NULL) { + int32_t dnodeId = strtol(pCmCfgDnode->ep, NULL, 10); + if (dnodeId <= 0 || dnodeId > 65536) { + mError("failed to cfg dnode, invalid dnodeEp:%s", pCmCfgDnode->ep); + return TSDB_CODE_MND_DNODE_NOT_EXIST; + } + + pDnode = mnodeGetDnode(dnodeId); if (pDnode == NULL) { mError("failed to cfg dnode, invalid dnodeId:%d", dnodeId); return TSDB_CODE_MND_DNODE_NOT_EXIST; } - epSet = mnodeGetEpSetFromIp(pDnode->dnodeEp); - mnodeDecDnodeRef(pDnode); } - SMDCfgDnodeMsg *pMdCfgDnode = rpcMallocCont(sizeof(SMDCfgDnodeMsg)); - strcpy(pMdCfgDnode->ep, pCmCfgDnode->ep); - strcpy(pMdCfgDnode->config, pCmCfgDnode->config); + SRpcEpSet epSet = mnodeGetEpSetFromIp(pDnode->dnodeEp); + mnodeDecDnodeRef(pDnode); - SRpcMsg rpcMdCfgDnodeMsg = { - .ahandle = 0, - .code = 0, - .msgType = TSDB_MSG_TYPE_MD_CONFIG_DNODE, - .pCont = pMdCfgDnode, - .contLen = sizeof(SMDCfgDnodeMsg) - }; + if (strncasecmp(pCmCfgDnode->config, "balance", 7) == 0) { + return balanceCfgDnode(pDnode, pCmCfgDnode->config + 8); + } else { + SMDCfgDnodeMsg *pMdCfgDnode = rpcMallocCont(sizeof(SMDCfgDnodeMsg)); + strcpy(pMdCfgDnode->ep, pCmCfgDnode->ep); + strcpy(pMdCfgDnode->config, pCmCfgDnode->config); - mInfo("dnode:%s, is configured by %s", pCmCfgDnode->ep, pMsg->pUser->user); - dnodeSendMsgToDnode(&epSet, &rpcMdCfgDnodeMsg); + SRpcMsg rpcMdCfgDnodeMsg = { + .ahandle = 0, + .code = 0, + .msgType = TSDB_MSG_TYPE_MD_CONFIG_DNODE, + .pCont = pMdCfgDnode, + .contLen = sizeof(SMDCfgDnodeMsg) + }; - return TSDB_CODE_SUCCESS; + mInfo("dnode:%s, is configured by %s", pCmCfgDnode->ep, pMsg->pUser->user); + dnodeSendMsgToDnode(&epSet, &rpcMdCfgDnodeMsg); + return TSDB_CODE_SUCCESS; + } } static void mnodeProcessCfgDnodeMsgRsp(SRpcMsg *rpcMsg) { diff --git a/src/plugins/monitor/src/monitorMain.c b/src/plugins/monitor/src/monitorMain.c index e6e8cf982b..0cc28bb82c 100644 --- a/src/plugins/monitor/src/monitorMain.c +++ b/src/plugins/monitor/src/monitorMain.c @@ -27,7 +27,6 @@ #include "dnode.h" #include "monitor.h" - #define monitorFatal(...) { if (monitorDebugFlag & DEBUG_FATAL) { taosPrintLog("MON FATAL ", 255, __VA_ARGS__); }} #define monitorError(...) { if (monitorDebugFlag & DEBUG_ERROR) { taosPrintLog("MON ERROR ", 255, __VA_ARGS__); }} #define monitorWarn(...) { if (monitorDebugFlag & DEBUG_WARN) { taosPrintLog("MON WARN ", 255, __VA_ARGS__); }} @@ -78,6 +77,7 @@ static void monitorStartTimer(); static void monitorSaveSystemInfo(); extern int32_t (*monitorStartSystemFp)(); extern void (*monitorStopSystemFp)(); +extern void (*monitorExecuteSQLFp)(char *sql); static void monitorCheckDiskUsage(void *para, void *unused) { taosGetDisk(); @@ -207,6 +207,7 @@ static void monitorInitDatabase() { taos_query_a(tsMonitorConn.conn, tsMonitorConn.sql, monitorInitDatabaseCb, NULL); } else { tsMonitorConn.state = MONITOR_STATE_INITIALIZED; + monitorExecuteSQLFp = monitorExecuteSQL; monitorInfo("monitor service init success"); monitorStartTimer(); @@ -230,6 +231,7 @@ static void monitorInitDatabaseCb(void *param, TAOS_RES *result, int32_t code) { void monitorStopSystem() { monitorInfo("monitor module is stopped"); + monitorExecuteSQLFp = NULL; tsMonitorConn.state = MONITOR_STATE_STOPPED; if (tsMonitorConn.initTimer != NULL) { taosTmrStopA(&(tsMonitorConn.initTimer)); @@ -248,33 +250,13 @@ static void monitorStartTimer() { taosTmrReset(monitorSaveSystemInfo, tsMonitorInterval * 1000, NULL, tscTmr, &tsMonitorConn.timer); } -static void dnodeMontiorInsertAcctCallback(void *param, TAOS_RES *result, int32_t code) { +static void dnodeMontiorLogCallback(void *param, TAOS_RES *result, int32_t code) { if (code < 0) { - monitorError("monitor:%p, save account info failed, code:%s", tsMonitorConn.conn, tstrerror(code)); + monitorError("monitor:%p, save %s failed, reason:%s", tsMonitorConn.conn, (char *)param, tstrerror(code)); } else if (code == 0) { - monitorError("monitor:%p, save account info failed, affect rows:%d", tsMonitorConn.conn, code); + monitorError("monitor:%p, save %s failed, affect rows:%d", tsMonitorConn.conn, (char *)param, code); } else { - monitorDebug("monitor:%p, save account info success, code:%s", tsMonitorConn.conn, tstrerror(code)); - } -} - -static void dnodeMontiorInsertSysCallback(void *param, TAOS_RES *result, int32_t code) { - if (code < 0) { - monitorError("monitor:%p, save system info failed, code:%s %s", tsMonitorConn.conn, tstrerror(code), tsMonitorConn.sql); - } else if (code == 0) { - monitorError("monitor:%p, save system info failed, affect rows:%d %s", tsMonitorConn.conn, code, tsMonitorConn.sql); - } else { - monitorDebug("monitor:%p, save system info success, code:%s %s", tsMonitorConn.conn, tstrerror(code), tsMonitorConn.sql); - } -} - -static void dnodeMontiorInsertLogCallback(void *param, TAOS_RES *result, int32_t code) { - if (code < 0) { - monitorError("monitor:%p, save log failed, code:%s", tsMonitorConn.conn, tstrerror(code)); - } else if (code == 0) { - monitorError("monitor:%p, save log failed, affect rows:%d", tsMonitorConn.conn, code); - } else { - monitorDebug("monitor:%p, save log info success, code:%s", tsMonitorConn.conn, tstrerror(code)); + monitorDebug("monitor:%p, save %s info success, reason:%s", tsMonitorConn.conn, (char *)param, tstrerror(code)); } } @@ -359,7 +341,7 @@ static void monitorSaveSystemInfo() { pos += monitorBuildReqSql(sql + pos); monitorDebug("monitor:%p, save system info, sql:%s", tsMonitorConn.conn, sql); - taos_query_a(tsMonitorConn.conn, sql, dnodeMontiorInsertSysCallback, "log"); + taos_query_a(tsMonitorConn.conn, sql, dnodeMontiorLogCallback, "sys"); if (tsMonitorConn.timer != NULL && tsMonitorConn.state != MONITOR_STATE_STOPPED) { monitorStartTimer(); @@ -397,7 +379,7 @@ void monitorSaveAcctLog(SAcctMonitorObj *pMon) { pMon->accessState); monitorDebug("monitor:%p, save account info, sql %s", tsMonitorConn.conn, sql); - taos_query_a(tsMonitorConn.conn, sql, dnodeMontiorInsertAcctCallback, "account"); + taos_query_a(tsMonitorConn.conn, sql, dnodeMontiorLogCallback, "account"); } void monitorSaveLog(int32_t level, const char *const format, ...) { @@ -421,14 +403,11 @@ void monitorSaveLog(int32_t level, const char *const format, ...) { sql[len++] = 0; monitorDebug("monitor:%p, save log, sql: %s", tsMonitorConn.conn, sql); - taos_query_a(tsMonitorConn.conn, sql, dnodeMontiorInsertLogCallback, "log"); + taos_query_a(tsMonitorConn.conn, sql, dnodeMontiorLogCallback, "log"); } void monitorExecuteSQL(char *sql) { if (tsMonitorConn.state != MONITOR_STATE_INITIALIZED) return; - monitorDebug("monitor:%p, execute sql: %s", tsMonitorConn.conn, sql); - - // bug while insert binary - // taos_query_a(tsMonitorConn.conn, sql, NULL, NULL); + taos_query_a(tsMonitorConn.conn, sql, dnodeMontiorLogCallback, "sql"); } From 8b1451b3eabd0c9ba2022e1811460fc652007f8e Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 22 Jul 2020 10:01:54 +0000 Subject: [PATCH 067/100] scripts --- tests/script/general/alter/dnode.sim | 71 ++++++++++++++++++++++++++++ tests/script/jenkins/basic.txt | 1 + 2 files changed, 72 insertions(+) create mode 100644 tests/script/general/alter/dnode.sim diff --git a/tests/script/general/alter/dnode.sim b/tests/script/general/alter/dnode.sim new file mode 100644 index 0000000000..20ce879979 --- /dev/null +++ b/tests/script/general/alter/dnode.sim @@ -0,0 +1,71 @@ +system sh/stop_dnodes.sh + +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c walLevel -v 2 +system sh/exec.sh -n dnode1 -s start + +sleep 3000 +sql connect + +print ======== step1 +sql alter dnode 1 resetlog +sql alter dnode 1 monitor 1 + +sleep 5000 +sql select * from log.dn +if $rows <= 0 then + return -1 +endi + +print ======== step2 + +sql alter dnode 1 resetquerycache +sql alter dnode 1 debugFlag 135 +sql alter dnode 1 debugFlag 131 +sql alter dnode 1 monitor 0 +sql alter dnode 1 debugFlag 135 +sql alter dnode 1 monitorDebugFlag 135 +sql alter dnode 1 vDebugFlag 135 +sql alter dnode 1 mDebugFlag 135 +sql alter dnode 1 cDebugFlag 135 +sql alter dnode 1 httpDebugFlag 135 +sql alter dnode 1 qDebugflag 135 +sql alter dnode 1 sdbDebugFlag 135 +sql alter dnode 1 uDebugFlag 135 +sql alter dnode 1 tsdbDebugFlag 135 +sql alter dnode 1 sDebugflag 135 +sql alter dnode 1 rpcDebugFlag 135 +sql alter dnode 1 dDebugFlag 135 +sql alter dnode 1 mqttDebugFlag 135 +sql alter dnode 1 wDebugFlag 135 +sql alter dnode 1 tmrDebugFlag 135 +sql_error alter dnode 2 wDebugFlag 135 +sql_error alter dnode 2 tmrDebugFlag 135 + +print ======== step3 +sql_error alter $hostname1 debugFlag 135 +sql_error alter $hostname1 monitorDebugFlag 135 +sql_error alter $hostname1 vDebugFlag 135 +sql_error alter $hostname1 mDebugFlag 135 +sql_error alter dnode $hostname2 debugFlag 135 +sql_error alter dnode $hostname2 monitorDebugFlag 135 +sql_error alter dnode $hostname2 vDebugFlag 135 +sql_error alter dnode $hostname2 mDebugFlag 135 +sql alter dnode $hostname1 debugFlag 135 +sql alter dnode $hostname1 monitorDebugFlag 135 +sql alter dnode $hostname1 vDebugFlag 135 +sql alter dnode $hostname1 tmrDebugFlag 131 + +print ======== step4 +sql_error sql alter dnode 1 balance 0 +sql_error sql alter dnode 1 balance vnode:1-dnode:1 +sql_error sql alter dnode 1 balance "vnode:1" +sql_error sql alter dnode 1 balance "vnode:1-dnode:1" +sql_error sql alter dnode 1 balance "dnode:1-vnode:1" +sql_error sql alter dnode 1 balance "dnode:1-" +sql_error sql alter dnode 1 balance "vnode:2-dnod" +sql alter dnode 1 balance "vnode:2-dnode:1" -x step4 +step4: + +print ======= over +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 3b9806558e..2fd5db0165 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -3,6 +3,7 @@ cd ../../../debug; make ./test.sh -f general/alter/cached_schema_after_alter.sim ./test.sh -f general/alter/count.sim +./test.sh -f general/alter/dnode.sim ./test.sh -f general/alter/import.sim ./test.sh -f general/alter/insert1.sim ./test.sh -f general/alter/insert2.sim From 63709430bfdbd8fdf488c340fc32612d445d3fc2 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 22 Jul 2020 16:34:43 +0000 Subject: [PATCH 068/100] TD-884: fixbug --- src/client/src/tscServer.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 9a1fa77629..6c1faf6f9d 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -106,13 +106,14 @@ static void tscUpdateVgroupInfo(SSqlObj *pObj, SRpcEpSet *pEpSet) { SCMCorVgroupInfo *pVgroupInfo = &pTableMetaInfo->pTableMeta->corVgroupInfo; taosCorBeginWrite(&pVgroupInfo->version); - //TODO(dengyihao), dont care vgid + tscDebug("before: Endpoint in use: %d", pVgroupInfo->inUse); pVgroupInfo->inUse = pEpSet->inUse; pVgroupInfo->numOfEps = pEpSet->numOfEps; - for (int32_t i = 0; pVgroupInfo->numOfEps; i++) { + for (int32_t i = 0; i < pVgroupInfo->numOfEps; i++) { strncpy(pVgroupInfo->epAddr[i].fqdn, pEpSet->fqdn[i], TSDB_FQDN_LEN); pVgroupInfo->epAddr[i].port = pEpSet->port[i]; } + tscDebug("after: EndPoint in use: %d", pVgroupInfo->inUse); taosCorEndWrite(&pVgroupInfo->version); } void tscPrintMgmtEp() { @@ -283,9 +284,7 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) { } if (pEpSet) { - //SRpcEpSet dump; - tscEpSetHtons(pEpSet); - if (tscEpSetIsEqual(&pSql->epSet, pEpSet)) { + if (!tscEpSetIsEqual(&pSql->epSet, pEpSet)) { if(pCmd->command < TSDB_SQL_MGMT) { tscUpdateVgroupInfo(pSql, pEpSet); } else { From 8e637b4f7024dbb1d79148f051941b1549f9c6e7 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 23 Jul 2020 10:13:12 +0800 Subject: [PATCH 069/100] TD-982 --- src/tsdb/src/tsdbMemTable.c | 5 ++--- src/tsdb/src/tsdbRWHelper.c | 15 +++++++++++++-- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index 5325eb5b0c..7708646a61 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -264,13 +264,11 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey } do { - if (numOfRows >= maxRowsToRead) break; - SDataRow row = tsdbNextIterRow(pIter); if (row == NULL) break; keyNext = dataRowKey(row); - if (keyNext < 0 || keyNext > maxKey) break; + if (keyNext > maxKey) break; bool keyFiltered = false; if (nFilterKeys != 0) { @@ -289,6 +287,7 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey } if (!keyFiltered) { + if (numOfRows >= maxRowsToRead) break; if (pCols) { if (pSchema == NULL || schemaVersion(pSchema) != dataRowVersion(row)) { pSchema = tsdbGetTableSchemaImpl(pTable, false, false, dataRowVersion(row)); diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index f8792469cb..3063f22a28 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -1477,6 +1477,11 @@ static int tsdbProcessAppendCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, if (tsdbInsertSuperBlock(pHelper, &compBlock, pIdx->numOfBlocks) < 0) return -1; } +#ifndef NDEBUG + TSKEY keyNext = tsdbNextIterKey(pCommitIter->pIter); + ASSERT(keyNext < 0 || keyNext > pIdx->maxKey); +#endif + return 0; } @@ -1561,11 +1566,12 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, if (tsdbInsertSuperBlock(pHelper, &compBlock, tblkIdx) < 0) return -1; tblkIdx++; } + ASSERT(tblkIdx == 0 || (tsdbNextIterKey(pCommitIter->pIter) < 0 || + tsdbNextIterKey(pCommitIter->pIter) > blockAtIdx(pHelper, tblkIdx - 1)->keyLast)); } else { ASSERT(keyFirst <= blkKeyLast); int16_t colId = 0; if (tsdbLoadBlockDataCols(pHelper, pCompBlock, NULL, &colId, 1) < 0) return -1; - ASSERT(pDataCols0->numOfRows == pCompBlock->numOfRows); slIter = *(pCommitIter->pIter); int rows1 = (pCfg->maxRowsPerFileBlock - pCompBlock->numOfRows); @@ -1574,9 +1580,10 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, if (rows2 == 0) { // all filtered out *(pCommitIter->pIter) = slIter; + ASSERT(tblkIdx == 0 || (tsdbNextIterKey(pCommitIter->pIter) < 0 || + tsdbNextIterKey(pCommitIter->pIter) > blockAtIdx(pHelper, tblkIdx - 1)->keyLast)); } else { int rows3 = tsdbLoadDataFromCache(pTable, &slIter, keyLimit, INT_MAX, NULL, NULL, 0) + rows2; - ASSERT(rows3 >= rows2); if (pCompBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS && rows1 >= rows2) { int rows = (rows1 >= rows3) ? rows3 : rows2; @@ -1588,6 +1595,8 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, return -1; if (tsdbAddSubBlock(pHelper, &compBlock, tblkIdx, rowsRead) < 0) return -1; tblkIdx++; + ASSERT(tblkIdx == 0 || (tsdbNextIterKey(pCommitIter->pIter) < 0 || + tsdbNextIterKey(pCommitIter->pIter) > blockAtIdx(pHelper, tblkIdx - 1)->keyLast)); } else { if (tsdbLoadBlockData(pHelper, pCompBlock, NULL) < 0) return -1; int round = 0; @@ -1608,6 +1617,8 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, round++; tblkIdx++; } + ASSERT(tblkIdx == 0 || (tsdbNextIterKey(pCommitIter->pIter) < 0 || + tsdbNextIterKey(pCommitIter->pIter) > blockAtIdx(pHelper, tblkIdx - 1)->keyLast)); } } } From fd7b5e6e8655e231d9fecbddf886db3cffdb500c Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 23 Jul 2020 13:29:14 +0800 Subject: [PATCH 070/100] scripts --- src/mnode/src/mnodeVgroup.c | 34 +++++-------------- tests/script/jenkins/basic.txt | 3 +- tests/script/sh/deploy.sh | 2 +- .../arbitrator/dn3_mn1_r2_vnode_delDir.sim | 12 +++---- .../arbitrator/dn3_mn1_r3_vnode_delDir.sim | 16 ++++----- .../arbitrator/dn3_mn1_replica_change.sim | 16 ++++----- .../arbitrator/dn3_mn1_vnode_change.sim | 12 +++---- .../dn3_mn1_vnode_corruptFile_offline.sim | 16 ++++----- .../dn3_mn1_vnode_corruptFile_online.sim | 12 +++---- .../dn3_mn1_vnode_createErrData_online.sim | 12 +++---- .../arbitrator/dn3_mn1_vnode_delDir.sim | 16 ++++----- .../dn3_mn1_vnode_noCorruptFile_offline.sim | 16 ++++----- .../arbitrator/dn3_mn1_vnode_nomaster.sim | 8 ++--- .../offline_replica2_alterTable_online.sim | 4 +-- .../offline_replica2_alterTag_online.sim | 4 +-- .../offline_replica2_createTable_online.sim | 4 +-- .../offline_replica2_dropDb_online.sim | 4 +-- .../offline_replica2_dropTable_online.sim | 4 +-- .../offline_replica3_alterTable_online.sim | 4 +-- .../offline_replica3_alterTag_online.sim | 4 +-- .../offline_replica3_createTable_online.sim | 4 +-- .../offline_replica3_dropDb_online.sim | 4 +-- .../offline_replica3_dropTable_online.sim | 4 +-- .../sync_replica2_alterTable_add.sim | 4 +-- .../sync_replica2_alterTable_drop.sim | 4 +-- .../arbitrator/sync_replica2_dropDb.sim | 4 +-- .../arbitrator/sync_replica2_dropTable.sim | 4 +-- .../sync_replica3_alterTable_add.sim | 4 +-- .../sync_replica3_alterTable_drop.sim | 4 +-- .../arbitrator/sync_replica3_createTable.sim | 4 +-- ...ca3_dnodeChang_DropAddAlterTableDropDb.sim | 28 +++++++-------- .../arbitrator/sync_replica3_dropDb.sim | 4 +-- .../arbitrator/sync_replica3_dropTable.sim | 4 +-- .../migrate/mn2_vn2_repl2_rmMnodeDir.sim | 12 +++---- .../migrate/mn2_vn2_repl2_rmMnodeVnodeDir.sim | 12 +++---- ..._repl2_rmMnodeVnodeDir_stopAll_starAll.sim | 12 +++---- .../migrate/mn2_vn2_repl2_rmVnodeDir.sim | 12 +++---- 37 files changed, 154 insertions(+), 173 deletions(-) diff --git a/src/mnode/src/mnodeVgroup.c b/src/mnode/src/mnodeVgroup.c index 46255a6c8f..089168c31e 100644 --- a/src/mnode/src/mnodeVgroup.c +++ b/src/mnode/src/mnodeVgroup.c @@ -592,7 +592,7 @@ static int32_t mnodeGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *p pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; - pShow->bytes[cols] = 12 + VARSTR_HEADER_SIZE; + pShow->bytes[cols] = 8 + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "status"); pSchema[cols].bytes = htons(pShow->bytes[cols]); @@ -619,12 +619,6 @@ static int32_t mnodeGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *p pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; - pShow->bytes[cols] = 40 + VARSTR_HEADER_SIZE; - pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "end_point"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); - cols++; - pShow->bytes[cols] = 9 + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "vstatus"); @@ -716,27 +710,15 @@ static int32_t mnodeRetrieveVgroups(SShowObj *pShow, char *data, int32_t rows, v *(int16_t *) pWrite = pVgroup->vnodeGid[i].dnodeId; cols++; - SDnodeObj *pDnode = pVgroup->vnodeGid[i].pDnode; - + SDnodeObj * pDnode = pVgroup->vnodeGid[i].pDnode; + const char *role = "NULL"; if (pDnode != NULL) { - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pDnode->dnodeEp, pShow->bytes[cols]); - cols++; - - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - char *role = mnodeGetMnodeRoleStr(pVgroup->vnodeGid[i].role); - STR_WITH_MAXSIZE_TO_VARSTR(pWrite, role, pShow->bytes[cols]); - cols++; - } else { - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - const char *src = "NULL"; - STR_WITH_SIZE_TO_VARSTR(pWrite, src, strlen(src)); - cols++; - - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - STR_WITH_SIZE_TO_VARSTR(pWrite, src, strlen(src)); - cols++; + role = mnodeGetMnodeRoleStr(pVgroup->vnodeGid[i].role); } + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, role, pShow->bytes[cols]); + cols++; } mnodeDecVgroupRef(pVgroup); diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 2fd5db0165..828b907b9c 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -327,10 +327,9 @@ cd ../../../debug; make ./test.sh -f unique/arbitrator/check_cluster_cfg_para.sim #./test.sh -f unique/arbitrator/dn2_mn1_cache_file_sync.sim ./test.sh -f unique/arbitrator/dn3_mn1_full_createTableFail.sim -./test.sh -f unique/arbitrator/dn3_mn1_full_dropDnodeFail.sim ./test.sh -f unique/arbitrator/dn3_mn1_multiCreateDropTable.sim ./test.sh -f unique/arbitrator/dn3_mn1_nw_disable_timeout_autoDropDnode.sim -./test.sh -f unique/arbitrator/dn3_mn1_replica2_wal1_AddDelDnode.sim +#./test.sh -f unique/arbitrator/dn3_mn1_replica2_wal1_AddDelDnode.sim ./test.sh -f unique/arbitrator/dn3_mn1_replica_change_dropDnod.sim ./test.sh -f unique/arbitrator/dn3_mn1_replica_change.sim ./test.sh -f unique/arbitrator/dn3_mn1_stopDnode_timeout.sim diff --git a/tests/script/sh/deploy.sh b/tests/script/sh/deploy.sh index adbb8207be..ac65237319 100755 --- a/tests/script/sh/deploy.sh +++ b/tests/script/sh/deploy.sh @@ -125,7 +125,7 @@ echo "mqttDebugFlag 135" >> $TAOS_CFG echo "qdebugFlag 135" >> $TAOS_CFG echo "rpcDebugFlag 135" >> $TAOS_CFG echo "tmrDebugFlag 131" >> $TAOS_CFG -echo "udebugFlag 143" >> $TAOS_CFG +echo "udebugFlag 135" >> $TAOS_CFG echo "sdebugFlag 135" >> $TAOS_CFG echo "wdebugFlag 135" >> $TAOS_CFG echo "monitor 0" >> $TAOS_CFG diff --git a/tests/script/unique/arbitrator/dn3_mn1_r2_vnode_delDir.sim b/tests/script/unique/arbitrator/dn3_mn1_r2_vnode_delDir.sim index 3df21a1c5d..b625619678 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_r2_vnode_delDir.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_r2_vnode_delDir.sim @@ -137,8 +137,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode3Vtatus = $data6_2 -$dnode2Vtatus = $data9_2 +$dnode3Vtatus = $data5_2 +$dnode2Vtatus = $data7_2 if $dnode3Vtatus != offline then sleep 2000 @@ -204,8 +204,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode3Vtatus = $data6_2 -$dnode2Vtatus = $data9_2 +$dnode3Vtatus = $data5_2 +$dnode2Vtatus = $data7_2 print dnode2Vtatus: $dnode3Vtatus print dnode3Vtatus: $dnode3Vtatus @@ -319,8 +319,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode3Vtatus = $data6_2 -$dnode2Vtatus = $data9_2 +$dnode3Vtatus = $data5_2 +$dnode2Vtatus = $data7_2 print dnode4Vtatus: $dnode4Vtatus print dnode3Vtatus: $dnode3Vtatus diff --git a/tests/script/unique/arbitrator/dn3_mn1_r3_vnode_delDir.sim b/tests/script/unique/arbitrator/dn3_mn1_r3_vnode_delDir.sim index 97b77aa66f..c647a35a5c 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_r3_vnode_delDir.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_r3_vnode_delDir.sim @@ -139,8 +139,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data6_2 -$dnode3Vtatus = $data9_2 +$dnode4Vtatus = $data5_2 +$dnode3Vtatus = $data7_2 if $dnode4Vtatus != offline then sleep 2000 @@ -206,8 +206,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data6_2 -$dnode3Vtatus = $data9_2 +$dnode4Vtatus = $data5_2 +$dnode3Vtatus = $data7_2 print dnode4Vtatus: $dnode4Vtatus print dnode3Vtatus: $dnode3Vtatus @@ -325,8 +325,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data6_2 -$dnode3Vtatus = $data9_2 +$dnode4Vtatus = $data5_2 +$dnode3Vtatus = $data7_2 print dnode4Vtatus: $dnode4Vtatus print dnode3Vtatus: $dnode3Vtatus @@ -386,8 +386,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data6_2 -$dnode3Vtatus = $data9_2 +$dnode4Vtatus = $data5_2 +$dnode3Vtatus = $data7_2 print dnode4Vtatus: $dnode4Vtatus print dnode3Vtatus: $dnode3Vtatus diff --git a/tests/script/unique/arbitrator/dn3_mn1_replica_change.sim b/tests/script/unique/arbitrator/dn3_mn1_replica_change.sim index 95fb21a017..23ace47e3c 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_replica_change.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_replica_change.sim @@ -358,10 +358,10 @@ print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $dat print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 $data5_4 $data6_4 $data7_4 $data8_4 $data9_4 $data10_4 #print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5 #print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6 -$thirdDnode_2 = $data10_1 -$thirdDnode_3 = $data10_2 -$thirdDnode_4 = $data10_3 -$thirdDnode_5 = $data10_4 +$thirdDnode_2 = $data8_1 +$thirdDnode_3 = $data8_2 +$thirdDnode_4 = $data8_3 +$thirdDnode_5 = $data8_4 if $thirdDnode_2 != null then sleep 2000 @@ -405,10 +405,10 @@ print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $dat print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 $data5_4 $data6_4 $data7_4 $data8_4 #print $data0_5 $data1_5 $data2_5 $data3_5 $data4_5 #print $data0_6 $data1_6 $data2_6 $data3_6 $data4_6 -$sencodDnode_2 = $data7_1 -$sencodDnode_3 = $data7_2 -$sencodDnode_4 = $data7_3 -$sencodDnode_5 = $data7_4 +$sencodDnode_2 = $data6_1 +$sencodDnode_3 = $data6_2 +$sencodDnode_4 = $data6_3 +$sencodDnode_5 = $data6_4 if $sencodDnode_2 != null then sleep 2000 diff --git a/tests/script/unique/arbitrator/dn3_mn1_vnode_change.sim b/tests/script/unique/arbitrator/dn3_mn1_vnode_change.sim index ee974ad6db..b18edf0670 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_vnode_change.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_vnode_change.sim @@ -138,8 +138,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data6_2 -$dnode3Vtatus = $data9_2 +$dnode4Vtatus = $data5_2 +$dnode3Vtatus = $data7_2 if $dnode4Vtatus != offline then sleep 2000 @@ -213,8 +213,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data6_2 -$dnode3Vtatus = $data9_2 +$dnode4Vtatus = $data5_2 +$dnode3Vtatus = $data7_2 print dnode4Vtatus: $dnode4Vtatus print dnode3Vtatus: $dnode3Vtatus @@ -287,8 +287,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data6_2 -$dnode3Vtatus = $data9_2 +$dnode4Vtatus = $data5_2 +$dnode3Vtatus = $data7_2 print dnode4Vtatus: $dnode4Vtatus print dnode3Vtatus: $dnode3Vtatus diff --git a/tests/script/unique/arbitrator/dn3_mn1_vnode_corruptFile_offline.sim b/tests/script/unique/arbitrator/dn3_mn1_vnode_corruptFile_offline.sim index 6a8d69c271..3b208e6f1a 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_vnode_corruptFile_offline.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_vnode_corruptFile_offline.sim @@ -144,8 +144,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode3Vtatus = $data6_2 -$dnode2Vtatus = $data9_2 +$dnode3Vtatus = $data5_2 +$dnode2Vtatus = $data7_2 if $dnode3Vtatus != offline then sleep 2000 @@ -234,8 +234,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode2Vtatus = $data9_2 -$dnode3Vtatus = $data6_2 +$dnode2Vtatus = $data5_2 +$dnode3Vtatus = $data7_2 if $dnode2Vtatus != master then sleep 2000 @@ -313,8 +313,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode2Vtatus = $data9_2 -$dnode3Vtatus = $data6_2 +$dnode2Vtatus = $data7_2 +$dnode3Vtatus = $data5_2 if $dnode2Vtatus != offline then sleep 2000 @@ -393,8 +393,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode2Vtatus = $data9_2 -$dnode3Vtatus = $data6_2 +$dnode2Vtatus = $data7_2 +$dnode3Vtatus = $data5_2 if $dnode2Vtatus == offline then sleep 2000 diff --git a/tests/script/unique/arbitrator/dn3_mn1_vnode_corruptFile_online.sim b/tests/script/unique/arbitrator/dn3_mn1_vnode_corruptFile_online.sim index 1642e8a322..4f737b9c40 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_vnode_corruptFile_online.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_vnode_corruptFile_online.sim @@ -196,8 +196,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode2Vtatus = $data9_2 -$dnode3Vtatus = $data6_2 +$dnode2Vtatus = $data7_2 +$dnode3Vtatus = $data5_2 if $dnode2Vtatus != offline then sleep 2000 @@ -269,8 +269,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode2Vtatus = $data9_2 -$dnode3Vtatus = $data6_2 +$dnode2Vtatus = $data7_2 +$dnode3Vtatus = $data5_2 if $dnode2Vtatus != offline then sleep 2000 @@ -325,8 +325,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode2Vtatus = $data9_2 -$dnode3Vtatus = $data6_2 +$dnode2Vtatus = $data7_2 +$dnode3Vtatus = $data5_2 if $dnode2Vtatus != slave then sleep 2000 diff --git a/tests/script/unique/arbitrator/dn3_mn1_vnode_createErrData_online.sim b/tests/script/unique/arbitrator/dn3_mn1_vnode_createErrData_online.sim index 59a62aa63d..f50081ea70 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_vnode_createErrData_online.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_vnode_createErrData_online.sim @@ -158,8 +158,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode2Vtatus = $data9_2 -$dnode3Vtatus = $data6_2 +$dnode2Vtatus = $data7_2 +$dnode3Vtatus = $data5_2 if $dnode2Vtatus != offline then sleep 2000 @@ -231,8 +231,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode2Vtatus = $data9_2 -$dnode3Vtatus = $data6_2 +$dnode2Vtatus = $data7_2 +$dnode3Vtatus = $data5_2 if $dnode2Vtatus != offline then sleep 2000 @@ -287,8 +287,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode2Vtatus = $data9_2 -$dnode3Vtatus = $data6_2 +$dnode2Vtatus = $data7_2 +$dnode3Vtatus = $data5_2 if $dnode2Vtatus != slave then sleep 2000 diff --git a/tests/script/unique/arbitrator/dn3_mn1_vnode_delDir.sim b/tests/script/unique/arbitrator/dn3_mn1_vnode_delDir.sim index a0629abab8..3582a3b7b2 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_vnode_delDir.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_vnode_delDir.sim @@ -141,8 +141,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data6_2 -$dnode3Vtatus = $data9_2 +$dnode4Vtatus = $data5_2 +$dnode3Vtatus = $data7_2 if $dnode4Vtatus != offline then sleep 2000 @@ -201,8 +201,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data6_2 -$dnode3Vtatus = $data9_2 +$dnode4Vtatus = $data5_2 +$dnode3Vtatus = $data7_2 print dnode4Vtatus: $dnode4Vtatus print dnode3Vtatus: $dnode3Vtatus @@ -301,8 +301,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data6_2 -$dnode3Vtatus = $data9_2 +$dnode4Vtatus = $data5_2 +$dnode3Vtatus = $data7_2 print dnode4Vtatus: $dnode4Vtatus print dnode3Vtatus: $dnode3Vtatus @@ -417,8 +417,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data6_2 -$dnode3Vtatus = $data9_2 +$dnode4Vtatus = $data5_2 +$dnode3Vtatus = $data7_2 print dnode4Vtatus: $dnode4Vtatus print dnode3Vtatus: $dnode3Vtatus diff --git a/tests/script/unique/arbitrator/dn3_mn1_vnode_noCorruptFile_offline.sim b/tests/script/unique/arbitrator/dn3_mn1_vnode_noCorruptFile_offline.sim index e2fa1fbe2f..4040318801 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_vnode_noCorruptFile_offline.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_vnode_noCorruptFile_offline.sim @@ -144,8 +144,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode3Vtatus = $data6_2 -$dnode2Vtatus = $data9_2 +$dnode3Vtatus = $data5_2 +$dnode2Vtatus = $data7_2 if $dnode3Vtatus != offline then sleep 2000 @@ -234,8 +234,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode2Vtatus = $data9_2 -$dnode3Vtatus = $data6_2 +$dnode2Vtatus = $data7_2 +$dnode3Vtatus = $data5_2 if $dnode2Vtatus != master then sleep 2000 @@ -313,8 +313,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode2Vtatus = $data9_2 -$dnode3Vtatus = $data6_2 +$dnode2Vtatus = $data7_2 +$dnode3Vtatus = $data5_2 if $dnode2Vtatus != offline then sleep 2000 @@ -392,8 +392,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode2Vtatus = $data9_2 -$dnode3Vtatus = $data6_2 +$dnode2Vtatus = $data7_2 +$dnode3Vtatus = $data5_2 if $dnode2Vtatus == offline then sleep 2000 diff --git a/tests/script/unique/arbitrator/dn3_mn1_vnode_nomaster.sim b/tests/script/unique/arbitrator/dn3_mn1_vnode_nomaster.sim index a425bc3e92..53209b0d5a 100644 --- a/tests/script/unique/arbitrator/dn3_mn1_vnode_nomaster.sim +++ b/tests/script/unique/arbitrator/dn3_mn1_vnode_nomaster.sim @@ -151,8 +151,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data6_2 -$dnode3Vtatus = $data9_2 +$dnode4Vtatus = $data5_2 +$dnode3Vtatus = $data7_2 if $dnode4Vtatus != offline then sleep 2000 @@ -259,8 +259,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode3Vtatus = $data6_3 -$dnode2Vtatus = $data9_3 +$dnode3Vtatus = $data5_3 +$dnode2Vtatus = $data7_3 if $dnode3Vtatus != offline then sleep 2000 diff --git a/tests/script/unique/arbitrator/offline_replica2_alterTable_online.sim b/tests/script/unique/arbitrator/offline_replica2_alterTable_online.sim index 12d0a19f73..c11bfa62f9 100644 --- a/tests/script/unique/arbitrator/offline_replica2_alterTable_online.sim +++ b/tests/script/unique/arbitrator/offline_replica2_alterTable_online.sim @@ -139,8 +139,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data6_2 -$dnode3Vtatus = $data9_2 +$dnode4Vtatus = $data5_2 +$dnode3Vtatus = $data7_2 if $dnode4Vtatus != offline then sleep 2000 diff --git a/tests/script/unique/arbitrator/offline_replica2_alterTag_online.sim b/tests/script/unique/arbitrator/offline_replica2_alterTag_online.sim index be032dfa76..bed5cefa76 100644 --- a/tests/script/unique/arbitrator/offline_replica2_alterTag_online.sim +++ b/tests/script/unique/arbitrator/offline_replica2_alterTag_online.sim @@ -143,8 +143,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data6_2 -$dnode3Vtatus = $data9_2 +$dnode4Vtatus = $data5_2 +$dnode3Vtatus = $data7_2 if $dnode4Vtatus != offline then sleep 2000 diff --git a/tests/script/unique/arbitrator/offline_replica2_createTable_online.sim b/tests/script/unique/arbitrator/offline_replica2_createTable_online.sim index d79ba724f7..9fdb70142b 100644 --- a/tests/script/unique/arbitrator/offline_replica2_createTable_online.sim +++ b/tests/script/unique/arbitrator/offline_replica2_createTable_online.sim @@ -143,8 +143,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data6_2 -$dnode3Vtatus = $data9_2 +$dnode4Vtatus = $data5_2 +$dnode3Vtatus = $data7_2 if $dnode4Vtatus != offline then sleep 2000 diff --git a/tests/script/unique/arbitrator/offline_replica2_dropDb_online.sim b/tests/script/unique/arbitrator/offline_replica2_dropDb_online.sim index 20f4552fe3..6752e0345d 100644 --- a/tests/script/unique/arbitrator/offline_replica2_dropDb_online.sim +++ b/tests/script/unique/arbitrator/offline_replica2_dropDb_online.sim @@ -143,8 +143,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data6_2 -$dnode3Vtatus = $data9_2 +$dnode4Vtatus = $data5_2 +$dnode3Vtatus = $data7_2 if $dnode4Vtatus != offline then sleep 2000 diff --git a/tests/script/unique/arbitrator/offline_replica2_dropTable_online.sim b/tests/script/unique/arbitrator/offline_replica2_dropTable_online.sim index 7131df5d24..1280cfe7a0 100644 --- a/tests/script/unique/arbitrator/offline_replica2_dropTable_online.sim +++ b/tests/script/unique/arbitrator/offline_replica2_dropTable_online.sim @@ -143,8 +143,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data6_2 -$dnode3Vtatus = $data9_2 +$dnode4Vtatus = $data5_2 +$dnode3Vtatus = $data7_2 if $dnode4Vtatus != offline then sleep 2000 diff --git a/tests/script/unique/arbitrator/offline_replica3_alterTable_online.sim b/tests/script/unique/arbitrator/offline_replica3_alterTable_online.sim index 48eb9512c8..27c2ba4328 100644 --- a/tests/script/unique/arbitrator/offline_replica3_alterTable_online.sim +++ b/tests/script/unique/arbitrator/offline_replica3_alterTable_online.sim @@ -143,8 +143,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data6_2 -$dnode3Vtatus = $data9_2 +$dnode4Vtatus = $data5_2 +$dnode3Vtatus = $data7_2 if $dnode4Vtatus != offline then sleep 2000 diff --git a/tests/script/unique/arbitrator/offline_replica3_alterTag_online.sim b/tests/script/unique/arbitrator/offline_replica3_alterTag_online.sim index 13d88f0f38..86fb51cfa9 100644 --- a/tests/script/unique/arbitrator/offline_replica3_alterTag_online.sim +++ b/tests/script/unique/arbitrator/offline_replica3_alterTag_online.sim @@ -143,8 +143,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data6_2 -$dnode3Vtatus = $data9_2 +$dnode4Vtatus = $data5_2 +$dnode3Vtatus = $data7_2 if $dnode4Vtatus != offline then sleep 2000 diff --git a/tests/script/unique/arbitrator/offline_replica3_createTable_online.sim b/tests/script/unique/arbitrator/offline_replica3_createTable_online.sim index c96d7f5b6e..4dcc5977f3 100644 --- a/tests/script/unique/arbitrator/offline_replica3_createTable_online.sim +++ b/tests/script/unique/arbitrator/offline_replica3_createTable_online.sim @@ -144,8 +144,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data6_2 -$dnode3Vtatus = $data9_2 +$dnode4Vtatus = $data5_2 +$dnode3Vtatus = $data7_2 if $dnode4Vtatus != offline then sleep 2000 diff --git a/tests/script/unique/arbitrator/offline_replica3_dropDb_online.sim b/tests/script/unique/arbitrator/offline_replica3_dropDb_online.sim index 1392e6c8a7..4f4cf6a263 100644 --- a/tests/script/unique/arbitrator/offline_replica3_dropDb_online.sim +++ b/tests/script/unique/arbitrator/offline_replica3_dropDb_online.sim @@ -143,8 +143,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data6_2 -$dnode3Vtatus = $data9_2 +$dnode4Vtatus = $data5_2 +$dnode3Vtatus = $data7_2 if $dnode4Vtatus != offline then sleep 2000 diff --git a/tests/script/unique/arbitrator/offline_replica3_dropTable_online.sim b/tests/script/unique/arbitrator/offline_replica3_dropTable_online.sim index 729aff56c0..98946c4b7c 100644 --- a/tests/script/unique/arbitrator/offline_replica3_dropTable_online.sim +++ b/tests/script/unique/arbitrator/offline_replica3_dropTable_online.sim @@ -143,8 +143,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data6_2 -$dnode3Vtatus = $data9_2 +$dnode4Vtatus = $data5_2 +$dnode3Vtatus = $data7_2 if $dnode4Vtatus != offline then sleep 2000 diff --git a/tests/script/unique/arbitrator/sync_replica2_alterTable_add.sim b/tests/script/unique/arbitrator/sync_replica2_alterTable_add.sim index 25da315be4..dc8b1d015d 100644 --- a/tests/script/unique/arbitrator/sync_replica2_alterTable_add.sim +++ b/tests/script/unique/arbitrator/sync_replica2_alterTable_add.sim @@ -143,8 +143,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data6_2 -$dnode3Vtatus = $data9_2 +$dnode4Vtatus = $data5_2 +$dnode3Vtatus = $data7_2 if $dnode4Vtatus != offline then sleep 2000 diff --git a/tests/script/unique/arbitrator/sync_replica2_alterTable_drop.sim b/tests/script/unique/arbitrator/sync_replica2_alterTable_drop.sim index a26c1b04ed..d2c82f98ff 100644 --- a/tests/script/unique/arbitrator/sync_replica2_alterTable_drop.sim +++ b/tests/script/unique/arbitrator/sync_replica2_alterTable_drop.sim @@ -143,8 +143,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data6_2 -$dnode3Vtatus = $data9_2 +$dnode4Vtatus = $data5_2 +$dnode3Vtatus = $data7_2 if $dnode4Vtatus != offline then sleep 2000 diff --git a/tests/script/unique/arbitrator/sync_replica2_dropDb.sim b/tests/script/unique/arbitrator/sync_replica2_dropDb.sim index 1556a03b67..96aec2dcf8 100644 --- a/tests/script/unique/arbitrator/sync_replica2_dropDb.sim +++ b/tests/script/unique/arbitrator/sync_replica2_dropDb.sim @@ -143,8 +143,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data6_2 -$dnode3Vtatus = $data9_2 +$dnode4Vtatus = $data5_2 +$dnode3Vtatus = $data7_2 if $dnode4Vtatus != offline then sleep 2000 diff --git a/tests/script/unique/arbitrator/sync_replica2_dropTable.sim b/tests/script/unique/arbitrator/sync_replica2_dropTable.sim index 669b41becb..a1d7d18c94 100644 --- a/tests/script/unique/arbitrator/sync_replica2_dropTable.sim +++ b/tests/script/unique/arbitrator/sync_replica2_dropTable.sim @@ -143,8 +143,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data6_2 -$dnode3Vtatus = $data9_2 +$dnode4Vtatus = $data5_2 +$dnode3Vtatus = $data7_2 if $dnode4Vtatus != offline then sleep 2000 diff --git a/tests/script/unique/arbitrator/sync_replica3_alterTable_add.sim b/tests/script/unique/arbitrator/sync_replica3_alterTable_add.sim index 00c42cb7e3..ba973cbe06 100644 --- a/tests/script/unique/arbitrator/sync_replica3_alterTable_add.sim +++ b/tests/script/unique/arbitrator/sync_replica3_alterTable_add.sim @@ -143,8 +143,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data6_2 -$dnode3Vtatus = $data9_2 +$dnode4Vtatus = $data5_2 +$dnode3Vtatus = $data7_2 if $dnode4Vtatus != offline then sleep 2000 diff --git a/tests/script/unique/arbitrator/sync_replica3_alterTable_drop.sim b/tests/script/unique/arbitrator/sync_replica3_alterTable_drop.sim index 1d774e195e..9b918acc8e 100644 --- a/tests/script/unique/arbitrator/sync_replica3_alterTable_drop.sim +++ b/tests/script/unique/arbitrator/sync_replica3_alterTable_drop.sim @@ -143,8 +143,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data6_2 -$dnode3Vtatus = $data9_2 +$dnode4Vtatus = $data5_2 +$dnode3Vtatus = $data7_2 if $dnode4Vtatus != offline then sleep 2000 diff --git a/tests/script/unique/arbitrator/sync_replica3_createTable.sim b/tests/script/unique/arbitrator/sync_replica3_createTable.sim index db29eaf838..3bf274a3ea 100644 --- a/tests/script/unique/arbitrator/sync_replica3_createTable.sim +++ b/tests/script/unique/arbitrator/sync_replica3_createTable.sim @@ -143,8 +143,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data6_2 -$dnode3Vtatus = $data9_2 +$dnode4Vtatus = $data5_2 +$dnode3Vtatus = $data7_2 if $dnode4Vtatus != offline then sleep 2000 diff --git a/tests/script/unique/arbitrator/sync_replica3_dnodeChang_DropAddAlterTableDropDb.sim b/tests/script/unique/arbitrator/sync_replica3_dnodeChang_DropAddAlterTableDropDb.sim index 5b751e96f8..c8fe96008b 100644 --- a/tests/script/unique/arbitrator/sync_replica3_dnodeChang_DropAddAlterTableDropDb.sim +++ b/tests/script/unique/arbitrator/sync_replica3_dnodeChang_DropAddAlterTableDropDb.sim @@ -139,8 +139,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data6_2 -$dnode3Vtatus = $data9_2 +$dnode4Vtatus = $data5_2 +$dnode3Vtatus = $data7_2 if $dnode4Vtatus != offline then sleep 2000 @@ -207,8 +207,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data6_2 -$dnode3Vtatus = $data9_2 +$dnode4Vtatus = $data5_2 +$dnode3Vtatus = $data7_2 if $dnode4Vtatus != slave then sleep 2000 @@ -240,8 +240,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data6_2 -$dnode3Vtatus = $data9_2 +$dnode4Vtatus = $data5_2 +$dnode3Vtatus = $data7_2 if $dnode4Vtatus != master then sleep 2000 @@ -314,8 +314,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data6_2 -$dnode3Vtatus = $data9_2 +$dnode4Vtatus = $data5_2 +$dnode3Vtatus = $data7_2 if $dnode4Vtatus != slave then sleep 2000 @@ -347,8 +347,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data6_2 -$dnode3Vtatus = $data9_2 +$dnode4Vtatus = $data5_2 +$dnode3Vtatus = $data7_2 if $dnode4Vtatus != master then sleep 2000 @@ -437,8 +437,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data6_2 -$dnode3Vtatus = $data9_2 +$dnode4Vtatus = $data5_2 +$dnode3Vtatus = $data7_2 if $dnode4Vtatus != slave then sleep 2000 @@ -470,8 +470,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data6_2 -$dnode3Vtatus = $data9_2 +$dnode4Vtatus = $data5_2 +$dnode3Vtatus = $data7_2 if $dnode4Vtatus != master then sleep 2000 diff --git a/tests/script/unique/arbitrator/sync_replica3_dropDb.sim b/tests/script/unique/arbitrator/sync_replica3_dropDb.sim index 83f1d16856..4c7bb3d26c 100644 --- a/tests/script/unique/arbitrator/sync_replica3_dropDb.sim +++ b/tests/script/unique/arbitrator/sync_replica3_dropDb.sim @@ -143,8 +143,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data6_2 -$dnode3Vtatus = $data9_2 +$dnode4Vtatus = $data5_2 +$dnode3Vtatus = $data7_2 if $dnode4Vtatus != offline then sleep 2000 diff --git a/tests/script/unique/arbitrator/sync_replica3_dropTable.sim b/tests/script/unique/arbitrator/sync_replica3_dropTable.sim index e874afeaed..7ac424666a 100644 --- a/tests/script/unique/arbitrator/sync_replica3_dropTable.sim +++ b/tests/script/unique/arbitrator/sync_replica3_dropTable.sim @@ -143,8 +143,8 @@ print show vgroups: print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 -$dnode4Vtatus = $data6_2 -$dnode3Vtatus = $data9_2 +$dnode4Vtatus = $data5_2 +$dnode3Vtatus = $data7_2 if $dnode4Vtatus != offline then sleep 2000 diff --git a/tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeDir.sim b/tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeDir.sim index 7dd911a63e..12f0013191 100644 --- a/tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeDir.sim +++ b/tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeDir.sim @@ -194,13 +194,13 @@ print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $dat print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 $data5_4 $data6_4 $data7_4 $data8_4 $data9_4 -$d2v2status = $data6_4 -$d2v3status = $data6_2 -$d2v4status = $data6_3 +$d2v2status = $data5_4 +$d2v3status = $data5_2 +$d2v4status = $data5_3 -$d1v2status = $data9_4 -$d1v3status = $data9_2 -$d1v4status = $data9_3 +$d1v2status = $data7_4 +$d1v3status = $data7_2 +$d1v4status = $data7_3 if $d2v2status != master then sleep 2000 diff --git a/tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir.sim b/tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir.sim index bcbf388946..29da1fd773 100644 --- a/tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir.sim +++ b/tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir.sim @@ -196,13 +196,13 @@ print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $dat print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 $data5_4 $data6_4 $data7_4 $data8_4 $data9_4 -$d2v2status = $data6_4 -$d2v3status = $data6_2 -$d2v4status = $data6_3 +$d2v2status = $data5_4 +$d2v3status = $data5_2 +$d2v4status = $data5_3 -$d1v2status = $data9_4 -$d1v3status = $data9_2 -$d1v4status = $data9_3 +$d1v2status = $data7_4 +$d1v3status = $data7_2 +$d1v4status = $data7_3 if $d2v2status != master then sleep 2000 diff --git a/tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir_stopAll_starAll.sim b/tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir_stopAll_starAll.sim index ab76affafc..691b2cb568 100644 --- a/tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir_stopAll_starAll.sim +++ b/tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir_stopAll_starAll.sim @@ -161,13 +161,13 @@ print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $dat print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 $data5_4 $data6_4 $data7_4 $data8_4 $data9_4 -$d2v2status = $data6_4 -$d2v3status = $data6_2 -$d2v4status = $data6_3 +$d2v2status = $data5_4 +$d2v3status = $data5_2 +$d2v4status = $data5_3 -$d1v2status = $data9_4 -$d1v3status = $data9_2 -$d1v4status = $data9_3 +$d1v2status = $data7_4 +$d1v3status = $data7_2 +$d1v4status = $data7_3 if $d2v2status != master then sleep 2000 diff --git a/tests/script/unique/migrate/mn2_vn2_repl2_rmVnodeDir.sim b/tests/script/unique/migrate/mn2_vn2_repl2_rmVnodeDir.sim index f6a60d6b08..8a1132de2e 100644 --- a/tests/script/unique/migrate/mn2_vn2_repl2_rmVnodeDir.sim +++ b/tests/script/unique/migrate/mn2_vn2_repl2_rmVnodeDir.sim @@ -194,13 +194,13 @@ print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $dat print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 print $data0_4 $data1_4 $data2_4 $data3_4 $data4_4 $data5_4 $data6_4 $data7_4 $data8_4 $data9_4 -$d2v2status = $data6_4 -$d2v3status = $data6_2 -$d2v4status = $data6_3 +$d2v2status = $data5_4 +$d2v3status = $data5_2 +$d2v4status = $data5_3 -$d1v2status = $data9_4 -$d1v3status = $data9_2 -$d1v4status = $data9_3 +$d1v2status = $data7_4 +$d1v3status = $data7_2 +$d1v4status = $data7_3 if $d2v2status != master then sleep 2000 From 8090b713f71c113410760a3f9c543ee6b8030690 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 23 Jul 2020 06:05:19 +0000 Subject: [PATCH 071/100] scripts --- tests/script/general/parser/stream_on_sys.sim | 2 +- tests/script/jenkins/basic.txt | 11 ++++------- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/tests/script/general/parser/stream_on_sys.sim b/tests/script/general/parser/stream_on_sys.sim index 5405e09256..5507b4db48 100644 --- a/tests/script/general/parser/stream_on_sys.sim +++ b/tests/script/general/parser/stream_on_sys.sim @@ -28,7 +28,7 @@ sql create table diskstrm as select count(*), avg(disk_used), last(disk_used), a sql create table bandstrm as select count(*), avg(band_speed), last(band_speed) from log.dn1 interval(4s) sliding(2s) sql create table reqstrm as select count(*), avg(req_http), last(req_http), avg(req_select), last(req_select), avg(req_insert), last(req_insert) from log.dn1 interval(4s) sliding(2s) sql create table iostrm as select count(*), avg(io_read), last(io_read), avg(io_write), last(io_write) from log.dn1 interval(4s) sliding(2s) -sleep 20000 +sleep 120000 sql select * from cpustrm if $rows <= 0 then return -1 diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 828b907b9c..e953d337e4 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -309,13 +309,10 @@ cd ../../../debug; make ./test.sh -f unique/vnode/replica3_repeat.sim ./test.sh -f unique/vnode/replica3_vgroup.sim -# stream still has bugs -#./test.sh -f general/parser/stream_on_sys.sim -#./test.sh -f general/parser/repeatStream.sim - -#./test.sh -f general/stream/metrics_del.sim +./test.sh -f general/parser/stream_on_sys.sim +./test.sh -f general/stream/metrics_del.sim ./test.sh -f general/stream/metrics_n.sim -#./test.sh -f general/stream/metrics_replica1_vnoden.sim +./test.sh -f general/stream/metrics_replica1_vnoden.sim ./test.sh -f general/stream/restart_stream.sim ./test.sh -f general/stream/stream_3.sim ./test.sh -f general/stream/stream_restart.sim @@ -332,7 +329,7 @@ cd ../../../debug; make #./test.sh -f unique/arbitrator/dn3_mn1_replica2_wal1_AddDelDnode.sim ./test.sh -f unique/arbitrator/dn3_mn1_replica_change_dropDnod.sim ./test.sh -f unique/arbitrator/dn3_mn1_replica_change.sim -./test.sh -f unique/arbitrator/dn3_mn1_stopDnode_timeout.sim +#./test.sh -f unique/arbitrator/dn3_mn1_stopDnode_timeout.sim # lower the priority while file corruption #./test.sh -f unique/arbitrator/dn3_mn1_vnode_change.sim #./test.sh -f unique/arbitrator/dn3_mn1_vnode_corruptFile_offline.sim From a9325d32d2b6656e6be8aa615ee5f1efd4c8eacd Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 23 Jul 2020 14:18:17 +0800 Subject: [PATCH 072/100] cluster test clients --- tests/test/c/CMakeLists.txt | 4 ++-- tests/test/c/insertPerRow.c | 36 +++++++++++++++++++++++------------- 2 files changed, 25 insertions(+), 15 deletions(-) diff --git a/tests/test/c/CMakeLists.txt b/tests/test/c/CMakeLists.txt index 4717a4f769..8686a23f65 100644 --- a/tests/test/c/CMakeLists.txt +++ b/tests/test/c/CMakeLists.txt @@ -10,8 +10,8 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM)) #add_executable(insertPerTable insertPerTable.c) #target_link_libraries(insertPerTable taos_static pthread) - #add_executable(insertPerRow insertPerRow.c) - #target_link_libraries(insertPerRow taos_static pthread) + add_executable(insertPerRow insertPerRow.c) + target_link_libraries(insertPerRow taos_static pthread) #add_executable(importOneRow importOneRow.c) #target_link_libraries(importOneRow taos_static pthread) diff --git a/tests/test/c/insertPerRow.c b/tests/test/c/insertPerRow.c index 9b71ce50dd..ba6fb5f47b 100644 --- a/tests/test/c/insertPerRow.c +++ b/tests/test/c/insertPerRow.c @@ -44,14 +44,16 @@ void createDbAndTable(); void insertData(); int32_t randomData[MAX_RANDOM_POINTS]; -int64_t rowsPerTable = 10000; +int64_t rowsPerTable = 1000000000; int64_t pointsPerTable = 1; -int64_t numOfThreads = 1; -int64_t numOfTablesPerThread = 200; +int64_t numOfThreads = 10; +int64_t numOfTablesPerThread = 100; char dbName[32] = "db"; char stableName[64] = "st"; -int32_t cache = 16; -int32_t tables = 5000; +int32_t cache = 4; +int32_t replica = 5000; +int32_t days = 10; +int32_t interval = 1000; int main(int argc, char *argv[]) { shellParseArgument(argc, argv); @@ -77,7 +79,7 @@ void createDbAndTable() { exit(1); } - sprintf(qstr, "create database if not exists %s cache %d maxtables %d", dbName, cache, tables); + sprintf(qstr, "create database if not exists %s cache %d replica %d days %d", dbName, cache, replica, days); pSql = taos_query(con, qstr); int32_t code = taos_errno(pSql); if (code != 0) { @@ -239,7 +241,7 @@ void *syncTest(void *param) { st = systemTime.tv_sec * 1000000 + systemTime.tv_usec; int64_t start = 1430000000000; - int64_t interval = 1000; // 1000 ms + interval = 1000; // 1000 ms char *sql = qstr; char inserStr[] = "insert into"; @@ -309,10 +311,14 @@ void printHelp() { printf("%s%s%s%" PRId64 "\n", indent, indent, "Number of threads to be used, default is ", numOfThreads); printf("%s%s\n", indent, "-n"); printf("%s%s%s%" PRId64 "\n", indent, indent, "Number of tables per thread, default is ", numOfTablesPerThread); - printf("%s%s\n", indent, "-tables"); - printf("%s%s%s%d\n", indent, indent, "Database parameters tables, default is ", tables); + printf("%s%s\n", indent, "-replica"); + printf("%s%s%s%d\n", indent, indent, "Database parameters replica, default is ", replica); printf("%s%s\n", indent, "-cache"); - printf("%s%s%s%d\n", indent, indent, "Database parameters cache, default is ", cache); + printf("%s%s%s%d\n", indent, indent, "Database parameters replica, default is ", replica); + printf("%s%s\n", indent, "-days"); + printf("%s%s%s%d\n", indent, indent, "Database parameters days, default is ", days); + printf("%s%s\n", indent, "-interval"); + printf("%s%s%s%d\n", indent, indent, "Interval of each rows in ms, default is ", interval); exit(EXIT_SUCCESS); } @@ -336,10 +342,14 @@ void shellParseArgument(int argc, char *argv[]) { numOfThreads = atoi(argv[++i]); } else if (strcmp(argv[i], "-n") == 0) { numOfTablesPerThread = atoi(argv[++i]); - } else if (strcmp(argv[i], "-tables") == 0) { - tables = atoi(argv[++i]); + } else if (strcmp(argv[i], "-replica") == 0) { + replica = atoi(argv[++i]); } else if (strcmp(argv[i], "-cache") == 0) { cache = atoi(argv[++i]); + } else if (strcmp(argv[i], "-days") == 0) { + days = atoi(argv[++i]); + } else if (strcmp(argv[i], "-interval") == 0) { + interval = atoi(argv[++i]); } else { } } @@ -349,7 +359,7 @@ void shellParseArgument(int argc, char *argv[]) { pPrint("%snumOfThreads:%" PRId64 "%s", GREEN, numOfThreads, NC); pPrint("%snumOfTablesPerThread:%" PRId64 "%s", GREEN, numOfTablesPerThread, NC); pPrint("%scache:%" PRId32 "%s", GREEN, cache, NC); - pPrint("%stables:%" PRId32 "%s", GREEN, tables, NC); + pPrint("%stables:%" PRId32 "%s", GREEN, replica, NC); pPrint("%sdbName:%s%s", GREEN, dbName, NC); pPrint("%stableName:%s%s", GREEN, stableName, NC); pPrint("%sstart to run%s", GREEN, NC); From c02b53fcb44f827a0f47cb96cfa64da1bfdcd4fe Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 23 Jul 2020 15:51:32 +0800 Subject: [PATCH 073/100] scripts --- tests/script/tmp/mnodes.sim | 70 ++++++++----------------------------- tests/test/c/insertPerRow.c | 4 +-- 2 files changed, 17 insertions(+), 57 deletions(-) diff --git a/tests/script/tmp/mnodes.sim b/tests/script/tmp/mnodes.sim index 5cb0d72920..48dbc19cb2 100644 --- a/tests/script/tmp/mnodes.sim +++ b/tests/script/tmp/mnodes.sim @@ -4,10 +4,6 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/deploy.sh -n dnode2 -i 2 system sh/deploy.sh -n dnode3 -i 3 -system sh/cfg.sh -n dnode1 -c walLevel -v 2 -system sh/cfg.sh -n dnode2 -c walLevel -v 2 -system sh/cfg.sh -n dnode3 -c walLevel -v 2 - system sh/cfg.sh -n dnode1 -c numOfMnodes -v 3 system sh/cfg.sh -n dnode2 -c numOfMnodes -v 3 system sh/cfg.sh -n dnode3 -c numOfMnodes -v 3 @@ -16,64 +12,28 @@ system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 4 system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4 system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4 -system sh/cfg.sh -n dnode1 -c http -v 1 -system sh/cfg.sh -n dnode2 -c http -v 1 -system sh/cfg.sh -n dnode3 -c http -v 1 +system sh/cfg.sh -n dnode1 -c http -v 0 +system sh/cfg.sh -n dnode2 -c http -v 0 +system sh/cfg.sh -n dnode3 -c http -v 0 -system sh/cfg.sh -n dnode1 -c mDebugFlag -v 143 -system sh/cfg.sh -n dnode2 -c mDebugFlag -v 143 -system sh/cfg.sh -n dnode3 -c mDebugFlag -v 143 +system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 20000 +system sh/cfg.sh -n dnode2 -c maxTablesPerVnode -v 20000 +system sh/cfg.sh -n dnode3 -c maxTablesPerVnode -v 20000 -system sh/cfg.sh -n dnode1 -c sdbDebugFlag -v 143 -system sh/cfg.sh -n dnode2 -c sdbDebugFlag -v 143 -system sh/cfg.sh -n dnode3 -c sdbDebugFlag -v 143 - -system sh/cfg.sh -n dnode1 -c sdebugFlag -v 143 -system sh/cfg.sh -n dnode2 -c sdebugFlag -v 143 -system sh/cfg.sh -n dnode3 -c sdebugFlag -v 143 - -system sh/cfg.sh -n dnode1 -c rpcDebugFlag -v 135 -system sh/cfg.sh -n dnode2 -c rpcDebugFlag -v 135 -system sh/cfg.sh -n dnode3 -c rpcDebugFlag -v 135 - -system sh/cfg.sh -n dnode1 -c tsdbDebugFlag -v 131 -system sh/cfg.sh -n dnode2 -c tsdbDebugFlag -v 131 -system sh/cfg.sh -n dnode3 -c tsdbDebugFlag -v 131 - -system sh/cfg.sh -n dnode1 -c mqttDebugFlag -v 131 -system sh/cfg.sh -n dnode2 -c mqttDebugFlag -v 131 -system sh/cfg.sh -n dnode3 -c mqttDebugFlag -v 131 - -system sh/cfg.sh -n dnode1 -c qdebugFlag -v 131 -system sh/cfg.sh -n dnode2 -c qdebugFlag -v 131 -system sh/cfg.sh -n dnode3 -c qdebugFlag -v 131 - -system sh/cfg.sh -n dnode1 -c cDebugFlag -v 135 -system sh/cfg.sh -n dnode2 -c cDebugFlag -v 135 -system sh/cfg.sh -n dnode3 -c cDebugFlag -v 135 - -system sh/cfg.sh -n dnode1 -c udebugFlag -v 131 -system sh/cfg.sh -n dnode2 -c udebugFlag -v 131 -system sh/cfg.sh -n dnode3 -c udebugFlag -v 131 - -system sh/cfg.sh -n dnode1 -c wdebugFlag -v 131 -system sh/cfg.sh -n dnode2 -c wdebugFlag -v 131 -system sh/cfg.sh -n dnode3 -c wdebugFlag -v 131 - -system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 1000000 -system sh/cfg.sh -n dnode2 -c maxTablesPerVnode -v 1000000 -system sh/cfg.sh -n dnode3 -c maxTablesPerVnode -v 1000000 +system sh/cfg.sh -n dnode1 -c replica -v 3 +system sh/cfg.sh -n dnode2 -c replica -v 3 +system sh/cfg.sh -n dnode3 -c replica -v 3 print ============== deploy -system sh/exec.sh -n dnode1 -s start -sleep 2001 +system sh/exec.sh -n dnode1 -s start +sleep 5001 sql connect sql create dnode $hostname2 sql create dnode $hostname3 -system sh/exec.sh -n dnode2 -s start -system sh/exec.sh -n dnode3 -s start +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s start print =============== step1 $x = 0 @@ -112,8 +72,8 @@ print $data0_3 $data2_3 $x = $x + 1 -sleep 2000 -if $x == 1000 then +sleep 5000 +if $x == 100000 then return -1 endi diff --git a/tests/test/c/insertPerRow.c b/tests/test/c/insertPerRow.c index ba6fb5f47b..9cb718e486 100644 --- a/tests/test/c/insertPerRow.c +++ b/tests/test/c/insertPerRow.c @@ -51,7 +51,7 @@ int64_t numOfTablesPerThread = 100; char dbName[32] = "db"; char stableName[64] = "st"; int32_t cache = 4; -int32_t replica = 5000; +int32_t replica = 3; int32_t days = 10; int32_t interval = 1000; @@ -314,7 +314,7 @@ void printHelp() { printf("%s%s\n", indent, "-replica"); printf("%s%s%s%d\n", indent, indent, "Database parameters replica, default is ", replica); printf("%s%s\n", indent, "-cache"); - printf("%s%s%s%d\n", indent, indent, "Database parameters replica, default is ", replica); + printf("%s%s%s%d\n", indent, indent, "Database parameters replica, default is ", cache); printf("%s%s\n", indent, "-days"); printf("%s%s%s%d\n", indent, indent, "Database parameters days, default is ", days); printf("%s%s\n", indent, "-interval"); From f606318248f50de0270fbcff0f13245fa4457731 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 23 Jul 2020 15:57:06 +0800 Subject: [PATCH 074/100] TD-987 --- src/inc/tsdb.h | 2 - src/kit/taosmigrate/taosmigrateVnodeCfg.c | 26 ++++--- src/tsdb/inc/tsdbMain.h | 8 ++- src/tsdb/src/tsdbFile.c | 2 +- src/tsdb/src/tsdbMain.c | 79 ++++++--------------- src/tsdb/src/tsdbMemTable.c | 84 +++++++++++++++++------ src/tsdb/src/tsdbMeta.c | 43 +++++++++--- src/tsdb/src/tsdbRead.c | 21 +++--- src/vnode/src/vnodeMain.c | 30 ++++---- 9 files changed, 162 insertions(+), 133 deletions(-) diff --git a/src/inc/tsdb.h b/src/inc/tsdb.h index a2258dcfd8..bab17322f0 100644 --- a/src/inc/tsdb.h +++ b/src/inc/tsdb.h @@ -53,14 +53,12 @@ typedef struct { int32_t tsdbId; int32_t cacheBlockSize; int32_t totalBlocks; - int32_t maxTables; // maximum number of tables this repository can have int32_t daysPerFile; // day per file sharding policy int32_t keep; // day of data to keep int32_t keep1; int32_t keep2; int32_t minRowsPerFileBlock; // minimum rows per file block int32_t maxRowsPerFileBlock; // maximum rows per file block - int32_t commitTime; int8_t precision; int8_t compression; } STsdbCfg; diff --git a/src/kit/taosmigrate/taosmigrateVnodeCfg.c b/src/kit/taosmigrate/taosmigrateVnodeCfg.c index e830ef78f5..e80e687f02 100644 --- a/src/kit/taosmigrate/taosmigrateVnodeCfg.c +++ b/src/kit/taosmigrate/taosmigrateVnodeCfg.c @@ -37,14 +37,12 @@ static int32_t saveVnodeCfg(SVnodeObj *pVnode, char* cfgFile) len += snprintf(content + len, maxLen - len, " \"cfgVersion\": %d,\n", pVnode->cfgVersion); len += snprintf(content + len, maxLen - len, " \"cacheBlockSize\": %d,\n", pVnode->tsdbCfg.cacheBlockSize); len += snprintf(content + len, maxLen - len, " \"totalBlocks\": %d,\n", pVnode->tsdbCfg.totalBlocks); - len += snprintf(content + len, maxLen - len, " \"maxTables\": %d,\n", pVnode->tsdbCfg.maxTables); len += snprintf(content + len, maxLen - len, " \"daysPerFile\": %d,\n", pVnode->tsdbCfg.daysPerFile); len += snprintf(content + len, maxLen - len, " \"daysToKeep\": %d,\n", pVnode->tsdbCfg.keep); len += snprintf(content + len, maxLen - len, " \"daysToKeep1\": %d,\n", pVnode->tsdbCfg.keep1); len += snprintf(content + len, maxLen - len, " \"daysToKeep2\": %d,\n", pVnode->tsdbCfg.keep2); len += snprintf(content + len, maxLen - len, " \"minRowsPerFileBlock\": %d,\n", pVnode->tsdbCfg.minRowsPerFileBlock); len += snprintf(content + len, maxLen - len, " \"maxRowsPerFileBlock\": %d,\n", pVnode->tsdbCfg.maxRowsPerFileBlock); - len += snprintf(content + len, maxLen - len, " \"commitTime\": %d,\n", pVnode->tsdbCfg.commitTime); len += snprintf(content + len, maxLen - len, " \"precision\": %d,\n", pVnode->tsdbCfg.precision); len += snprintf(content + len, maxLen - len, " \"compression\": %d,\n", pVnode->tsdbCfg.compression); len += snprintf(content + len, maxLen - len, " \"walLevel\": %d,\n", pVnode->walCfg.walLevel); @@ -136,12 +134,12 @@ static int32_t readVnodeCfg(SVnodeObj *pVnode, char* cfgFile) } pVnode->tsdbCfg.totalBlocks = totalBlocks->valueint; - cJSON *maxTables = cJSON_GetObjectItem(root, "maxTables"); - if (!maxTables || maxTables->type != cJSON_Number) { - printf("vgId:%d, failed to read vnode cfg, maxTables not found\n", pVnode->vgId); - goto PARSE_OVER; - } - pVnode->tsdbCfg.maxTables = maxTables->valueint; + // cJSON *maxTables = cJSON_GetObjectItem(root, "maxTables"); + // if (!maxTables || maxTables->type != cJSON_Number) { + // printf("vgId:%d, failed to read vnode cfg, maxTables not found\n", pVnode->vgId); + // goto PARSE_OVER; + // } + // pVnode->tsdbCfg.maxTables = maxTables->valueint; cJSON *daysPerFile = cJSON_GetObjectItem(root, "daysPerFile"); if (!daysPerFile || daysPerFile->type != cJSON_Number) { @@ -185,12 +183,12 @@ static int32_t readVnodeCfg(SVnodeObj *pVnode, char* cfgFile) } pVnode->tsdbCfg.maxRowsPerFileBlock = maxRowsPerFileBlock->valueint; - cJSON *commitTime = cJSON_GetObjectItem(root, "commitTime"); - if (!commitTime || commitTime->type != cJSON_Number) { - printf("vgId:%d, failed to read vnode cfg, commitTime not found\n", pVnode->vgId); - goto PARSE_OVER; - } - pVnode->tsdbCfg.commitTime = (int8_t)commitTime->valueint; + // cJSON *commitTime = cJSON_GetObjectItem(root, "commitTime"); + // if (!commitTime || commitTime->type != cJSON_Number) { + // printf("vgId:%d, failed to read vnode cfg, commitTime not found\n", pVnode->vgId); + // goto PARSE_OVER; + // } + // pVnode->tsdbCfg.commitTime = (int8_t)commitTime->valueint; cJSON *precision = cJSON_GetObjectItem(root, "precision"); if (!precision || precision->type != cJSON_Number) { diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h index 762d2253e2..3e92c01765 100644 --- a/src/tsdb/inc/tsdbMain.h +++ b/src/tsdb/inc/tsdbMain.h @@ -70,6 +70,7 @@ typedef struct { pthread_rwlock_t rwLock; int32_t nTables; + int32_t maxTables; STable** tables; SList* superList; SHashObj* uidMap; @@ -111,9 +112,11 @@ typedef struct { typedef struct { T_REF_DECLARE(); + SRWLatch latch; TSKEY keyFirst; TSKEY keyLast; int64_t numOfRows; + int32_t maxTables; STableData** tData; SList* actList; SList* bufBlockList; @@ -304,6 +307,7 @@ typedef struct { // Operations // ------------------ tsdbMeta.c +#define TSDB_INIT_NTABLES 1024 #define TABLE_TYPE(t) (t)->type #define TABLE_NAME(t) (t)->name #define TABLE_CHAR_NAME(t) TABLE_NAME(t)->data @@ -395,6 +399,7 @@ int tsdbInsertRowToMem(STsdbRepo* pRepo, SDataRow row, STable* pTable); int tsdbRefMemTable(STsdbRepo* pRepo, SMemTable* pMemTable); int tsdbUnRefMemTable(STsdbRepo* pRepo, SMemTable* pMemTable); int tsdbTakeMemSnapshot(STsdbRepo* pRepo, SMemTable** pMem, SMemTable** pIMem); +void tsdbUnTakeMemSnapShot(STsdbRepo* pRepo, SMemTable* pMem, SMemTable* pIMem); void* tsdbAllocBytes(STsdbRepo* pRepo, int bytes); int tsdbAsyncCommit(STsdbRepo* pRepo); int tsdbLoadDataFromCache(STable* pTable, SSkipListIterator* pIter, TSKEY maxKey, int maxRowsToRead, SDataCols* pCols, @@ -429,7 +434,7 @@ STsdbFileH* tsdbNewFileH(STsdbCfg* pCfg); void tsdbFreeFileH(STsdbFileH* pFileH); int tsdbOpenFileH(STsdbRepo* pRepo); void tsdbCloseFileH(STsdbRepo* pRepo); -SFileGroup* tsdbCreateFGroupIfNeed(STsdbRepo* pRepo, char* dataDir, int fid, int maxTables); +SFileGroup* tsdbCreateFGroupIfNeed(STsdbRepo* pRepo, char* dataDir, int fid); void tsdbInitFileGroupIter(STsdbFileH* pFileH, SFileGroupIter* pIter, int direction); void tsdbSeekFileGroupIter(SFileGroupIter* pIter, int fid); SFileGroup* tsdbGetFileGroupNext(SFileGroupIter* pIter); @@ -511,6 +516,7 @@ void tsdbGetDataFileName(STsdbRepo* pRepo, int fid, int type, char* fname int tsdbLockRepo(STsdbRepo* pRepo); int tsdbUnlockRepo(STsdbRepo* pRepo); char* tsdbGetDataDirName(char* rootDir); +int tsdbGetNextMaxTables(int tid); STsdbMeta* tsdbGetMeta(TSDB_REPO_T* pRepo); STsdbFileH* tsdbGetFile(TSDB_REPO_T* pRepo); diff --git a/src/tsdb/src/tsdbFile.c b/src/tsdb/src/tsdbFile.c index 767fbc8252..f698878698 100644 --- a/src/tsdb/src/tsdbFile.c +++ b/src/tsdb/src/tsdbFile.c @@ -149,7 +149,7 @@ void tsdbCloseFileH(STsdbRepo *pRepo) { } } -SFileGroup *tsdbCreateFGroupIfNeed(STsdbRepo *pRepo, char *dataDir, int fid, int maxTables) { +SFileGroup *tsdbCreateFGroupIfNeed(STsdbRepo *pRepo, char *dataDir, int fid) { STsdbFileH *pFileH = pRepo->tsdbFileH; if (pFileH->nFGroups >= pFileH->maxFGroups) return NULL; diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index 772bcf48d6..11a84cd552 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -62,7 +62,6 @@ static int tsdbRestoreInfo(STsdbRepo *pRepo); static int tsdbInitSubmitBlkIter(SSubmitBlk *pBlock, SSubmitBlkIter *pIter); static void tsdbAlterCompression(STsdbRepo *pRepo, int8_t compression); static int tsdbAlterKeep(STsdbRepo *pRepo, int32_t keep); -static int tsdbAlterMaxTables(STsdbRepo *pRepo, int32_t maxTables); static int tsdbAlterCacheTotalBlocks(STsdbRepo *pRepo, int totalBlocks); static int keyFGroupCompFunc(const void *key, const void *fgroup); static int tsdbEncodeCfg(void **buf, STsdbCfg *pCfg); @@ -85,10 +84,10 @@ int32_t tsdbCreateRepo(char *rootDir, STsdbCfg *pCfg) { if (tsdbSetRepoEnv(rootDir, pCfg) < 0) return -1; tsdbDebug( - "vgId:%d tsdb env create succeed! cacheBlockSize %d totalBlocks %d maxTables %d daysPerFile %d keep " + "vgId:%d tsdb env create succeed! cacheBlockSize %d totalBlocks %d daysPerFile %d keep " "%d minRowsPerFileBlock %d maxRowsPerFileBlock %d precision %d compression %d", - pCfg->tsdbId, pCfg->cacheBlockSize, pCfg->totalBlocks, pCfg->maxTables, pCfg->daysPerFile, pCfg->keep, - pCfg->minRowsPerFileBlock, pCfg->maxRowsPerFileBlock, pCfg->precision, pCfg->compression); + pCfg->tsdbId, pCfg->cacheBlockSize, pCfg->totalBlocks, pCfg->daysPerFile, pCfg->keep, pCfg->minRowsPerFileBlock, + pCfg->maxRowsPerFileBlock, pCfg->precision, pCfg->compression); return 0; } @@ -307,13 +306,6 @@ int32_t tsdbConfigRepo(TSDB_REPO_T *repo, STsdbCfg *pCfg) { tsdbAlterCacheTotalBlocks(pRepo, pCfg->totalBlocks); configChanged = true; } - if (pRCfg->maxTables != pCfg->maxTables) { - if (tsdbAlterMaxTables(pRepo, pCfg->maxTables) < 0) { - tsdbError("vgId:%d failed to configure repo when alter maxTables since %s", REPO_ID(pRepo), tstrerror(terrno)); - return -1; - } - configChanged = true; - } if (configChanged) { if (tsdbSaveConfig(pRepo->rootDir, &pRepo->config) < 0) { @@ -385,6 +377,18 @@ char *tsdbGetDataDirName(char *rootDir) { return fname; } +int tsdbGetNextMaxTables(int tid) { + ASSERT(tid >= 1 && tid <= TSDB_MAX_TABLES); + int maxTables = TSDB_INIT_NTABLES; + while (true) { + maxTables = MIN(maxTables, TSDB_MAX_TABLES); + if (tid <= maxTables + 1) break; + maxTables *= 2; + } + + return maxTables + 1; +} + STsdbMeta * tsdbGetMeta(TSDB_REPO_T *pRepo) { return ((STsdbRepo *)pRepo)->tsdbMeta; } STsdbFileH * tsdbGetFile(TSDB_REPO_T *pRepo) { return ((STsdbRepo *)pRepo)->tsdbFileH; } STsdbRepoInfo *tsdbGetStatus(TSDB_REPO_T *pRepo) { return NULL; } @@ -417,17 +421,6 @@ static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) { goto _err; } - // Check maxTables - if (pCfg->maxTables == -1) { - pCfg->maxTables = TSDB_DEFAULT_TABLES+1; - } else { - if (pCfg->maxTables - 1 < TSDB_MIN_TABLES || pCfg->maxTables - 1 > TSDB_MAX_TABLES) { - tsdbError("vgId:%d invalid maxTables configuration! maxTables %d TSDB_MIN_TABLES %d TSDB_MAX_TABLES %d", - pCfg->tsdbId, pCfg->maxTables - 1, TSDB_MIN_TABLES, TSDB_MAX_TABLES); - goto _err; - } - } - // Check daysPerFile if (pCfg->daysPerFile == -1) { pCfg->daysPerFile = TSDB_DEFAULT_DAYS_PER_FILE; @@ -713,6 +706,7 @@ static int32_t tsdbInsertDataToTable(STsdbRepo *pRepo, SSubmitBlk *pBlock, TSKEY STsdbMeta *pMeta = pRepo->tsdbMeta; int64_t points = 0; + ASSERT(pBlock->tid < pMeta->maxTables); STable *pTable = pMeta->tables[pBlock->tid]; ASSERT(pTable != NULL && TABLE_UID(pTable) == pBlock->uid); @@ -779,7 +773,6 @@ static SDataRow tsdbGetSubmitBlkNext(SSubmitBlkIter *pIter) { } static int tsdbRestoreInfo(STsdbRepo *pRepo) { - // TODO STsdbMeta * pMeta = pRepo->tsdbMeta; STsdbFileH *pFileH = pRepo->tsdbFileH; SFileGroup *pFGroup = NULL; @@ -792,7 +785,7 @@ static int tsdbRestoreInfo(STsdbRepo *pRepo) { tsdbInitFileGroupIter(pFileH, &iter, TSDB_ORDER_DESC); while ((pFGroup = tsdbGetFileGroupNext(&iter)) != NULL) { if (tsdbSetAndOpenHelperFile(&rhelper, pFGroup) < 0) goto _err; - for (int i = 1; i < pRepo->config.maxTables; i++) { + for (int i = 1; i < pMeta->maxTables; i++) { STable *pTable = pMeta->tables[i]; if (pTable == NULL) continue; tsdbSetHelperTable(&rhelper, pTable, pRepo); @@ -868,36 +861,6 @@ static int tsdbAlterKeep(STsdbRepo *pRepo, int32_t keep) { return 0; } -static int tsdbAlterMaxTables(STsdbRepo *pRepo, int32_t maxTables) { - // TODO - int oldMaxTables = pRepo->config.maxTables; - if (oldMaxTables < pRepo->config.maxTables) { - terrno = TSDB_CODE_TDB_INVALID_ACTION; - return -1; - } - - STsdbMeta *pMeta = pRepo->tsdbMeta; - - pMeta->tables = realloc(pMeta->tables, maxTables * sizeof(STable *)); - memset(&pMeta->tables[oldMaxTables], 0, sizeof(STable *) * (maxTables - oldMaxTables)); - pRepo->config.maxTables = maxTables; - - if (pRepo->mem) { - pRepo->mem->tData = realloc(pRepo->mem->tData, maxTables * sizeof(STableData *)); - memset(POINTER_SHIFT(pRepo->mem->tData, sizeof(STableData *) * oldMaxTables), 0, - sizeof(STableData *) * (maxTables - oldMaxTables)); - } - - if (pRepo->imem) { - pRepo->imem->tData = realloc(pRepo->imem->tData, maxTables * sizeof(STableData *)); - memset(POINTER_SHIFT(pRepo->imem->tData, sizeof(STableData *) * oldMaxTables), 0, - sizeof(STableData *) * (maxTables - oldMaxTables)); - } - - tsdbDebug("vgId:%d, tsdb maxTables is changed from %d to %d!", pRepo->config.tsdbId, oldMaxTables, maxTables); - return 0; -} - static int keyFGroupCompFunc(const void *key, const void *fgroup) { int fid = *(int *)key; SFileGroup *pFGroup = (SFileGroup *)fgroup; @@ -914,7 +877,6 @@ static int tsdbEncodeCfg(void **buf, STsdbCfg *pCfg) { tlen += taosEncodeVariantI32(buf, pCfg->tsdbId); tlen += taosEncodeFixedI32(buf, pCfg->cacheBlockSize); tlen += taosEncodeVariantI32(buf, pCfg->totalBlocks); - tlen += taosEncodeVariantI32(buf, pCfg->maxTables); tlen += taosEncodeVariantI32(buf, pCfg->daysPerFile); tlen += taosEncodeVariantI32(buf, pCfg->keep); tlen += taosEncodeVariantI32(buf, pCfg->keep1); @@ -931,7 +893,6 @@ static void *tsdbDecodeCfg(void *buf, STsdbCfg *pCfg) { buf = taosDecodeVariantI32(buf, &(pCfg->tsdbId)); buf = taosDecodeFixedI32(buf, &(pCfg->cacheBlockSize)); buf = taosDecodeVariantI32(buf, &(pCfg->totalBlocks)); - buf = taosDecodeVariantI32(buf, &(pCfg->maxTables)); buf = taosDecodeVariantI32(buf, &(pCfg->daysPerFile)); buf = taosDecodeVariantI32(buf, &(pCfg->keep)); buf = taosDecodeVariantI32(buf, &(pCfg->keep1)); @@ -1037,7 +998,7 @@ static int tsdbScanAndConvertSubmitMsg(STsdbRepo *pRepo, SSubmitMsg *pMsg) { pBlock->schemaLen = htonl(pBlock->schemaLen); pBlock->numOfRows = htons(pBlock->numOfRows); - if (pBlock->tid <= 0 || pBlock->tid >= pRepo->config.maxTables) { + if (pBlock->tid <= 0 || pBlock->tid >= pMeta->maxTables) { tsdbError("vgId:%d failed to get table to insert data, uid %" PRIu64 " tid %d", REPO_ID(pRepo), pBlock->uid, pBlock->tid); terrno = TSDB_CODE_TDB_INVALID_TABLE_ID; @@ -1120,7 +1081,7 @@ TSKEY tsdbGetTableLastKey(TSDB_REPO_T *repo, uint64_t uid) { static void tsdbStartStream(STsdbRepo *pRepo) { STsdbMeta *pMeta = pRepo->tsdbMeta; - for (int i = 0; i < pRepo->config.maxTables; i++) { + for (int i = 0; i < pMeta->maxTables; i++) { STable *pTable = pMeta->tables[i]; if (pTable && pTable->type == TSDB_STREAM_TABLE) { pTable->cqhandle = (*pRepo->appH.cqCreateFunc)(pRepo->appH.cqH, TABLE_UID(pTable), TABLE_TID(pTable), pTable->sql, @@ -1133,7 +1094,7 @@ static void tsdbStartStream(STsdbRepo *pRepo) { static void tsdbStopStream(STsdbRepo *pRepo) { STsdbMeta *pMeta = pRepo->tsdbMeta; - for (int i = 0; i < pRepo->config.maxTables; i++) { + for (int i = 0; i < pMeta->maxTables; i++) { STable *pTable = pMeta->tables[i]; if (pTable && pTable->type == TSDB_STREAM_TABLE) { (*pRepo->appH.cqDropFunc)(pTable->cqhandle); diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index 7708646a61..9ca28d06cb 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -21,7 +21,7 @@ static FORCE_INLINE STsdbBufBlock *tsdbGetCurrBufBlock(STsdbRepo *pRepo); static void tsdbFreeBytes(STsdbRepo *pRepo, void *ptr, int bytes); -static SMemTable * tsdbNewMemTable(STsdbCfg *pCfg); +static SMemTable * tsdbNewMemTable(STsdbRepo *pRepo); static void tsdbFreeMemTable(SMemTable *pMemTable); static STableData *tsdbNewTableData(STsdbCfg *pCfg, STable *pTable); static void tsdbFreeTableData(STableData *pTableData); @@ -30,13 +30,15 @@ static void * tsdbCommitData(void *arg); static int tsdbCommitMeta(STsdbRepo *pRepo); static void tsdbEndCommit(STsdbRepo *pRepo); static int tsdbHasDataToCommit(SCommitIter *iters, int nIters, TSKEY minKey, TSKEY maxKey); -static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHelper *pHelper, SDataCols *pDataCols); +static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHelper *pHelper, SDataCols *pDataCols); static SCommitIter *tsdbCreateCommitIters(STsdbRepo *pRepo); static void tsdbDestroyCommitIters(SCommitIter *iters, int maxTables); +static int tsdbAdjustMemMaxTables(SMemTable *pMemTable, int maxTables); // ---------------- INTERNAL FUNCTIONS ---------------- int tsdbInsertRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable) { STsdbCfg * pCfg = &pRepo->config; + STsdbMeta * pMeta = pRepo->tsdbMeta; int32_t level = 0; int32_t headSize = 0; TSKEY key = dataRowKey(row); @@ -45,7 +47,7 @@ int tsdbInsertRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable) { SSkipList * pSList = NULL; int bytes = 0; - if (pMemTable != NULL && pMemTable->tData[TABLE_TID(pTable)] != NULL && + if (pMemTable != NULL && TABLE_TID(pTable) < pMemTable->maxTables && pMemTable->tData[TABLE_TID(pTable)] != NULL && pMemTable->tData[TABLE_TID(pTable)]->uid == TABLE_UID(pTable)) { pTableData = pMemTable->tData[TABLE_TID(pTable)]; pSList = pTableData->pData; @@ -66,13 +68,20 @@ int tsdbInsertRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable) { // Operations above may change pRepo->mem, retake those values ASSERT(pRepo->mem != NULL); pMemTable = pRepo->mem; + + if (TABLE_TID(pTable) >= pMemTable->maxTables) { + if (tsdbAdjustMemMaxTables(pMemTable, pMeta->maxTables) < 0) return -1;; + } pTableData = pMemTable->tData[TABLE_TID(pTable)]; if (pTableData == NULL || pTableData->uid != TABLE_UID(pTable)) { if (pTableData != NULL) { // destroy the table skiplist (may have race condition problem) + taosWLockLatch(&(pMemTable->latch)); pMemTable->tData[TABLE_TID(pTable)] = NULL; tsdbFreeTableData(pTableData); + taosWUnLockLatch(&(pMemTable->latch)); } + pTableData = tsdbNewTableData(pCfg, pTable); if (pTableData == NULL) { tsdbError("vgId:%d failed to insert row with key %" PRId64 @@ -122,7 +131,6 @@ int tsdbUnRefMemTable(STsdbRepo *pRepo, SMemTable *pMemTable) { int ref = T_REF_DEC(pMemTable); tsdbDebug("vgId:%d unref memtable %p ref %d", REPO_ID(pRepo), pMemTable, ref); if (ref == 0) { - STsdbCfg * pCfg = &pRepo->config; STsdbBufPool *pBufPool = pRepo->pPool; SListNode *pNode = NULL; @@ -139,7 +147,7 @@ int tsdbUnRefMemTable(STsdbRepo *pRepo, SMemTable *pMemTable) { } if (tsdbUnlockRepo(pRepo) < 0) return -1; - for (int i = 0; i < pCfg->maxTables; i++) { + for (int i = 0; i < pMemTable->maxTables; i++) { if (pMemTable->tData[i] != NULL) { tsdbFreeTableData(pMemTable->tData[i]); } @@ -161,11 +169,24 @@ int tsdbTakeMemSnapshot(STsdbRepo *pRepo, SMemTable **pMem, SMemTable **pIMem) { tsdbRefMemTable(pRepo, *pIMem); if (tsdbUnlockRepo(pRepo) < 0) return -1; - tsdbDebug("vgId:%d take memory snapshot, pMem %p pIMem %p", REPO_ID(pRepo), *pMem, *pIMem); + if (*pMem != NULL) taosRLockLatch(&((*pMem)->latch)); + + tsdbDebug("vgId:%d take memory snapshot, pMem %p pIMem %p", REPO_ID(pRepo), *pMem, *pIMem); return 0; } +void tsdbUnTakeMemSnapShot(STsdbRepo *pRepo, SMemTable *pMem, SMemTable *pIMem) { + if (pMem != NULL) { + taosRUnLockLatch(&(pMem->latch)); + tsdbUnRefMemTable(pRepo, pMem); + } + + if (pIMem != NULL) { + tsdbUnRefMemTable(pRepo, pIMem); + } +} + void *tsdbAllocBytes(STsdbRepo *pRepo, int bytes) { STsdbCfg * pCfg = &pRepo->config; STsdbBufBlock *pBufBlock = tsdbGetCurrBufBlock(pRepo); @@ -182,7 +203,7 @@ void *tsdbAllocBytes(STsdbRepo *pRepo, int bytes) { } if (pRepo->mem == NULL) { - SMemTable *pMemTable = tsdbNewMemTable(&pRepo->config); + SMemTable *pMemTable = tsdbNewMemTable(pRepo); if (pMemTable == NULL) return NULL; if (tsdbLockRepo(pRepo) < 0) { @@ -329,7 +350,9 @@ static void tsdbFreeBytes(STsdbRepo *pRepo, void *ptr, int bytes) { listNEles(pRepo->mem->bufBlockList), pBufBlock->offset, pBufBlock->remain); } -static SMemTable* tsdbNewMemTable(STsdbCfg* pCfg) { +static SMemTable* tsdbNewMemTable(STsdbRepo *pRepo) { + STsdbMeta *pMeta = pRepo->tsdbMeta; + SMemTable *pMemTable = (SMemTable *)calloc(1, sizeof(*pMemTable)); if (pMemTable == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; @@ -340,7 +363,8 @@ static SMemTable* tsdbNewMemTable(STsdbCfg* pCfg) { pMemTable->keyLast = 0; pMemTable->numOfRows = 0; - pMemTable->tData = (STableData**)calloc(pCfg->maxTables, sizeof(STableData*)); + pMemTable->maxTables = pMeta->maxTables; + pMemTable->tData = (STableData **)calloc(pMemTable->maxTables, sizeof(STableData *)); if (pMemTable->tData == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; goto _err; @@ -398,9 +422,6 @@ static STableData *tsdbNewTableData(STsdbCfg *pCfg, STable *pTable) { goto _err; } - // TODO: operation here should not be here, remove it - pTableData->pData->level = 1; - return pTableData; _err: @@ -473,7 +494,7 @@ static void *tsdbCommitData(void *arg) { _exit: tdFreeDataCols(pDataCols); - tsdbDestroyCommitIters(iters, pCfg->maxTables); + tsdbDestroyCommitIters(iters, pMem->maxTables); tsdbDestroyHelper(&whelper); tsdbEndCommit(pRepo); tsdbInfo("vgId:%d commit over", pRepo->config.tsdbId); @@ -552,12 +573,13 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHe STsdbCfg * pCfg = &pRepo->config; STsdbFileH *pFileH = pRepo->tsdbFileH; SFileGroup *pGroup = NULL; + SMemTable * pMem = pRepo->imem; TSKEY minKey = 0, maxKey = 0; tsdbGetFidKeyRange(pCfg->daysPerFile, pCfg->precision, fid, &minKey, &maxKey); // Check if there are data to commit to this file - int hasDataToCommit = tsdbHasDataToCommit(iters, pCfg->maxTables, minKey, maxKey); + int hasDataToCommit = tsdbHasDataToCommit(iters, pMem->maxTables, minKey, maxKey); if (!hasDataToCommit) { tsdbDebug("vgId:%d no data to commit to file %d", REPO_ID(pRepo), fid); return 0; @@ -570,7 +592,7 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHe return -1; } - if ((pGroup = tsdbCreateFGroupIfNeed(pRepo, dataDir, fid, pCfg->maxTables)) == NULL) { + if ((pGroup = tsdbCreateFGroupIfNeed(pRepo, dataDir, fid)) == NULL) { tsdbError("vgId:%d failed to create file group %d since %s", REPO_ID(pRepo), fid, tstrerror(terrno)); goto _err; } @@ -582,7 +604,7 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHe } // Loop to commit data in each table - for (int tid = 1; tid < pCfg->maxTables; tid++) { + for (int tid = 1; tid < pMem->maxTables; tid++) { SCommitIter *pIter = iters + tid; if (pIter->pTable == NULL) continue; @@ -643,11 +665,10 @@ _err: } static SCommitIter *tsdbCreateCommitIters(STsdbRepo *pRepo) { - STsdbCfg * pCfg = &(pRepo->config); SMemTable *pMem = pRepo->imem; STsdbMeta *pMeta = pRepo->tsdbMeta; - SCommitIter *iters = (SCommitIter *)calloc(pCfg->maxTables, sizeof(SCommitIter)); + SCommitIter *iters = (SCommitIter *)calloc(pMem->maxTables, sizeof(SCommitIter)); if (iters == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; return NULL; @@ -656,7 +677,7 @@ static SCommitIter *tsdbCreateCommitIters(STsdbRepo *pRepo) { if (tsdbRLockRepoMeta(pRepo) < 0) goto _err; // reference all tables - for (int i = 0; i < pCfg->maxTables; i++) { + for (int i = 0; i < pMem->maxTables; i++) { if (pMeta->tables[i] != NULL) { tsdbRefTable(pMeta->tables[i]); iters[i].pTable = pMeta->tables[i]; @@ -665,7 +686,7 @@ static SCommitIter *tsdbCreateCommitIters(STsdbRepo *pRepo) { if (tsdbUnlockRepoMeta(pRepo) < 0) goto _err; - for (int i = 0; i < pCfg->maxTables; i++) { + for (int i = 0; i < pMem->maxTables; i++) { if ((iters[i].pTable != NULL) && (pMem->tData[i] != NULL) && (TABLE_UID(iters[i].pTable) == pMem->tData[i]->uid)) { if ((iters[i].pIter = tSkipListCreateIter(pMem->tData[i]->pData)) == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; @@ -679,7 +700,7 @@ static SCommitIter *tsdbCreateCommitIters(STsdbRepo *pRepo) { return iters; _err: - tsdbDestroyCommitIters(iters, pCfg->maxTables); + tsdbDestroyCommitIters(iters, pMem->maxTables); return NULL; } @@ -694,4 +715,25 @@ static void tsdbDestroyCommitIters(SCommitIter *iters, int maxTables) { } free(iters); +} + +static int tsdbAdjustMemMaxTables(SMemTable *pMemTable, int maxTables) { + ASSERT(pMemTable->maxTables < maxTables); + + STableData **pTableData = (STableData **)calloc(maxTables, sizeof(STableData *)); + if (pTableData == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return -1; + } + + STableData **tData = pMemTable->tData; + + taosWLockLatch(&(pMemTable->latch)); + pMemTable->maxTables = maxTables; + pMemTable->tData = pTableData; + taosWUnLockLatch(&(pMemTable->latch)); + + tfree(tData); + + return 0; } \ No newline at end of file diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index b25e734694..8d237ab673 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -49,6 +49,7 @@ static int tsdbGetTableEncodeSize(int8_t act, STable *pTable); static void * tsdbInsertTableAct(STsdbRepo *pRepo, int8_t act, void *buf, STable *pTable); static int tsdbRemoveTableFromStore(STsdbRepo *pRepo, STable *pTable); static int tsdbRmTableFromMeta(STsdbRepo *pRepo, STable *pTable); +static int tsdbAdjustMetaTables(STsdbRepo *pRepo, int tid); // ------------------ OUTER FUNCTIONS ------------------ int tsdbCreateTable(TSDB_REPO_T *repo, STableCfg *pCfg) { @@ -60,13 +61,13 @@ int tsdbCreateTable(TSDB_REPO_T *repo, STableCfg *pCfg) { int tid = pCfg->tableId.tid; STable * pTable = NULL; - if (tid < 0 || tid >= pRepo->config.maxTables) { + if (tid < 1 || tid > TSDB_MAX_TABLES) { tsdbError("vgId:%d failed to create table since invalid tid %d", REPO_ID(pRepo), tid); terrno = TSDB_CODE_TDB_IVD_CREATE_TABLE_INFO; goto _err; } - if (pMeta->tables[tid] != NULL) { + if (tid < pMeta->maxTables && pMeta->tables[tid] != NULL) { if (TABLE_UID(pMeta->tables[tid]) == pCfg->tableId.uid) { tsdbError("vgId:%d table %s already exists, tid %d uid %" PRId64, REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable)); @@ -422,7 +423,8 @@ STsdbMeta *tsdbNewMeta(STsdbCfg *pCfg) { goto _err; } - pMeta->tables = (STable **)calloc(pCfg->maxTables, sizeof(STable *)); + pMeta->maxTables = TSDB_INIT_NTABLES + 1; + pMeta->tables = (STable **)calloc(pMeta->maxTables, sizeof(STable *)); if (pMeta->tables == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; goto _err; @@ -434,7 +436,7 @@ STsdbMeta *tsdbNewMeta(STsdbCfg *pCfg) { goto _err; } - pMeta->uidMap = taosHashInit(pCfg->maxTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false); + pMeta->uidMap = taosHashInit(TSDB_INIT_NTABLES * 1.1, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false); if (pMeta->uidMap == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; goto _err; @@ -484,14 +486,13 @@ _err: } int tsdbCloseMeta(STsdbRepo *pRepo) { - STsdbCfg * pCfg = &pRepo->config; STsdbMeta *pMeta = pRepo->tsdbMeta; SListNode *pNode = NULL; STable * pTable = NULL; if (pMeta == NULL) return 0; tdCloseKVStore(pMeta->pStore); - for (int i = 1; i < pCfg->maxTables; i++) { + for (int i = 1; i < pMeta->maxTables; i++) { tsdbFreeTable(pMeta->tables[i]); } @@ -624,9 +625,8 @@ static int tsdbRestoreTable(void *pHandle, void *cont, int contLen) { static void tsdbOrgMeta(void *pHandle) { STsdbRepo *pRepo = (STsdbRepo *)pHandle; STsdbMeta *pMeta = pRepo->tsdbMeta; - STsdbCfg * pCfg = &pRepo->config; - for (int i = 1; i < pCfg->maxTables; i++) { + for (int i = 1; i < pMeta->maxTables; i++) { STable *pTable = pMeta->tables[i]; if (pTable != NULL && pTable->type == TSDB_CHILD_TABLE) { tsdbAddTableIntoIndex(pMeta, pTable, true); @@ -781,6 +781,7 @@ static int tsdbAddTableToMeta(STsdbRepo *pRepo, STable *pTable, bool addIdx, boo goto _err; } } else { + if (tsdbAdjustMetaTables(pRepo, TABLE_TID(pTable)) < 0) goto _err; if (TABLE_TYPE(pTable) == TSDB_CHILD_TABLE && addIdx) { // add STABLE to the index if (tsdbAddTableIntoIndex(pMeta, pTable, true) < 0) { tsdbDebug("vgId:%d failed to add table %s to meta while add table to index since %s", REPO_ID(pRepo), @@ -827,7 +828,6 @@ static void tsdbRemoveTableFromMeta(STsdbRepo *pRepo, STable *pTable, bool rmFro SListIter lIter = {0}; SListNode *pNode = NULL; STable * tTable = NULL; - STsdbCfg * pCfg = &(pRepo->config); STSchema *pSchema = tsdbGetTableSchemaImpl(pTable, false, false, -1); int maxCols = schemaNCols(pSchema); @@ -860,7 +860,7 @@ static void tsdbRemoveTableFromMeta(STsdbRepo *pRepo, STable *pTable, bool rmFro if (maxCols == pMeta->maxCols || maxRowBytes == pMeta->maxRowBytes) { maxCols = 0; maxRowBytes = 0; - for (int i = 0; i < pCfg->maxTables; i++) { + for (int i = 0; i < pMeta->maxTables; i++) { STable *pTable = pMeta->tables[i]; if (pTable != NULL) { pSchema = tsdbGetTableSchemaImpl(pTable, false, false, -1); @@ -1266,5 +1266,28 @@ static int tsdbRmTableFromMeta(STsdbRepo *pRepo, STable *pTable) { tsdbRemoveTableFromMeta(pRepo, pTable, true, true); } + return 0; +} + +static int tsdbAdjustMetaTables(STsdbRepo *pRepo, int tid) { + STsdbMeta *pMeta = pRepo->tsdbMeta; + if (pMeta->maxTables >= tid) return 0; + + int maxTables = tsdbGetNextMaxTables(tid); + + STable **tables = (STable **)calloc(maxTables, sizeof(STable *)); + if (tables == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return -1; + } + + memcpy((void *)tables, (void *)pMeta->tables, sizeof(STable *) * pMeta->maxTables); + pMeta->maxTables = maxTables; + + STable **tTables = pMeta->tables; + pMeta->tables = tables; + tfree(tTables); + tsdbDebug("vgId:%d tsdb meta maxTables is adjusted as %d", REPO_ID(pRepo), maxTables); + return 0; } \ No newline at end of file diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index c82d1f905a..577a0965d8 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -317,17 +317,20 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh } assert(pCheckInfo->iter == NULL && pCheckInfo->iiter == NULL); - - if (pHandle->mem && pHandle->mem->tData[pCheckInfo->tableId.tid] != NULL) { + + // TODO: add uid check + if (pHandle->mem && pCheckInfo->tableId.tid < pHandle->mem->maxTables && + pHandle->mem->tData[pCheckInfo->tableId.tid] != NULL) { pCheckInfo->iter = tSkipListCreateIterFromVal(pHandle->mem->tData[pCheckInfo->tableId.tid]->pData, - (const char*) &pCheckInfo->lastKey, TSDB_DATA_TYPE_TIMESTAMP, order); + (const char*)&pCheckInfo->lastKey, TSDB_DATA_TYPE_TIMESTAMP, order); } - - if (pHandle->imem && pHandle->imem->tData[pCheckInfo->tableId.tid] != NULL) { + + if (pHandle->imem && pCheckInfo->tableId.tid < pHandle->imem->maxTables && + pHandle->imem->tData[pCheckInfo->tableId.tid] != NULL) { pCheckInfo->iiter = tSkipListCreateIterFromVal(pHandle->imem->tData[pCheckInfo->tableId.tid]->pData, - (const char*) &pCheckInfo->lastKey, TSDB_DATA_TYPE_TIMESTAMP, order); + (const char*)&pCheckInfo->lastKey, TSDB_DATA_TYPE_TIMESTAMP, order); } - + // both iterators are NULL, no data in buffer right now if (pCheckInfo->iter == NULL && pCheckInfo->iiter == NULL) { return false; @@ -1529,7 +1532,6 @@ static int32_t getDataBlocksInFiles(STsdbQueryHandle* pQueryHandle, bool* exists static bool doHasDataInBuffer(STsdbQueryHandle* pQueryHandle) { size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); - assert(numOfTables <= ((STsdbRepo*)pQueryHandle->pTsdb)->config.maxTables); while (pQueryHandle->activeIndex < numOfTables) { if (hasMoreDataInCache(pQueryHandle)) { @@ -2418,8 +2420,7 @@ void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle) { tfree(pQueryHandle->statis); // todo check error - tsdbUnRefMemTable(pQueryHandle->pTsdb, pQueryHandle->mem); - tsdbUnRefMemTable(pQueryHandle->pTsdb, pQueryHandle->imem); + tsdbUnTakeMemSnapShot(pQueryHandle->pTsdb, pQueryHandle->mem, pQueryHandle->imem); tsdbDestroyHelper(&pQueryHandle->rhelper); diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index 249fb428e7..dd5ec18439 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -123,7 +123,7 @@ int32_t vnodeCreate(SMDCreateVnodeMsg *pVnodeCfg) { tsdbCfg.tsdbId = pVnodeCfg->cfg.vgId; tsdbCfg.cacheBlockSize = pVnodeCfg->cfg.cacheBlockSize; tsdbCfg.totalBlocks = pVnodeCfg->cfg.totalBlocks; - tsdbCfg.maxTables = pVnodeCfg->cfg.maxTables; + // tsdbCfg.maxTables = pVnodeCfg->cfg.maxTables; tsdbCfg.daysPerFile = pVnodeCfg->cfg.daysPerFile; tsdbCfg.keep = pVnodeCfg->cfg.daysToKeep; tsdbCfg.minRowsPerFileBlock = pVnodeCfg->cfg.minRowsPerFileBlock; @@ -630,14 +630,14 @@ static int32_t vnodeSaveCfg(SMDCreateVnodeMsg *pVnodeCfg) { len += snprintf(content + len, maxLen - len, " \"cfgVersion\": %d,\n", pVnodeCfg->cfg.cfgVersion); len += snprintf(content + len, maxLen - len, " \"cacheBlockSize\": %d,\n", pVnodeCfg->cfg.cacheBlockSize); len += snprintf(content + len, maxLen - len, " \"totalBlocks\": %d,\n", pVnodeCfg->cfg.totalBlocks); - len += snprintf(content + len, maxLen - len, " \"maxTables\": %d,\n", pVnodeCfg->cfg.maxTables); + // len += snprintf(content + len, maxLen - len, " \"maxTables\": %d,\n", pVnodeCfg->cfg.maxTables); len += snprintf(content + len, maxLen - len, " \"daysPerFile\": %d,\n", pVnodeCfg->cfg.daysPerFile); len += snprintf(content + len, maxLen - len, " \"daysToKeep\": %d,\n", pVnodeCfg->cfg.daysToKeep); len += snprintf(content + len, maxLen - len, " \"daysToKeep1\": %d,\n", pVnodeCfg->cfg.daysToKeep1); len += snprintf(content + len, maxLen - len, " \"daysToKeep2\": %d,\n", pVnodeCfg->cfg.daysToKeep2); len += snprintf(content + len, maxLen - len, " \"minRowsPerFileBlock\": %d,\n", pVnodeCfg->cfg.minRowsPerFileBlock); len += snprintf(content + len, maxLen - len, " \"maxRowsPerFileBlock\": %d,\n", pVnodeCfg->cfg.maxRowsPerFileBlock); - len += snprintf(content + len, maxLen - len, " \"commitTime\": %d,\n", pVnodeCfg->cfg.commitTime); + // len += snprintf(content + len, maxLen - len, " \"commitTime\": %d,\n", pVnodeCfg->cfg.commitTime); len += snprintf(content + len, maxLen - len, " \"precision\": %d,\n", pVnodeCfg->cfg.precision); len += snprintf(content + len, maxLen - len, " \"compression\": %d,\n", pVnodeCfg->cfg.compression); len += snprintf(content + len, maxLen - len, " \"walLevel\": %d,\n", pVnodeCfg->cfg.walLevel); @@ -729,12 +729,12 @@ static int32_t vnodeReadCfg(SVnodeObj *pVnode) { } pVnode->tsdbCfg.totalBlocks = totalBlocks->valueint; - cJSON *maxTables = cJSON_GetObjectItem(root, "maxTables"); - if (!maxTables || maxTables->type != cJSON_Number) { - vError("vgId:%d, failed to read vnode cfg, maxTables not found", pVnode->vgId); - goto PARSE_OVER; - } - pVnode->tsdbCfg.maxTables = maxTables->valueint; + // cJSON *maxTables = cJSON_GetObjectItem(root, "maxTables"); + // if (!maxTables || maxTables->type != cJSON_Number) { + // vError("vgId:%d, failed to read vnode cfg, maxTables not found", pVnode->vgId); + // goto PARSE_OVER; + // } + // pVnode->tsdbCfg.maxTables = maxTables->valueint; cJSON *daysPerFile = cJSON_GetObjectItem(root, "daysPerFile"); if (!daysPerFile || daysPerFile->type != cJSON_Number) { @@ -778,12 +778,12 @@ static int32_t vnodeReadCfg(SVnodeObj *pVnode) { } pVnode->tsdbCfg.maxRowsPerFileBlock = maxRowsPerFileBlock->valueint; - cJSON *commitTime = cJSON_GetObjectItem(root, "commitTime"); - if (!commitTime || commitTime->type != cJSON_Number) { - vError("vgId:%d, failed to read vnode cfg, commitTime not found", pVnode->vgId); - goto PARSE_OVER; - } - pVnode->tsdbCfg.commitTime = (int8_t)commitTime->valueint; + // cJSON *commitTime = cJSON_GetObjectItem(root, "commitTime"); + // if (!commitTime || commitTime->type != cJSON_Number) { + // vError("vgId:%d, failed to read vnode cfg, commitTime not found", pVnode->vgId); + // goto PARSE_OVER; + // } + // pVnode->tsdbCfg.commitTime = (int8_t)commitTime->valueint; cJSON *precision = cJSON_GetObjectItem(root, "precision"); if (!precision || precision->type != cJSON_Number) { From 218e83f4ee1054959aed97cc0444865403b55638 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 23 Jul 2020 16:02:28 +0800 Subject: [PATCH 075/100] fix TSDB test file --- src/tsdb/tests/tsdbTests.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tsdb/tests/tsdbTests.cpp b/src/tsdb/tests/tsdbTests.cpp index 88bd524c20..5e678a8ee5 100644 --- a/src/tsdb/tests/tsdbTests.cpp +++ b/src/tsdb/tests/tsdbTests.cpp @@ -98,7 +98,7 @@ static void tsdbSetCfg(STsdbCfg *pCfg, int32_t tsdbId, int32_t cacheBlockSize, i pCfg->tsdbId = tsdbId; pCfg->cacheBlockSize = cacheBlockSize; pCfg->totalBlocks = totalBlocks; - pCfg->maxTables = maxTables; + // pCfg->maxTables = maxTables; pCfg->daysPerFile = daysPerFile; pCfg->keep = keep; pCfg->minRowsPerFileBlock = minRows; From dc8e7ebc8257755ff7149bcff2be3098e93295c2 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 23 Jul 2020 17:56:35 +0800 Subject: [PATCH 076/100] fix invalid write --- src/tsdb/src/tsdbMain.c | 2 +- src/tsdb/src/tsdbMeta.c | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index 11a84cd552..89a19c15af 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -382,7 +382,7 @@ int tsdbGetNextMaxTables(int tid) { int maxTables = TSDB_INIT_NTABLES; while (true) { maxTables = MIN(maxTables, TSDB_MAX_TABLES); - if (tid <= maxTables + 1) break; + if (tid <= maxTables) break; maxTables *= 2; } diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index 8d237ab673..684c87462c 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -781,7 +781,9 @@ static int tsdbAddTableToMeta(STsdbRepo *pRepo, STable *pTable, bool addIdx, boo goto _err; } } else { - if (tsdbAdjustMetaTables(pRepo, TABLE_TID(pTable)) < 0) goto _err; + if (TABLE_TID(pTable) >= pMeta->maxTables) { + if (tsdbAdjustMetaTables(pRepo, TABLE_TID(pTable)) < 0) goto _err; + } if (TABLE_TYPE(pTable) == TSDB_CHILD_TABLE && addIdx) { // add STABLE to the index if (tsdbAddTableIntoIndex(pMeta, pTable, true) < 0) { tsdbDebug("vgId:%d failed to add table %s to meta while add table to index since %s", REPO_ID(pRepo), @@ -789,6 +791,7 @@ static int tsdbAddTableToMeta(STsdbRepo *pRepo, STable *pTable, bool addIdx, boo goto _err; } } + ASSERT(TABLE_TID(pTable) < pMeta->maxTables); pMeta->tables[TABLE_TID(pTable)] = pTable; pMeta->nTables++; } @@ -1271,7 +1274,7 @@ static int tsdbRmTableFromMeta(STsdbRepo *pRepo, STable *pTable) { static int tsdbAdjustMetaTables(STsdbRepo *pRepo, int tid) { STsdbMeta *pMeta = pRepo->tsdbMeta; - if (pMeta->maxTables >= tid) return 0; + ASSERT(tid >= pMeta->maxTables); int maxTables = tsdbGetNextMaxTables(tid); From 03f0231765c9348e4489ac98f77099d419aed797 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 23 Jul 2020 10:05:30 +0000 Subject: [PATCH 077/100] [TD-926] --- src/client/src/tscSQLParser.c | 6 +++--- src/common/inc/tglobal.h | 2 +- src/common/src/tglobal.c | 8 ++++---- src/inc/taoserror.h | 1 + src/inc/tbalance.h | 2 +- src/mnode/src/mnodeBalance.c | 2 +- src/mnode/src/mnodeDnode.c | 14 ++++++++++++-- tests/test/c/insertPerRow.c | 2 +- 8 files changed, 24 insertions(+), 13 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 32d44dfb81..55e74bcc48 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -4700,10 +4700,10 @@ int32_t validateDNodeConfig(tDCLSQL* pOptions) { } else if ((strncasecmp(cfgOptions[tokenBalance].name, pOptionToken->z, pOptionToken->n) == 0) && (cfgOptions[tokenBalance].len == pOptionToken->n)) { SSQLToken* pValToken = &pOptions->a[2]; - int32_t vnodeIndex = 0; - int32_t dnodeIndex = 0; + int32_t vnodeId = 0; + int32_t dnodeId = 0; strdequote(pValToken->z); - bool parseOk = taosCheckBalanceCfgOptions(pValToken->z, &vnodeIndex, &dnodeIndex); + bool parseOk = taosCheckBalanceCfgOptions(pValToken->z, &vnodeId, &dnodeId); if (!parseOk) { return TSDB_CODE_TSC_INVALID_SQL; // options value is invalid } diff --git a/src/common/inc/tglobal.h b/src/common/inc/tglobal.h index 7ba96ceb60..b9afef65d9 100644 --- a/src/common/inc/tglobal.h +++ b/src/common/inc/tglobal.h @@ -174,7 +174,7 @@ bool taosCheckGlobalCfg(); void taosSetAllDebugFlag(); bool taosCfgDynamicOptions(char *msg); int taosGetFqdnPortFromEp(const char *ep, char *fqdn, uint16_t *port); -bool taosCheckBalanceCfgOptions(const char *option, int32_t *vnodeIndex, int32_t *dnodeIndex); +bool taosCheckBalanceCfgOptions(const char *option, int32_t *vnodeId, int32_t *dnodeId); #ifdef __cplusplus } diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index c79b016b93..08349374b1 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -1315,7 +1315,7 @@ int taosGetFqdnPortFromEp(const char *ep, char *fqdn, uint16_t *port) { * alter dnode 1 balance "vnode:1-dnode:2" */ -bool taosCheckBalanceCfgOptions(const char *option, int32_t *vnodeIndex, int32_t *dnodeIndex) { +bool taosCheckBalanceCfgOptions(const char *option, int32_t *vnodeId, int32_t *dnodeId) { int len = strlen(option); if (strncasecmp(option, "vnode:", 6) != 0) { return false; @@ -1331,9 +1331,9 @@ bool taosCheckBalanceCfgOptions(const char *option, int32_t *vnodeIndex, int32_t return false; } - *vnodeIndex = strtol(option + 6, NULL, 10); - *dnodeIndex = strtol(option + pos + 6, NULL, 10); - if (*vnodeIndex <= 1 || *dnodeIndex <= 0) { + *vnodeId = strtol(option + 6, NULL, 10); + *dnodeId = strtol(option + pos + 6, NULL, 10); + if (*vnodeId <= 1 || *dnodeId <= 0) { return false; } diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index 59b2c0220b..6889818062 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -126,6 +126,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_NOT_EXIST, 0, 0x0332, "mnode vgro TAOS_DEFINE_ERROR(TSDB_CODE_MND_NO_REMOVE_MASTER, 0, 0x0333, "mnode cant not remove master") TAOS_DEFINE_ERROR(TSDB_CODE_MND_NO_ENOUGH_DNODES, 0, 0x0334, "mnode no enough dnodes") TAOS_DEFINE_ERROR(TSDB_CODE_MND_CLUSTER_CFG_INCONSISTENT, 0, 0x0335, "mnode cluster cfg inconsistent") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DNODE_CFG_OPTION, 0, 0x0336, "mnode invalid dnode cfg option") TAOS_DEFINE_ERROR(TSDB_CODE_MND_ACCT_ALREADY_EXIST, 0, 0x0340, "mnode accounts already exist") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_ACCT, 0, 0x0341, "mnode invalid account") diff --git a/src/inc/tbalance.h b/src/inc/tbalance.h index 9ee8d73189..f0da4a3747 100644 --- a/src/inc/tbalance.h +++ b/src/inc/tbalance.h @@ -29,7 +29,7 @@ void balanceAsyncNotify(); void balanceSyncNotify(); void balanceReset(); int32_t balanceAllocVnodes(struct SVgObj *pVgroup); -int32_t balanceCfgDnode(struct SDnodeObj *pDnode, const char *option); +int32_t balanceAlterDnode(struct SDnodeObj *pDnode, int32_t vnodeId, int32_t dnodeId); int32_t balanceDropDnode(struct SDnodeObj *pDnode); #ifdef __cplusplus diff --git a/src/mnode/src/mnodeBalance.c b/src/mnode/src/mnodeBalance.c index d2ec6dd36e..95a97bd055 100644 --- a/src/mnode/src/mnodeBalance.c +++ b/src/mnode/src/mnodeBalance.c @@ -28,7 +28,7 @@ void balanceCleanUp() {} void balanceAsyncNotify() {} void balanceSyncNotify() {} void balanceReset() {} -int32_t balanceCfgDnode(struct SDnodeObj *pDnode, const char *option) { return TSDB_CODE_SYN_NOT_ENABLED; } +int32_t balanceAlterDnode(struct SDnodeObj *pDnode, int32_t vnodeId, int32_t dnodeId) { return TSDB_CODE_SYN_NOT_ENABLED; } int32_t balanceAllocVnodes(SVgObj *pVgroup) { void * pIter = NULL; diff --git a/src/mnode/src/mnodeDnode.c b/src/mnode/src/mnodeDnode.c index 26c4b7a3ea..687bb1873b 100644 --- a/src/mnode/src/mnodeDnode.c +++ b/src/mnode/src/mnodeDnode.c @@ -295,10 +295,19 @@ static int32_t mnodeProcessCfgDnodeMsg(SMnodeMsg *pMsg) { } SRpcEpSet epSet = mnodeGetEpSetFromIp(pDnode->dnodeEp); - mnodeDecDnodeRef(pDnode); if (strncasecmp(pCmCfgDnode->config, "balance", 7) == 0) { - return balanceCfgDnode(pDnode, pCmCfgDnode->config + 8); + int32_t vnodeId = 0; + int32_t dnodeId = 0; + bool parseOk = taosCheckBalanceCfgOptions(pCmCfgDnode->config + 8, &vnodeId, &dnodeId); + if (!parseOk) { + mnodeDecDnodeRef(pDnode); + return TSDB_CODE_MND_INVALID_DNODE_CFG_OPTION; + } + + int32_t code = balanceAlterDnode(pDnode, vnodeId, dnodeId); + mnodeDecDnodeRef(pDnode); + return code; } else { SMDCfgDnodeMsg *pMdCfgDnode = rpcMallocCont(sizeof(SMDCfgDnodeMsg)); strcpy(pMdCfgDnode->ep, pCmCfgDnode->ep); @@ -314,6 +323,7 @@ static int32_t mnodeProcessCfgDnodeMsg(SMnodeMsg *pMsg) { mInfo("dnode:%s, is configured by %s", pCmCfgDnode->ep, pMsg->pUser->user); dnodeSendMsgToDnode(&epSet, &rpcMdCfgDnodeMsg); + mnodeDecDnodeRef(pDnode); return TSDB_CODE_SUCCESS; } } diff --git a/tests/test/c/insertPerRow.c b/tests/test/c/insertPerRow.c index 9cb718e486..824c1b925a 100644 --- a/tests/test/c/insertPerRow.c +++ b/tests/test/c/insertPerRow.c @@ -50,7 +50,7 @@ int64_t numOfThreads = 10; int64_t numOfTablesPerThread = 100; char dbName[32] = "db"; char stableName[64] = "st"; -int32_t cache = 4; +int32_t cache = 1; int32_t replica = 3; int32_t days = 10; int32_t interval = 1000; From 42288d325f0f125d1044f97e4c7d85efa361c708 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 23 Jul 2020 18:25:18 +0800 Subject: [PATCH 078/100] add case to test minTablesPerVnode.py case. [TD-988] --- tests/pytest/fulltest.sh | 3 +- tests/pytest/test.sh | 23 +++++ tests/pytest/testMinTablesPerVnode.py | 131 ++++++++++++++++++++++++++ 3 files changed, 156 insertions(+), 1 deletion(-) create mode 100755 tests/pytest/test.sh create mode 100644 tests/pytest/testMinTablesPerVnode.py diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index 904e712fa4..cbe82b2c8c 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -162,6 +162,7 @@ python3 ./test.py -f client/client.py # Misc python3 testCompress.py python3 testNoCompress.py +python3 testMinTablesPerVnode.py # functions python3 ./test.py -f functions/function_avg.py @@ -180,4 +181,4 @@ python3 ./test.py -f functions/function_spread.py python3 ./test.py -f functions/function_stddev.py python3 ./test.py -f functions/function_sum.py python3 ./test.py -f functions/function_top.py -python3 ./test.py -f functions/function_twa.py \ No newline at end of file +python3 ./test.py -f functions/function_twa.py diff --git a/tests/pytest/test.sh b/tests/pytest/test.sh new file mode 100755 index 0000000000..fbb9ba9879 --- /dev/null +++ b/tests/pytest/test.sh @@ -0,0 +1,23 @@ +EXEC_DIR=`dirname "$0"` +if [[ $EXEC_DIR != "." ]] +then + echo "ERROR: Please execute `basename "$0"` in its own directory (for now anyway, pardon the dust)" + exit -1 +fi +CURR_DIR=`pwd` +IN_TDINTERNAL="community" +if [[ "$CURR_DIR" == *"$IN_TDINTERNAL"* ]]; then + TAOS_DIR=$CURR_DIR/../../.. +else + TAOS_DIR=$CURR_DIR/../.. +fi +TAOSD_DIR=`find $TAOS_DIR -name "taosd"|grep bin|head -n1` +LIB_DIR=`echo $TAOSD_DIR|rev|cut -d '/' -f 3,4,5,6|rev`/lib +export PYTHONPATH=$(pwd)/../../src/connector/python/linux/python3 +export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$LIB_DIR + +if [[ "$1" == *"test.py"* ]]; then + python3 ./test.py $@ +else + python3 $1 $@ +fi diff --git a/tests/pytest/testMinTablesPerVnode.py b/tests/pytest/testMinTablesPerVnode.py new file mode 100644 index 0000000000..91cea833e7 --- /dev/null +++ b/tests/pytest/testMinTablesPerVnode.py @@ -0,0 +1,131 @@ +#!/usr/bin/python +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### +# install pip +# pip install src/connector/python/linux/python2/ + +# -*- coding: utf-8 -*- +import sys +import getopt +import subprocess +from distutils.log import warn as printf + +from util.log import * +from util.dnodes import * +from util.cases import * +from util.sql import * + +import taos + + +if __name__ == "__main__": + fileName = "all" + deployPath = "" + testCluster = False + valgrind = 0 + logSql = True + stop = 0 + opts, args = getopt.gnu_getopt(sys.argv[1:], 'l:sgh', [ + 'logSql', 'stop', 'valgrind', 'help']) + for key, value in opts: + if key in ['-h', '--help']: + tdLog.printNoPrefix( + 'A collection of test cases written using Python') + tdLog.printNoPrefix('-l logSql Flag') + tdLog.printNoPrefix('-s stop All dnodes') + tdLog.printNoPrefix('-g valgrind Test Flag') + sys.exit(0) + + if key in ['-l', '--logSql']: + if (value.upper() == "TRUE"): + logSql = True + elif (value.upper() == "FALSE"): + logSql = False + else: + tdLog.printNoPrefix("logSql value %s is invalid" % logSql) + sys.exit(0) + + if key in ['-g', '--valgrind']: + valgrind = 1 + + if key in ['-s', '--stop']: + stop = 1 + + if (stop != 0): + if (valgrind == 0): + toBeKilled = "taosd" + else: + toBeKilled = "valgrind.bin" + + killCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print $2}' | xargs kill -HUP > /dev/null 2>&1" % toBeKilled + + psCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print $2}'" % toBeKilled + processID = subprocess.check_output(psCmd, shell=True) + + while(processID): + os.system(killCmd) + time.sleep(1) + processID = subprocess.check_output(psCmd, shell=True) + + for port in range(6030, 6041): + usePortPID = "lsof -i tcp:%d | grep LISTEn | awk '{print $2}'" % port + processID = subprocess.check_output(usePortPID, shell=True) + + if processID: + killCmd = "kill -TERM %s" % processID + os.system(killCmd) + fuserCmd = "fuser -k -n tcp %d" % port + os.system(fuserCmd) + if valgrind: + time.sleep(2) + + tdLog.info('stop All dnodes') + sys.exit(0) + + tdDnodes.init(deployPath) + tdDnodes.setTestCluster(testCluster) + tdDnodes.setValgrind(valgrind) + + tdDnodes.stopAll() + tdDnodes.addSimExtraCfg("minTablesPerVnode", "100") + tdDnodes.deploy(1) + tdDnodes.start(1) + + host = '127.0.0.1' + + tdLog.info("Procedures for tdengine deployed in %s" % (host)) + + tdCases.logSql(logSql) + + conn = taos.connect( + host, + config=tdDnodes.getSimCfgPath()) + + tdSql.init(conn.cursor(), True) + + tdSql.execute("DROP DATABASE IF EXISTS db") + tdSql.execute("CREATE DATABASE IF NOT EXISTS db") + tdSql.execute("USE db") + + for i in range(0, 100): + tdSql.execute( + "CREATE TABLE IF NOT EXISTS tb%d (ts TIMESTAMP, temperature INT, humidity FLOAT)" % i) + + for i in range(1, 6): + tdSql.execute("INSERT INTO tb99 values (now + %da, %d, %f)" % (i, i, i * 1.0)) + + tdSql.execute("DROP TABLE tb99") + tdSql.execute( + "CREATE TABLE IF NOT EXISTS tb99 (ts TIMESTAMP, temperature INT, humidity FLOAT)") + tdSql.query("SELECT * FROM tb99") + tdSql.checkRows(0) + + conn.close() From c5adbd69897ff7efad29e9a36652527130ecb7de Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 23 Jul 2020 18:25:56 +0800 Subject: [PATCH 079/100] fix bug --- src/tsdb/src/tsdbMemTable.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index 9ca28d06cb..a65d956eb0 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -725,6 +725,7 @@ static int tsdbAdjustMemMaxTables(SMemTable *pMemTable, int maxTables) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; return -1; } + memcpy((void *)pTableData, (void *)pMemTable->tData, sizeof(STableData *) * pMemTable->maxTables); STableData **tData = pMemTable->tData; From 387d3d6e4b14e6b1a89ce28c212bb6e4c48ffb46 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 23 Jul 2020 14:52:42 +0000 Subject: [PATCH 080/100] [TD-926] --- src/inc/taoserror.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index 6889818062..533171d883 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -127,6 +127,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_NO_REMOVE_MASTER, 0, 0x0333, "mnode cant TAOS_DEFINE_ERROR(TSDB_CODE_MND_NO_ENOUGH_DNODES, 0, 0x0334, "mnode no enough dnodes") TAOS_DEFINE_ERROR(TSDB_CODE_MND_CLUSTER_CFG_INCONSISTENT, 0, 0x0335, "mnode cluster cfg inconsistent") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DNODE_CFG_OPTION, 0, 0x0336, "mnode invalid dnode cfg option") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_BALANCE_NOT_ENABLED, 0, 0x0337, "mnode balance not enabled") TAOS_DEFINE_ERROR(TSDB_CODE_MND_ACCT_ALREADY_EXIST, 0, 0x0340, "mnode accounts already exist") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_ACCT, 0, 0x0341, "mnode invalid account") From 746477086ba813c8897be8a5a510e1a1cd1d5188 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 24 Jul 2020 03:35:46 +0000 Subject: [PATCH 081/100] [TD-926] --- src/client/src/tscSQLParser.c | 2 +- src/inc/taoserror.h | 19 +-- tests/script/jenkins/basic.txt | 3 +- tests/script/unique/cluster/alter.sim | 174 ++++++++++++++++++++++++++ tests/tsim/src/simExe.c | 12 +- 5 files changed, 194 insertions(+), 16 deletions(-) create mode 100644 tests/script/unique/cluster/alter.sim diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 55e74bcc48..f58431849c 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -358,7 +358,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { } case TSDB_SQL_CFG_DNODE: { - const char* msg2 = "invalid configure options or values, such as resetlog / debugFlag 135 / balance 'vnode:1-dnode:2' / monitor 1 "; + const char* msg2 = "invalid configure options or values, such as resetlog / debugFlag 135 / balance 'vnode:2-dnode:2' / monitor 1 "; const char* msg3 = "invalid dnode ep"; /* validate the ip address */ diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index 533171d883..c440fd3c06 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -120,14 +120,17 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_SDB_OBJ_NOT_THERE, 0, 0x0323, "sdb object TAOS_DEFINE_ERROR(TSDB_CODE_MND_SDB_INVAID_META_ROW, 0, 0x0324, "sdb invalid meta row") TAOS_DEFINE_ERROR(TSDB_CODE_MND_SDB_INVAID_KEY_TYPE, 0, 0x0325, "sdb invalid key type") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_DNODE_ALREADY_EXIST, 0, 0x0330, "mnode dnode already exist") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_DNODE_NOT_EXIST, 0, 0x0331, "mnode dnode not exist") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_NOT_EXIST, 0, 0x0332, "mnode vgroup not exist") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_NO_REMOVE_MASTER, 0, 0x0333, "mnode cant not remove master") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_NO_ENOUGH_DNODES, 0, 0x0334, "mnode no enough dnodes") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_CLUSTER_CFG_INCONSISTENT, 0, 0x0335, "mnode cluster cfg inconsistent") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DNODE_CFG_OPTION, 0, 0x0336, "mnode invalid dnode cfg option") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_BALANCE_NOT_ENABLED, 0, 0x0337, "mnode balance not enabled") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_DNODE_ALREADY_EXIST, 0, 0x0330, "dnode already exist") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_DNODE_NOT_EXIST, 0, 0x0331, "dnode not exist") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_NOT_EXIST, 0, 0x0332, "vgroup not exist") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_NO_REMOVE_MASTER, 0, 0x0333, "cant not remove master") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_NO_ENOUGH_DNODES, 0, 0x0334, "no enough dnodes") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_CLUSTER_CFG_INCONSISTENT, 0, 0x0335, "cluster cfg inconsistent") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DNODE_CFG_OPTION, 0, 0x0336, "invalid dnode cfg option") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_BALANCE_ENABLED, 0, 0x0337, "balance already enabled") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_NOT_IN_DNODE, 0, 0x0338, "vgroup not in dnode") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_ALREADY_IN_DNODE, 0, 0x0339, "vgroup already in dnode") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_DNODE_NOT_FREE, 0, 0x033A, "dnode not avaliable") TAOS_DEFINE_ERROR(TSDB_CODE_MND_ACCT_ALREADY_EXIST, 0, 0x0340, "mnode accounts already exist") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_ACCT, 0, 0x0341, "mnode invalid account") diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index e953d337e4..e0f6b7b197 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -249,6 +249,7 @@ cd ../../../debug; make ./test.sh -f unique/big/maxvnodes.sim ./test.sh -f unique/big/tcp.sim +./test.sh -f unique/cluster/alter.sim ./test.sh -f unique/cluster/balance1.sim ./test.sh -f unique/cluster/balance2.sim ./test.sh -f unique/cluster/balance3.sim @@ -325,7 +326,7 @@ cd ../../../debug; make #./test.sh -f unique/arbitrator/dn2_mn1_cache_file_sync.sim ./test.sh -f unique/arbitrator/dn3_mn1_full_createTableFail.sim ./test.sh -f unique/arbitrator/dn3_mn1_multiCreateDropTable.sim -./test.sh -f unique/arbitrator/dn3_mn1_nw_disable_timeout_autoDropDnode.sim +#./test.sh -f unique/arbitrator/dn3_mn1_nw_disable_timeout_autoDropDnode.sim #./test.sh -f unique/arbitrator/dn3_mn1_replica2_wal1_AddDelDnode.sim ./test.sh -f unique/arbitrator/dn3_mn1_replica_change_dropDnod.sim ./test.sh -f unique/arbitrator/dn3_mn1_replica_change.sim diff --git a/tests/script/unique/cluster/alter.sim b/tests/script/unique/cluster/alter.sim new file mode 100644 index 0000000000..7e56707a69 --- /dev/null +++ b/tests/script/unique/cluster/alter.sim @@ -0,0 +1,174 @@ +system sh/stop_dnodes.sh + +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 +system sh/deploy.sh -n dnode4 -i 4 + +system sh/cfg.sh -n dnode1 -c role -v 1 +system sh/cfg.sh -n dnode2 -c role -v 2 +system sh/cfg.sh -n dnode3 -c role -v 2 +system sh/cfg.sh -n dnode4 -c role -v 2 + +system sh/cfg.sh -n dnode1 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode2 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode3 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode4 -c balanceInterval -v 10 + +system sh/cfg.sh -n dnode1 -c wallevel -v 2 +system sh/cfg.sh -n dnode2 -c wallevel -v 2 +system sh/cfg.sh -n dnode3 -c wallevel -v 2 +system sh/cfg.sh -n dnode4 -c wallevel -v 2 + +system sh/cfg.sh -n dnode1 -c balance -v 0 +system sh/cfg.sh -n dnode2 -c balance -v 0 +system sh/cfg.sh -n dnode3 -c balance -v 0 +system sh/cfg.sh -n dnode4 -c balance -v 0 + +print ========== step1 +system sh/exec.sh -n dnode1 -s start +sql connect +sleep 3000 + +sql create dnode $hostname2 +system sh/exec.sh -n dnode2 -s start +sleep 3000 + +print ========== step2 +sql create database d1 +sql create table d1.t1 (t timestamp, i int) +sql insert into d1.t1 values(now+1s, 15) +sql insert into d1.t1 values(now+2s, 14) +sql insert into d1.t1 values(now+3s, 13) +sql insert into d1.t1 values(now+4s, 12) +sql insert into d1.t1 values(now+5s, 11) + +print ========== step3 +sleep 2000 +sql create dnode $hostname3 +system sh/exec.sh -n dnode3 -s start +sql create dnode $hostname4 +system sh/exec.sh -n dnode4 -s start + +sql show dnodes +print dnode1 openVnodes $data2_1 +print dnode2 openVnodes $data2_2 +print dnode3 openVnodes $data2_3 +print dnode4 openVnodes $data2_4 +if $data2_1 != 0 then + return -1 +endi +if $data2_2 != 1 then + return -1 +endi +if $data2_3 != 0 then + return -1 +endi +if $data2_4 != 0 then + return -1 +endi + +print ========== step4 +sql alter dnode 2 balance "vnode:2-dnode:3" + +$x = 0 +show4: + $x = $x + 1 + sleep 2000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 openVnodes $data2_1 +print dnode2 openVnodes $data2_2 +print dnode3 openVnodes $data2_3 +print dnode4 openVnodes $data2_4 +if $data2_2 != 0 then + goto show4 +endi +if $data2_3 != 1 then + goto show4 +endi +if $data2_4 != 0 then + goto show4 +endi + +print ========== step5 +sql alter dnode 3 balance "vnode:2-dnode:4" + +$x = 0 +show5: + $x = $x + 1 + sleep 2000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 openVnodes $data2_1 +print dnode2 openVnodes $data2_2 +print dnode3 openVnodes $data2_3 +print dnode4 openVnodes $data2_4 +if $data2_2 != 0 then + goto show5 +endi +if $data2_3 != 0 then + goto show5 +endi +if $data2_4 != 1 then + goto show5 +endi + +print ========== step6 +sql alter dnode 4 balance "vnode:2-dnode:2" + +$x = 0 +show6: + $x = $x + 1 + sleep 2000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 openVnodes $data2_1 +print dnode2 openVnodes $data2_2 +print dnode3 openVnodes $data2_3 +print dnode4 openVnodes $data2_4 +if $data2_2 != 1 then + goto show6 +endi +if $data2_3 != 0 then + goto show6 +endi +if $data2_4 != 0 then + goto show6 +endi + +print ========== step7 +sql select * from d1.t1 order by t desc +print $data01 $data11 $data21 $data31 $data41 +if $data01 != 11 then + return -1 +endi +if $data11 != 12 then + return -1 +endi +if $data21 != 13 then + return -1 +endi +if $data31 != 14 then + return -1 +endi +if $data41 != 15 then + return -1 +endi + +print ========== step8 +sql_error sql alter dnode 4 balance "vnode:2-dnode:5" + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT +system sh/exec.sh -n dnode4 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/tsim/src/simExe.c b/tests/tsim/src/simExe.c index 3febb927c4..3469180607 100644 --- a/tests/tsim/src/simExe.c +++ b/tests/tsim/src/simExe.c @@ -670,12 +670,12 @@ bool simExecuteNativeSqlCommand(SScript *script, char *rest, bool isSlow) { ret = taos_errno(pSql); if (ret == TSDB_CODE_MND_TABLE_ALREADY_EXIST || ret == TSDB_CODE_MND_DB_ALREADY_EXIST) { - simDebug("script:%s, taos:%p, %s success, ret:%d:%s", script->fileName, script->taos, rest, ret, tstrerror(ret)); + simDebug("script:%s, taos:%p, %s success, ret:%d:%s", script->fileName, script->taos, rest, ret & 0XFFFF, tstrerror(ret)); ret = 0; break; } else if (ret != 0) { simDebug("script:%s, taos:%p, %s failed, ret:%d:%s, error:%s", - script->fileName, script->taos, rest, ret, tstrerror(ret), taos_errstr(pSql)); + script->fileName, script->taos, rest, ret & 0XFFFF, tstrerror(ret), taos_errstr(pSql)); if (line->errorJump == SQL_JUMP_TRUE) { script->linePos = line->jump; @@ -691,7 +691,7 @@ bool simExecuteNativeSqlCommand(SScript *script, char *rest, bool isSlow) { } if (ret) { - sprintf(script->error, "lineNum:%d. sql:%s failed, ret:%d:%s", line->lineNum, rest, ret, tstrerror(ret)); + sprintf(script->error, "lineNum:%d. sql:%s failed, ret:%d:%s", line->lineNum, rest, ret & 0XFFFF, tstrerror(ret)); return false; } @@ -821,7 +821,7 @@ bool simExecuteRestFulSqlCommand(SScript *script, char *rest) { ret = simExecuteRestFulCommand(script, command); if (ret == TSDB_CODE_MND_TABLE_ALREADY_EXIST || ret == TSDB_CODE_MND_DB_ALREADY_EXIST) { - simDebug("script:%s, taos:%p, %s success, ret:%d:%s", script->fileName, script->taos, rest, ret, tstrerror(ret)); + simDebug("script:%s, taos:%p, %s success, ret:%d:%s", script->fileName, script->taos, rest, ret & 0XFFFF, tstrerror(ret)); ret = 0; break; } else if (ret != 0) { @@ -957,12 +957,12 @@ bool simExecuteSqlErrorCmd(SScript *script, char *rest) { if (ret != TSDB_CODE_SUCCESS) { simDebug("script:%s, taos:%p, %s execute, expect failed, so success, ret:%d:%s", - script->fileName, script->taos, rest, ret, tstrerror(ret)); + script->fileName, script->taos, rest, ret & 0XFFFF, tstrerror(ret)); script->linePos++; return true; } - sprintf(script->error, "lineNum:%d. sql:%s expect failed, but success, ret:%d:%s", line->lineNum, rest, ret, tstrerror(ret)); + sprintf(script->error, "lineNum:%d. sql:%s expect failed, but success, ret:%d:%s", line->lineNum, rest, ret & 0XFFFF, tstrerror(ret)); return false; } From 64acc4e4f3eab7b6f867153c71716e61064cfd0f Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 24 Jul 2020 11:46:11 +0800 Subject: [PATCH 082/100] update a offset --- src/tsdb/src/tsdbRWHelper.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index 3063f22a28..27f95f31de 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -1231,8 +1231,8 @@ static int tsdbLoadColData(SRWHelper *pHelper, SFile *pFile, SCompBlock *pCompBl if (tsdbCheckAndDecodeColumnData(pDataCol, pHelper->pBuffer, pCompCol->len, pCompBlock->algorithm, pCompBlock->numOfRows, pHelper->pRepo->config.maxRowsPerFileBlock, pHelper->compBuffer, tsizeof(pHelper->compBuffer)) < 0) { - tsdbError("vgId:%d file %s is broken at column %d offset %" PRId64, REPO_ID(pHelper->pRepo), pFile->fname, pCompCol->colId, - (int64_t)pCompCol->offset); + tsdbError("vgId:%d file %s is broken at column %d offset %" PRId64, REPO_ID(pHelper->pRepo), pFile->fname, + pCompCol->colId, offset); return -1; } From c9f72ce620270bdfec5b129fd0ca6cd582e3ade1 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 24 Jul 2020 04:01:21 +0000 Subject: [PATCH 083/100] [TD-977] --- src/inc/taosmsg.h | 1 + src/mnode/src/mnodeTable.c | 95 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 96 insertions(+) diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index 96386aab65..e095567d75 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -149,6 +149,7 @@ enum _mgmt_table { #define TSDB_ALTER_TABLE_ADD_COLUMN 5 #define TSDB_ALTER_TABLE_DROP_COLUMN 6 +#define TSDB_ALTER_TABLE_CHANGE_COLUMN 7 #define TSDB_FILL_NONE 0 #define TSDB_FILL_NULL 1 diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index 12ab58d949..dd106af692 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -1223,6 +1223,55 @@ static int32_t mnodeDropSuperTableColumn(SMnodeMsg *pMsg, char *colName) { return code; } +static int32_t mnodeChangeSuperTableColumnCb(SMnodeMsg *pMsg, int32_t code) { + SSuperTableObj *pStable = (SSuperTableObj *)pMsg->pTable; + mLInfo("app:%p:%p, stable %s, change column result:%s", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId, + tstrerror(code)); + return code; +} + +static int32_t mnodeChangeSuperTableColumn(SMnodeMsg *pMsg, char *oldName, char *newName) { + SSuperTableObj *pStable = (SSuperTableObj *)pMsg->pTable; + int32_t col = mnodeFindSuperTableColumnIndex(pStable, oldName); + if (col < 0) { + mError("app:%p:%p, stable:%s, change column, oldName: %s, newName: %s", pMsg->rpcMsg.ahandle, pMsg, + pStable->info.tableId, oldName, newName); + return TSDB_CODE_MND_FIELD_NOT_EXIST; + } + + // int32_t rowSize = 0; + uint32_t len = strlen(newName); + if (len >= TSDB_COL_NAME_LEN) { + return TSDB_CODE_MND_COL_NAME_TOO_LONG; + } + + if (mnodeFindSuperTableColumnIndex(pStable, newName) >= 0) { + return TSDB_CODE_MND_FIELD_ALREAY_EXIST; + } + + // update + SSchema *schema = (SSchema *) (pStable->schema + col); + tstrncpy(schema->name, newName, sizeof(schema->name)); + + mInfo("app:%p:%p, stable %s, start to modify column %s to %s", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId, + oldName, newName); + + SSdbOper oper = { + .type = SDB_OPER_GLOBAL, + .table = tsSuperTableSdb, + .pObj = pStable, + .pMsg = pMsg, + .cb = mnodeChangeSuperTableColumnCb + }; + + int32_t code = sdbUpdateRow(&oper); + if (code == TSDB_CODE_SUCCESS) { + code = TSDB_CODE_MND_ACTION_IN_PROGRESS; + } + + return code; +} + // show super tables static int32_t mnodeGetShowSuperTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { SDbObj *pDb = mnodeGetDb(pShow->db); @@ -1977,6 +2026,48 @@ static int32_t mnodeDropNormalTableColumn(SMnodeMsg *pMsg, char *colName) { return code; } +static int32_t mnodeChangeNormalTableColumn(SMnodeMsg *pMsg, char *oldName, char *newName) { + SChildTableObj *pTable = (SChildTableObj *)pMsg->pTable; + int32_t col = mnodeFindNormalTableColumnIndex(pTable, oldName); + if (col < 0) { + mError("app:%p:%p, ctable:%s, change column, oldName: %s, newName: %s", pMsg->rpcMsg.ahandle, pMsg, + pTable->info.tableId, oldName, newName); + return TSDB_CODE_MND_FIELD_NOT_EXIST; + } + + // int32_t rowSize = 0; + uint32_t len = strlen(newName); + if (len >= TSDB_COL_NAME_LEN) { + return TSDB_CODE_MND_COL_NAME_TOO_LONG; + } + + if (mnodeFindNormalTableColumnIndex(pTable, newName) >= 0) { + return TSDB_CODE_MND_FIELD_ALREAY_EXIST; + } + + // update + SSchema *schema = (SSchema *) (pTable->schema + col); + tstrncpy(schema->name, newName, sizeof(schema->name)); + + mInfo("app:%p:%p, ctable %s, start to modify column %s to %s", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId, + oldName, newName); + + SSdbOper oper = { + .type = SDB_OPER_GLOBAL, + .table = tsChildTableSdb, + .pObj = pTable, + .pMsg = pMsg, + .cb = mnodeAlterNormalTableColumnCb + }; + + int32_t code = sdbUpdateRow(&oper); + if (code == TSDB_CODE_SUCCESS) { + code = TSDB_CODE_MND_ACTION_IN_PROGRESS; + } + + return code; +} + static int32_t mnodeSetSchemaFromNormalTable(SSchema *pSchema, SChildTableObj *pTable) { int32_t numOfCols = pTable->numOfColumns; for (int32_t i = 0; i < numOfCols; ++i) { @@ -2596,6 +2687,8 @@ static int32_t mnodeProcessAlterTableMsg(SMnodeMsg *pMsg) { code = mnodeAddSuperTableColumn(pMsg, pAlter->schema, 1); } else if (pAlter->type == TSDB_ALTER_TABLE_DROP_COLUMN) { code = mnodeDropSuperTableColumn(pMsg, pAlter->schema[0].name); + } else if (pAlter->type == TSDB_ALTER_TABLE_CHANGE_COLUMN) { + code = mnodeChangeSuperTableColumn(pMsg, pAlter->schema[0].name, pAlter->schema[1].name); } else { } } else { @@ -2606,6 +2699,8 @@ static int32_t mnodeProcessAlterTableMsg(SMnodeMsg *pMsg) { code = mnodeAddNormalTableColumn(pMsg, pAlter->schema, 1); } else if (pAlter->type == TSDB_ALTER_TABLE_DROP_COLUMN) { code = mnodeDropNormalTableColumn(pMsg, pAlter->schema[0].name); + } else if (pAlter->type == TSDB_ALTER_TABLE_CHANGE_COLUMN) { + code = mnodeChangeNormalTableColumn(pMsg, pAlter->schema[0].name, pAlter->schema[1].name); } else { } } From 3ca44498bfd79061c0e535f0b9a3bc7d62bad07b Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Fri, 24 Jul 2020 13:57:53 +0800 Subject: [PATCH 084/100] TD-905: fix some crash when memory allocation fail --- src/mnode/src/mnodeTable.c | 3 +++ src/query/src/qExecutor.c | 6 ++++-- src/query/src/qUtil.c | 3 +++ src/tsdb/src/tsdbMeta.c | 4 +++- src/tsdb/src/tsdbRead.c | 5 ++++- 5 files changed, 17 insertions(+), 4 deletions(-) diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index 12ab58d949..795a4d3621 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -1405,6 +1405,9 @@ static int32_t mnodeSetSchemaFromSuperTable(SSchema *pSchema, SSuperTableObj *pT static int32_t mnodeGetSuperTableMeta(SMnodeMsg *pMsg) { SSuperTableObj *pTable = (SSuperTableObj *)pMsg->pTable; STableMetaMsg *pMeta = rpcMallocCont(sizeof(STableMetaMsg) + sizeof(SSchema) * (TSDB_MAX_TAGS + TSDB_MAX_COLUMNS + 16)); + if (pMeta == NULL) { + return TSDB_CODE_MND_OUT_OF_MEMORY; + } pMeta->uid = htobe64(pTable->uid); pMeta->sversion = htons(pTable->sversion); pMeta->tversion = htons(pTable->tversion); diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 1220c5ca31..8afc2fd87a 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -5913,8 +5913,10 @@ _cleanup_qinfo: tsdbDestroyTableGroup(pTableGroupInfo); _cleanup_query: - taosArrayDestroy(pGroupbyExpr->columnInfo); - tfree(pGroupbyExpr); + if (pGroupbyExpr != NULL) { + taosArrayDestroy(pGroupbyExpr->columnInfo); + free(pGroupbyExpr); + } tfree(pTagCols); for (int32_t i = 0; i < numOfOutput; ++i) { SExprInfo* pExprInfo = &pExprs[i]; diff --git a/src/query/src/qUtil.c b/src/query/src/qUtil.c index be84471493..7b03dd9d94 100644 --- a/src/query/src/qUtil.c +++ b/src/query/src/qUtil.c @@ -41,6 +41,9 @@ int32_t initWindowResInfo(SWindowResInfo *pWindowResInfo, SQueryRuntimeEnv *pRun pWindowResInfo->type = type; _hash_fn_t fn = taosGetDefaultHashFunction(type); pWindowResInfo->hashList = taosHashInit(threshold, fn, false); + if (pWindowResInfo->hashList == NULL) { + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } pWindowResInfo->curIndex = -1; pWindowResInfo->size = 0; diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index b25e734694..d7ed879fa5 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -1215,7 +1215,9 @@ static void *tsdbInsertTableAct(STsdbRepo *pRepo, int8_t act, void *buf, STable static int tsdbRemoveTableFromStore(STsdbRepo *pRepo, STable *pTable) { int tlen = tsdbGetTableEncodeSize(TSDB_DROP_META, pTable); void *buf = tsdbAllocBytes(pRepo, tlen); - ASSERT(buf != NULL); + if (buf == NULL) { + return -1; + } void *pBuf = buf; if (TABLE_TYPE(pTable) == TSDB_SUPER_TABLE) { diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index c82d1f905a..974819d4de 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -175,6 +175,9 @@ static SArray* getDefaultLoadColumns(STsdbQueryHandle* pQueryHandle, bool loadTS TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STableGroupInfo* groupList, void* qinfo) { STsdbQueryHandle* pQueryHandle = calloc(1, sizeof(STsdbQueryHandle)); + if (pQueryHandle == NULL) { + goto out_of_memory; + } pQueryHandle->order = pCond->order; pQueryHandle->window = pCond->twindow; pQueryHandle->pTsdb = tsdb; @@ -260,8 +263,8 @@ TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STab return (TsdbQueryHandleT) pQueryHandle; out_of_memory: - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; tsdbCleanupQueryHandle(pQueryHandle); + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; return NULL; } From f6c113954cfd9064b6df32eacaa0841ff7a08ca4 Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Fri, 24 Jul 2020 15:44:40 +0800 Subject: [PATCH 085/100] td-990: add python test cases for subscribe --- src/client/src/tscSub.c | 3 +- tests/pytest/subscribe/__init__.py | 0 tests/pytest/subscribe/singlemeter.py | 79 ++++++++++++++++++ tests/pytest/subscribe/supertable.py | 114 ++++++++++++++++++++++++++ tests/pytest/util/sub.py | 43 ++++++++++ 5 files changed, 238 insertions(+), 1 deletion(-) create mode 100644 tests/pytest/subscribe/__init__.py create mode 100644 tests/pytest/subscribe/singlemeter.py create mode 100644 tests/pytest/subscribe/supertable.py create mode 100644 tests/pytest/util/sub.py diff --git a/src/client/src/tscSub.c b/src/client/src/tscSub.c index 526f380475..8e554d62ff 100644 --- a/src/client/src/tscSub.c +++ b/src/client/src/tscSub.c @@ -256,11 +256,12 @@ static int tscUpdateSubscription(STscObj* pObj, SSub* pSub) { } size_t numOfTables = taosArrayGetSize(tables); + SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); SArray* progress = taosArrayInit(numOfTables, sizeof(SSubscriptionProgress)); for( size_t i = 0; i < numOfTables; i++ ) { STidTags* tt = taosArrayGet( tables, i ); SSubscriptionProgress p = { .uid = tt->uid }; - p.key = tscGetSubscriptionProgress(pSub, tt->uid, INT64_MIN); + p.key = tscGetSubscriptionProgress(pSub, tt->uid, pQueryInfo->window.skey); taosArrayPush(progress, &p); } taosArraySort(progress, tscCompareSubscriptionProgress); diff --git a/tests/pytest/subscribe/__init__.py b/tests/pytest/subscribe/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/pytest/subscribe/singlemeter.py b/tests/pytest/subscribe/singlemeter.py new file mode 100644 index 0000000000..879e0a75eb --- /dev/null +++ b/tests/pytest/subscribe/singlemeter.py @@ -0,0 +1,79 @@ +################################################################### + # Copyright (c) 2020 by TAOS Technologies, Inc. + # All rights reserved. + # + # This file is proprietary and confidential to TAOS Technologies. + # No part of this file may be reproduced, stored, transmitted, + # disclosed or used in any form or by any means other than as + # expressly provided by the written permission from Jianhui Tao + # +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +import time +from util.log import * +from util.cases import * +from util.sql import * +from util.sub import * + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + self.conn = conn + + def run(self): + sqlstr = "select * from t0" + topic = "test" + now = int(time.time() * 1000) + tdSql.prepare() + + tdLog.info("create a table and insert 10 rows.") + tdSql.execute("create table t0(ts timestamp, a int, b int);") + for i in range(0, 10): + tdSql.execute("insert into t0 values (%d, %d, %d);" % (now + i, i, i)) + + tdLog.info("consumption 01.") + tdSub.init(self.conn.subscribe(True, topic, sqlstr, 0)) + tdSub.consume() + tdSub.checkRows(10) + + tdLog.info("consumption 02: no new rows inserted") + tdSub.consume() + tdSub.checkRows(0) + + tdLog.info("consumption 03: after one new rows inserted") + tdSql.execute("insert into t0 values (%d, 10, 10);" % (now + 10)) + tdSub.consume() + tdSub.checkRows(1) + + tdLog.info("consumption 04: keep progress and continue previous subscription") + tdSub.close(True) + tdSub.init(self.conn.subscribe(False, topic, sqlstr, 0)) + tdSub.consume() + tdSub.checkRows(0) + + tdLog.info("consumption 05: remove progress and continue previous subscription") + tdSub.close(False) + tdSub.init(self.conn.subscribe(False, topic, sqlstr, 0)) + tdSub.consume() + tdSub.checkRows(11) + + tdLog.info("consumption 06: keep progress and restart the subscription") + tdSub.close(True) + tdSub.init(self.conn.subscribe(True, topic, sqlstr, 0)) + tdSub.consume() + tdSub.checkRows(11) + + tdSub.close(True) + + def stop(self): + tdSub.close(False) + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/subscribe/supertable.py b/tests/pytest/subscribe/supertable.py new file mode 100644 index 0000000000..c6cc2969aa --- /dev/null +++ b/tests/pytest/subscribe/supertable.py @@ -0,0 +1,114 @@ +################################################################### + # Copyright (c) 2020 by TAOS Technologies, Inc. + # All rights reserved. + # + # This file is proprietary and confidential to TAOS Technologies. + # No part of this file may be reproduced, stored, transmitted, + # disclosed or used in any form or by any means other than as + # expressly provided by the written permission from Jianhui Tao + # +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +import time +from util.log import * +from util.cases import * +from util.sql import * +from util.sub import * + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + self.conn = conn + + def run(self): + sqlstr = "select * from meters" + topic = "test" + now = int(time.time() * 1000) + tdSql.prepare() + + tdLog.info("create a super table and 10 sub-tables, then insert 5 rows into each sub-table.") + tdSql.execute("create table meters(ts timestamp, a int, b int) tags(area int, loc binary(20));") + for i in range(0, 10): + for j in range(0, 5): + tdSql.execute("insert into t%d using meters tags(%d, 'area%d') values (%d, %d, %d);" % (i, i, i, now + j, j, j)) + + tdLog.info("consumption 01.") + tdSub.init(self.conn.subscribe(True, topic, sqlstr, 0)) + tdSub.consume() + tdSub.checkRows(50) + + tdLog.info("consumption 02: no new rows inserted") + tdSub.consume() + tdSub.checkRows(0) + + tdLog.info("consumption 03: after one new rows inserted") + tdSql.execute("insert into t0 values (%d, 10, 10);" % (now + 10)) + tdSub.consume() + tdSub.checkRows(1) + + tdLog.info("consumption 04: keep progress and continue previous subscription") + tdSub.close(True) + tdSub.init(self.conn.subscribe(False, topic, sqlstr, 0)) + tdSub.consume() + tdSub.checkRows(0) + + tdLog.info("consumption 05: remove progress and continue previous subscription") + tdSub.close(False) + tdSub.init(self.conn.subscribe(False, topic, sqlstr, 0)) + tdSub.consume() + tdSub.checkRows(51) + + tdLog.info("consumption 06: keep progress and restart the subscription") + tdSub.close(True) + tdSub.init(self.conn.subscribe(True, topic, sqlstr, 0)) + tdSub.consume() + tdSub.checkRows(51) + + tdLog.info("consumption 07: insert one row to two table then remove one table") + tdSql.execute("insert into t0 values (%d, 11, 11);" % (now + 11)) + tdSql.execute("insert into t1 values (%d, 11, 11);" % (now + 11)) + tdSql.execute("drop table t0") + tdSub.consume() + tdSub.checkRows(1) + + tdLog.info("consumption 08: check timestamp criteria") + tdSub.close(False) + tdSub.init(self.conn.subscribe(True, topic, sqlstr + " where ts > %d" % now, 0)) + tdSub.consume() + tdSub.checkRows(37) + + tdLog.info("consumption 09: insert large timestamp to t2 then insert smaller timestamp to t1") + tdSql.execute("insert into t2 values (%d, 100, 100);" % (now + 100)) + tdSub.consume() + tdSub.checkRows(1) + tdSql.execute("insert into t1 values (%d, 12, 12);" % (now + 12)) + tdSub.consume() + tdSub.checkRows(1) + + tdLog.info("consumption 10: field criteria") + tdSub.close(True) + tdSub.init(self.conn.subscribe(False, topic, sqlstr + " where a > 100", 0)) + tdSql.execute("insert into t2 values (%d, 101, 100);" % (now + 101)) + tdSql.execute("insert into t2 values (%d, 100, 100);" % (now + 102)) + tdSql.execute("insert into t2 values (%d, 102, 100);" % (now + 103)) + tdSub.consume() + tdSub.checkRows(2) + + tdLog.info("consumption 11: two vnodes") + tdSql.execute("insert into t2 values (%d, 102, 100);" % (now + 104)) + tdSql.execute("insert into t9 values (%d, 102, 100);" % (now + 104)) + tdSub.consume() + tdSub.checkRows(2) + + def stop(self): + tdSub.close(False) + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/util/sub.py b/tests/pytest/util/sub.py new file mode 100644 index 0000000000..2e3c2a96b7 --- /dev/null +++ b/tests/pytest/util/sub.py @@ -0,0 +1,43 @@ +################################################################### + # Copyright (c) 2020 by TAOS Technologies, Inc. + # All rights reserved. + # + # This file is proprietary and confidential to TAOS Technologies. + # No part of this file may be reproduced, stored, transmitted, + # disclosed or used in any form or by any means other than as + # expressly provided by the written permission from Jianhui Tao + # +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import os +import time +import datetime +from util.log import * + +class TDSub: + def __init__(self): + self.consumedRows = 0 + self.consumedCols = 0 + + def init(self, sub): + self.sub = sub + + def close(self, keepProgress): + self.sub.close(keepProgress) + + def consume(self): + self.data = self.sub.consume() + self.consumedRows = len(self.data) + self.consumedCols = len(self.sub.fields) + return self.consumedRows + + def checkRows(self, expectRows): + if self.consumedRows != expectRows: + tdLog.exit("consumed rows:%d != expect:%d" % (self.consumedRows, expectRows)) + tdLog.info("consumed rows:%d == expect:%d" % (self.consumedRows, expectRows)) + + +tdSub = TDSub() From a9cf6cfd2c2d7374b8d0fb051c6255cc4764e88c Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Fri, 24 Jul 2020 16:40:05 +0800 Subject: [PATCH 086/100] update kill parameter for python tests --- tests/pytest/util/dnodes.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py index 582bd0abae..ec3865f4f2 100644 --- a/tests/pytest/util/dnodes.py +++ b/tests/pytest/util/dnodes.py @@ -351,7 +351,7 @@ class TDDnodes: psCmd = "ps -ef|grep -w taosd| grep -v grep | awk '{print $2}'" processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") while(processID): - killCmd = "kill -KILL %s > /dev/null 2>&1" % processID + killCmd = "kill -TERM %s > /dev/null 2>&1" % processID os.system(killCmd) time.sleep(1) processID = subprocess.check_output( @@ -360,7 +360,7 @@ class TDDnodes: psCmd = "ps -ef|grep -w valgrind.bin| grep -v grep | awk '{print $2}'" processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") while(processID): - killCmd = "kill -KILL %s > /dev/null 2>&1" % processID + killCmd = "kill -TERM %s > /dev/null 2>&1" % processID os.system(killCmd) time.sleep(1) processID = subprocess.check_output( @@ -467,7 +467,7 @@ class TDDnodes: psCmd = "ps -ef|grep -w taosd| grep -v grep | awk '{print $2}'" processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") while(processID): - killCmd = "kill -KILL %s > /dev/null 2>&1" % processID + killCmd = "kill -TERM %s > /dev/null 2>&1" % processID os.system(killCmd) time.sleep(1) processID = subprocess.check_output( @@ -476,7 +476,7 @@ class TDDnodes: psCmd = "ps -ef|grep -w valgrind.bin| grep -v grep | awk '{print $2}'" processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") while(processID): - killCmd = "kill -KILL %s > /dev/null 2>&1" % processID + killCmd = "kill -TERM %s > /dev/null 2>&1" % processID os.system(killCmd) time.sleep(1) processID = subprocess.check_output( From ab90a5ee71bd325216396ecdf974d78d3eb2fdcc Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 24 Jul 2020 17:47:09 +0800 Subject: [PATCH 087/100] fix last file bug --- src/tsdb/src/tsdbRWHelper.c | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index 27f95f31de..cc6494fd4b 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -147,6 +147,7 @@ int tsdbSetAndOpenHelperFile(SRWHelper *pHelper, SFileGroup *pGroup) { if (tsdbOpenFile(pFile, O_WRONLY | O_CREAT) < 0) goto _err; pFile->info.size = TSDB_FILE_HEAD_SIZE; pFile->info.magic = TSDB_FILE_INIT_MAGIC; + pFile->info.len = 0; if (tsdbUpdateFileHeader(pFile, 0) < 0) return -1; } } else { @@ -555,10 +556,6 @@ int tsdbLoadCompIdx(SRWHelper *pHelper, void *target) { } helperSetState(pHelper, TSDB_HELPER_IDX_LOAD); - if (helperType(pHelper) == TSDB_WRITE_HELPER) { - pFile->info.len = 0; - } - // Copy the memory for outside usage if (target && pHelper->idxH.numOfIdx > 0) memcpy(target, pHelper->idxH.pIdxArray, sizeof(SCompIdx) * pHelper->idxH.numOfIdx); @@ -1517,8 +1514,8 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, if (rows2 == 0) { // all data filtered out *(pCommitIter->pIter) = slIter; } else { - if (rows1 + rows2 < pCfg->minRowsPerFileBlock && pCompBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS && - !TSDB_NLAST_FILE_OPENED(pHelper)) { + if (pCompBlock->numOfRows + rows2 < pCfg->minRowsPerFileBlock && + pCompBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS && !TSDB_NLAST_FILE_OPENED(pHelper)) { tdResetDataCols(pDataCols); int rowsRead = tsdbLoadDataFromCache(pTable, pCommitIter->pIter, maxKey, rows1, pDataCols, pDataCols0->cols[0].pData, pDataCols0->numOfRows); From 661d64f3f63252c884532b48d6c2d66109caf2de Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Fri, 24 Jul 2020 11:38:59 +0000 Subject: [PATCH 088/100] reset the dest ID if link ID is mismatched --- src/rpc/src/rpcMain.c | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index 4790c22668..95840af1a2 100644 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -660,7 +660,7 @@ static SRpcConn *rpcAllocateClientConn(SRpcInfo *pRpc) { pConn->spi = pRpc->spi; pConn->encrypt = pRpc->encrypt; if (pConn->spi) memcpy(pConn->secret, pRpc->secret, TSDB_KEY_LEN); - tDebug("%s %p client connection is allocated", pRpc->label, pConn); + tDebug("%s %p client connection is allocated, uid:0x%x", pRpc->label, pConn, pConn->linkUid); } return pConn; @@ -721,7 +721,7 @@ static SRpcConn *rpcAllocateServerConn(SRpcInfo *pRpc, SRecvInfo *pRecv) { } taosHashPut(pRpc->hash, hashstr, size, (char *)&pConn, POINTER_BYTES); - tDebug("%s %p server connection is allocated", pRpc->label, pConn); + tDebug("%s %p server connection is allocated, uid:0x%x", pRpc->label, pConn, pConn->linkUid); } return pConn; @@ -848,6 +848,16 @@ static int rpcProcessRspHead(SRpcConn *pConn, SRpcHead *pHead) { return TSDB_CODE_RPC_ALREADY_PROCESSED; } + if (pHead->code == TSDB_CODE_RPC_MISMATCHED_LINK_ID) { + tDebug("%s, mismatched linkUid, link shall be restarted", pConn->info); + pConn->secured = 0; + ((SRpcHead *)pConn->pReqMsg)->destId = 0; + rpcSendMsgToPeer(pConn, pConn->pReqMsg, pConn->reqMsgLen); + if (pConn->connType != RPC_CONN_TCPC) + pConn->pTimer = taosTmrStart(rpcProcessRetryTimer, tsRpcTimer, pConn, pRpc->tmrCtrl); + return TSDB_CODE_RPC_ALREADY_PROCESSED; + } + if (pHead->code == TSDB_CODE_RPC_ACTION_IN_PROGRESS) { if (pConn->tretry <= tsRpcMaxRetry) { tDebug("%s, peer is still processing the transaction, retry:%d", pConn->info, pConn->tretry); From 1c800f1d79a26194ad162ff4c14a2b336ee8c0fa Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 25 Jul 2020 02:33:09 +0000 Subject: [PATCH 089/100] update cfg file --- packaging/cfg/taos.cfg | 57 ++++++++++++++++-------------------------- 1 file changed, 22 insertions(+), 35 deletions(-) diff --git a/packaging/cfg/taos.cfg b/packaging/cfg/taos.cfg index f652c65f56..2d4ebb4095 100644 --- a/packaging/cfg/taos.cfg +++ b/packaging/cfg/taos.cfg @@ -15,28 +15,15 @@ # arbitrator arbitrator_hostname:6030 # the full-qualified domain name (FQDN) of dnode -# fqdn hostname:6030 +# fqdn hostname -# port for MNode connect to Client, default udp[6030-6055] tcp[6030] +# port for MNode connect to Client, default udp/tcp [6030-6040] # serverPort 6030 -# http service port, default tcp[6020] +# http service port, default tcp [6020] # httpPort 6020 -# set socket type ("udp" and "tcp") -# the server and client should have the same socket type. Otherwise, connect will fail -# sockettype udp - -# for the cluster version, data file's directory is configured this way -# option mount_path tier_level -# dataDir /mnt/disk1/taos 0 -# dataDir /mnt/disk2/taos 0 -# dataDir /mnt/disk3/taos 0 -# dataDir /mnt/disk4/taos 0 -# dataDir /mnt/disk5/taos 0 -# dataDir /mnt/disk6/taos 1 -# dataDir /mnt/disk7/taos 1 -# for the stand-alone version, data file's directory is configured this way +# data file's directory # dataDir /var/lib/taos # log file's directory @@ -46,12 +33,12 @@ # numOfMnodes 3 # optional roles for dnode. 0 - any, 1 - mnode, 2 - dnode -# alternativeRole 0 +# role 0 # number of threads per CPU core # numOfThreadsPerCore 1.0 -# number of vgroups per db +# max number of vgroups per db # maxVgroupsPerDb 0 # max number of tables per vnode @@ -60,9 +47,6 @@ # the ratio of threads responsible for querying in the total thread # ratioOfQueryThreads 0.5 -# interval of check load balance when the management node is in normal operation -# balanceInterval 300 - # interval of DNode report status to MNode, unit is Second, for cluster version only # statusInterval 1 @@ -82,13 +66,13 @@ # tableMetaKeepTimer 7200 # Minimum sliding window time -# minSlidingTime 10 +# minSlidingTime 10 # Time window minimum -# minIntervalTime 10 +# minIntervalTime 10 # max length of an SQL -# maxSQLLength 65380 +# maxSQLLength 65480 # Support the maximum number of records allowed for super table time sorting # maxNumOfOrderedRes 100000 @@ -153,11 +137,8 @@ # Stop writing data when the disk size of the log folder is less than this value # minimalDataDirGB 0.1 -# mnode take into account while balance, for cluster version only -# mnodeEqualVnodeNum 4 - # number of seconds allowed for a dnode to be offline, for cluster version only -# offlineThreshold 864000 +# offlineThreshold 8640000 # start http service # http 1 @@ -165,7 +146,7 @@ # start system monitor module # monitor 1 -# start http service +# start muqq service # mqtt 0 # mqtt uri @@ -201,19 +182,22 @@ # 131: output warning and error，135: output info, warning and error to log. # 199: output debug, info, warning and error to both screen and file -# debug flag for basic utils -# debugFlag 131 +# debug flag for all log type, take effect when non-zero value +# debugFlag 0 # debug flag for meta management messages # mDebugFlag 135 # debug flag for dnode messages -# dDebugFlag 131 +# dDebugFlag 135 -# debug flag for TDengine SDB +# debug flag for sync module # sDebugFlag 135 -# debug flag for TDengine SDB +# debug flag for WAL +# wDebugFlag 135 + +# debug flag for SDB # sdbDebugFlag 135 # debug flag for RPC @@ -246,6 +230,9 @@ # debug flag for query # qDebugflag 131 +# debug flag for vnode +# vDebugflag 131 + # debug flag for http server # tsdbDebugFlag 131 From b7f6b35e348a8e2d606518770d7f9739714dd53e Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 25 Jul 2020 10:55:24 +0800 Subject: [PATCH 090/100] fix bug caused by refactor --- src/tsdb/src/tsdbRWHelper.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index cc6494fd4b..263e1ea3d5 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -303,6 +303,10 @@ void tsdbSetHelperTable(SRWHelper *pHelper, STable *pTable, STsdbRepo *pRepo) { memset(&(pHelper->curCompIdx), 0, sizeof(SCompIdx)); } + if (helperType(pHelper) == TSDB_WRITE_HELPER && pHelper->curCompIdx.hasLast) { + pHelper->hasOldLastBlock = true; + } + helperSetState(pHelper, TSDB_HELPER_TABLE_SET); ASSERT(pHelper->state == ((TSDB_HELPER_TABLE_SET << 1) - 1)); } From 872edbd07518f5b1fc9e98751ebe0a652532fc2b Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sat, 25 Jul 2020 14:31:31 +0800 Subject: [PATCH 091/100] change more KILL signal to TERM for coverage test. --- tests/pytest/util/dnodes-default.py | 502 +++++++++++++++++++++ tests/pytest/util/dnodes-no-random-fail.py | 8 +- tests/pytest/util/dnodes-random-fail.py | 8 +- tests/script/sh/exec-default.sh | 115 +++++ 4 files changed, 625 insertions(+), 8 deletions(-) create mode 100644 tests/pytest/util/dnodes-default.py create mode 100755 tests/script/sh/exec-default.sh diff --git a/tests/pytest/util/dnodes-default.py b/tests/pytest/util/dnodes-default.py new file mode 100644 index 0000000000..ec3865f4f2 --- /dev/null +++ b/tests/pytest/util/dnodes-default.py @@ -0,0 +1,502 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import os +import os.path +import subprocess +from util.log import * + + +class TDSimClient: + def __init__(self): + self.testCluster = False + + self.cfgDict = { + "numOfLogLines": "100000000", + "numOfThreadsPerCore": "2.0", + "locale": "en_US.UTF-8", + "charset": "UTF-8", + "asyncLog": "0", + "minTablesPerVnode": "4", + "maxTablesPerVnode": "1000", + "tableIncStepPerVnode": "10000", + "maxVgroupsPerDb": "1000", + "sdbDebugFlag": "143", + "rpcDebugFlag": "135", + "tmrDebugFlag": "131", + "cDebugFlag": "135", + "udebugFlag": "135", + "jnidebugFlag": "135", + "qdebugFlag": "135", + } + def init(self, path): + self.__init__() + self.path = path + + def getLogDir(self): + self.logDir = "%s/sim/psim/log" % (self.path) + return self.logDir + + def getCfgDir(self): + self.cfgDir = "%s/sim/psim/cfg" % (self.path) + return self.cfgDir + + def setTestCluster(self, value): + self.testCluster = value + + def addExtraCfg(self, option, value): + self.cfgDict.update({option: value}) + + def cfg(self, option, value): + cmd = "echo '%s %s' >> %s" % (option, value, self.cfgPath) + if os.system(cmd) != 0: + tdLog.exit(cmd) + + def deploy(self): + self.logDir = "%s/sim/psim/log" % (self.path) + self.cfgDir = "%s/sim/psim/cfg" % (self.path) + self.cfgPath = "%s/sim/psim/cfg/taos.cfg" % (self.path) + + cmd = "rm -rf " + self.logDir + if os.system(cmd) != 0: + tdLog.exit(cmd) + + cmd = "mkdir -p " + self.logDir + if os.system(cmd) != 0: + tdLog.exit(cmd) + + cmd = "rm -rf " + self.cfgDir + if os.system(cmd) != 0: + tdLog.exit(cmd) + + cmd = "mkdir -p " + self.cfgDir + if os.system(cmd) != 0: + tdLog.exit(cmd) + + cmd = "touch " + self.cfgPath + if os.system(cmd) != 0: + tdLog.exit(cmd) + + if self.testCluster: + self.cfg("masterIp", "192.168.0.1") + self.cfg("secondIp", "192.168.0.2") + self.cfg("logDir", self.logDir) + + for key, value in self.cfgDict.items(): + self.cfg(key, value) + + tdLog.debug("psim is deployed and configured by %s" % (self.cfgPath)) + + +class TDDnode: + def __init__(self, index): + self.index = index + self.running = 0 + self.deployed = 0 + self.testCluster = False + self.valgrind = 0 + + def init(self, path): + self.path = path + + def setTestCluster(self, value): + self.testCluster = value + + def setValgrind(self, value): + self.valgrind = value + + def getDataSize(self): + totalSize = 0 + + if (self.deployed == 1): + for dirpath, dirnames, filenames in os.walk(self.dataDir): + for f in filenames: + fp = os.path.join(dirpath, f) + + if not os.path.islink(fp): + totalSize = totalSize + os.path.getsize(fp) + + return totalSize + + def deploy(self): + self.logDir = "%s/sim/dnode%d/log" % (self.path, self.index) + self.dataDir = "%s/sim/dnode%d/data" % (self.path, self.index) + self.cfgDir = "%s/sim/dnode%d/cfg" % (self.path, self.index) + self.cfgPath = "%s/sim/dnode%d/cfg/taos.cfg" % ( + self.path, self.index) + + cmd = "rm -rf " + self.dataDir + if os.system(cmd) != 0: + tdLog.exit(cmd) + + cmd = "rm -rf " + self.logDir + if os.system(cmd) != 0: + tdLog.exit(cmd) + + cmd = "rm -rf " + self.cfgDir + if os.system(cmd) != 0: + tdLog.exit(cmd) + + cmd = "mkdir -p " + self.dataDir + if os.system(cmd) != 0: + tdLog.exit(cmd) + + cmd = "mkdir -p " + self.logDir + if os.system(cmd) != 0: + tdLog.exit(cmd) + + cmd = "mkdir -p " + self.cfgDir + if os.system(cmd) != 0: + tdLog.exit(cmd) + + cmd = "touch " + self.cfgPath + if os.system(cmd) != 0: + tdLog.exit(cmd) + + if self.testCluster: + self.startIP() + + if self.testCluster: + self.cfg("masterIp", "192.168.0.1") + self.cfg("secondIp", "192.168.0.2") + self.cfg("publicIp", "192.168.0.%d" % (self.index)) + self.cfg("internalIp", "192.168.0.%d" % (self.index)) + self.cfg("privateIp", "192.168.0.%d" % (self.index)) + self.cfg("dataDir", self.dataDir) + self.cfg("logDir", self.logDir) + self.cfg("numOfLogLines", "100000000") + self.cfg("mnodeEqualVnodeNum", "0") + self.cfg("walLevel", "2") + self.cfg("fsync", "1000") + self.cfg("statusInterval", "1") + self.cfg("numOfMnodes", "3") + self.cfg("numOfThreadsPerCore", "2.0") + self.cfg("monitor", "0") + self.cfg("maxVnodeConnections", "30000") + self.cfg("maxMgmtConnections", "30000") + self.cfg("maxMeterConnections", "30000") + self.cfg("maxShellConns", "30000") + self.cfg("locale", "en_US.UTF-8") + self.cfg("charset", "UTF-8") + self.cfg("asyncLog", "0") + self.cfg("anyIp", "0") + self.cfg("dDebugFlag", "135") + self.cfg("mDebugFlag", "135") + self.cfg("sdbDebugFlag", "135") + self.cfg("rpcDebugFlag", "135") + self.cfg("tmrDebugFlag", "131") + self.cfg("cDebugFlag", "135") + self.cfg("httpDebugFlag", "135") + self.cfg("monitorDebugFlag", "135") + self.cfg("udebugFlag", "135") + self.cfg("jnidebugFlag", "135") + self.cfg("qdebugFlag", "135") + self.deployed = 1 + tdLog.debug( + "dnode:%d is deployed and configured by %s" % + (self.index, self.cfgPath)) + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root)-len("/build/bin")] + break + return buildPath + + def start(self): + buildPath = self.getBuildPath() + + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info("taosd found in %s" % buildPath) + + binPath = buildPath + "/build/bin/taosd" + + if self.deployed == 0: + tdLog.exit("dnode:%d is not deployed" % (self.index)) + + if self.valgrind == 0: + cmd = "nohup %s -c %s > /dev/null 2>&1 & " % ( + binPath, self.cfgDir) + else: + valgrindCmdline = "valgrind --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes" + + cmd = "nohup %s %s -c %s 2>&1 & " % ( + valgrindCmdline, binPath, self.cfgDir) + + print(cmd) + + if os.system(cmd) != 0: + tdLog.exit(cmd) + self.running = 1 + tdLog.debug("dnode:%d is running with %s " % (self.index, cmd)) + + tdLog.debug("wait 5 seconds for the dnode:%d to start." % (self.index)) + time.sleep(5) + + def stop(self): + if self.valgrind == 0: + toBeKilled = "taosd" + else: + toBeKilled = "valgrind.bin" + + if self.running != 0: + psCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print $2}'" % toBeKilled + processID = subprocess.check_output( + psCmd, shell=True).decode("utf-8") + + while(processID): + killCmd = "kill -INT %s > /dev/null 2>&1" % processID + os.system(killCmd) + time.sleep(1) + processID = subprocess.check_output( + psCmd, shell=True).decode("utf-8") + for port in range(6030, 6041): + fuserCmd = "fuser -k -n tcp %d" % port + os.system(fuserCmd) + if self.valgrind: + time.sleep(2) + + self.running = 0 + tdLog.debug("dnode:%d is stopped by kill -INT" % (self.index)) + + def forcestop(self): + if self.valgrind == 0: + toBeKilled = "taosd" + else: + toBeKilled = "valgrind.bin" + + if self.running != 0: + psCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print $2}'" % toBeKilled + processID = subprocess.check_output( + psCmd, shell=True).decode("utf-8") + + while(processID): + killCmd = "kill -KILL %s > /dev/null 2>&1" % processID + os.system(killCmd) + time.sleep(1) + processID = subprocess.check_output( + psCmd, shell=True).decode("utf-8") + for port in range(6030, 6041): + fuserCmd = "fuser -k -n tcp %d" % port + os.system(fuserCmd) + if self.valgrind: + time.sleep(2) + + self.running = 0 + tdLog.debug("dnode:%d is stopped by kill -KILL" % (self.index)) + + def startIP(self): + cmd = "sudo ifconfig lo:%d 192.168.0.%d up" % (self.index, self.index) + if os.system(cmd) != 0: + tdLog.exit(cmd) + + def stopIP(self): + cmd = "sudo ifconfig lo:%d 192.168.0.%d down" % ( + self.index, self.index) + if os.system(cmd) != 0: + tdLog.exit(cmd) + + def cfg(self, option, value): + cmd = "echo '%s %s' >> %s" % (option, value, self.cfgPath) + if os.system(cmd) != 0: + tdLog.exit(cmd) + + def getDnodeRootDir(self, index): + dnodeRootDir = "%s/sim/psim/dnode%d" % (self.path, index) + return dnodeRootDir + + def getDnodesRootDir(self): + dnodesRootDir = "%s/sim/psim" % (self.path) + return dnodesRootDir + + +class TDDnodes: + def __init__(self): + self.dnodes = [] + self.dnodes.append(TDDnode(1)) + self.dnodes.append(TDDnode(2)) + self.dnodes.append(TDDnode(3)) + self.dnodes.append(TDDnode(4)) + self.dnodes.append(TDDnode(5)) + self.dnodes.append(TDDnode(6)) + self.dnodes.append(TDDnode(7)) + self.dnodes.append(TDDnode(8)) + self.dnodes.append(TDDnode(9)) + self.dnodes.append(TDDnode(10)) + self.simDeployed = False + + def init(self, path): + psCmd = "ps -ef|grep -w taosd| grep -v grep | awk '{print $2}'" + processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") + while(processID): + killCmd = "kill -TERM %s > /dev/null 2>&1" % processID + os.system(killCmd) + time.sleep(1) + processID = subprocess.check_output( + psCmd, shell=True).decode("utf-8") + + psCmd = "ps -ef|grep -w valgrind.bin| grep -v grep | awk '{print $2}'" + processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") + while(processID): + killCmd = "kill -TERM %s > /dev/null 2>&1" % processID + os.system(killCmd) + time.sleep(1) + processID = subprocess.check_output( + psCmd, shell=True).decode("utf-8") + + binPath = os.path.dirname(os.path.realpath(__file__)) + binPath = binPath + "/../../../debug/" + tdLog.debug("binPath %s" % (binPath)) + binPath = os.path.realpath(binPath) + tdLog.debug("binPath real path %s" % (binPath)) + + # cmd = "sudo cp %s/build/lib/libtaos.so /usr/local/lib/taos/" % (binPath) + # tdLog.debug(cmd) + # os.system(cmd) + + # cmd = "sudo cp %s/build/bin/taos /usr/local/bin/taos/" % (binPath) + # if os.system(cmd) != 0 : + # tdLog.exit(cmd) + # tdLog.debug("execute %s" % (cmd)) + + # cmd = "sudo cp %s/build/bin/taosd /usr/local/bin/taos/" % (binPath) + # if os.system(cmd) != 0 : + # tdLog.exit(cmd) + # tdLog.debug("execute %s" % (cmd)) + + if path == "": + # self.path = os.path.expanduser('~') + self.path = os.path.abspath(binPath + "../../") + else: + self.path = os.path.realpath(path) + + for i in range(len(self.dnodes)): + self.dnodes[i].init(self.path) + + self.sim = TDSimClient() + self.sim.init(self.path) + + def setTestCluster(self, value): + self.testCluster = value + + def setValgrind(self, value): + self.valgrind = value + + def deploy(self, index): + self.sim.setTestCluster(self.testCluster) + + if (self.simDeployed == False): + self.sim.deploy() + self.simDeployed = True + + self.check(index) + self.dnodes[index - 1].setTestCluster(self.testCluster) + self.dnodes[index - 1].setValgrind(self.valgrind) + self.dnodes[index - 1].deploy() + + def cfg(self, index, option, value): + self.check(index) + self.dnodes[index - 1].cfg(option, value) + + def start(self, index): + self.check(index) + self.dnodes[index - 1].start() + + def stop(self, index): + self.check(index) + self.dnodes[index - 1].stop() + + def getDataSize(self, index): + self.check(index) + return self.dnodes[index - 1].getDataSize() + + def forcestop(self, index): + self.check(index) + self.dnodes[index - 1].forcestop() + + def startIP(self, index): + self.check(index) + + if self.testCluster: + self.dnodes[index - 1].startIP() + + def stopIP(self, index): + self.check(index) + + if self.dnodes[index - 1].testCluster: + self.dnodes[index - 1].stopIP() + + def check(self, index): + if index < 1 or index > 10: + tdLog.exit("index:%d should on a scale of [1, 10]" % (index)) + + def stopAll(self): + tdLog.info("stop all dnodes") + for i in range(len(self.dnodes)): + self.dnodes[i].stop() + + psCmd = "ps -ef | grep -w taosd | grep 'root' | grep -v grep | awk '{print $2}'" + processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") + if processID: + cmd = "sudo systemctl stop taosd" + os.system(cmd) + # if os.system(cmd) != 0 : + # tdLog.exit(cmd) + psCmd = "ps -ef|grep -w taosd| grep -v grep | awk '{print $2}'" + processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") + while(processID): + killCmd = "kill -TERM %s > /dev/null 2>&1" % processID + os.system(killCmd) + time.sleep(1) + processID = subprocess.check_output( + psCmd, shell=True).decode("utf-8") + + psCmd = "ps -ef|grep -w valgrind.bin| grep -v grep | awk '{print $2}'" + processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") + while(processID): + killCmd = "kill -TERM %s > /dev/null 2>&1" % processID + os.system(killCmd) + time.sleep(1) + processID = subprocess.check_output( + psCmd, shell=True).decode("utf-8") + + # if os.system(cmd) != 0 : + # tdLog.exit(cmd) + + def getDnodesRootDir(self): + dnodesRootDir = "%s/sim" % (self.path) + return dnodesRootDir + + def getSimCfgPath(self): + return self.sim.getCfgDir() + + def getSimLogPath(self): + return self.sim.getLogDir() + + def addSimExtraCfg(self, option, value): + self.sim.addExtraCfg(option, value) + + +tdDnodes = TDDnodes() diff --git a/tests/pytest/util/dnodes-no-random-fail.py b/tests/pytest/util/dnodes-no-random-fail.py index 150b8f0c81..485c1b4b3f 100644 --- a/tests/pytest/util/dnodes-no-random-fail.py +++ b/tests/pytest/util/dnodes-no-random-fail.py @@ -349,7 +349,7 @@ class TDDnodes: psCmd = "ps -ef|grep -w taosd| grep -v grep | awk '{print $2}'" processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") while(processID): - killCmd = "kill -KILL %s > /dev/null 2>&1" % processID + killCmd = "kill -TERM %s > /dev/null 2>&1" % processID os.system(killCmd) time.sleep(1) processID = subprocess.check_output( @@ -358,7 +358,7 @@ class TDDnodes: psCmd = "ps -ef|grep -w valgrind.bin| grep -v grep | awk '{print $2}'" processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") while(processID): - killCmd = "kill -KILL %s > /dev/null 2>&1" % processID + killCmd = "kill -TERM %s > /dev/null 2>&1" % processID os.system(killCmd) time.sleep(1) processID = subprocess.check_output( @@ -465,7 +465,7 @@ class TDDnodes: psCmd = "ps -ef|grep -w taosd| grep -v grep | awk '{print $2}'" processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") while(processID): - killCmd = "kill -KILL %s > /dev/null 2>&1" % processID + killCmd = "kill -TERM %s > /dev/null 2>&1" % processID os.system(killCmd) time.sleep(1) processID = subprocess.check_output( @@ -474,7 +474,7 @@ class TDDnodes: psCmd = "ps -ef|grep -w valgrind.bin| grep -v grep | awk '{print $2}'" processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") while(processID): - killCmd = "kill -KILL %s > /dev/null 2>&1" % processID + killCmd = "kill -TERM %s > /dev/null 2>&1" % processID os.system(killCmd) time.sleep(1) processID = subprocess.check_output( diff --git a/tests/pytest/util/dnodes-random-fail.py b/tests/pytest/util/dnodes-random-fail.py index 36c0f9d796..7c4edde47f 100644 --- a/tests/pytest/util/dnodes-random-fail.py +++ b/tests/pytest/util/dnodes-random-fail.py @@ -349,7 +349,7 @@ class TDDnodes: psCmd = "ps -ef|grep -w taosd| grep -v grep | awk '{print $2}'" processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") while(processID): - killCmd = "kill -KILL %s > /dev/null 2>&1" % processID + killCmd = "kill -TERM %s > /dev/null 2>&1" % processID os.system(killCmd) time.sleep(1) processID = subprocess.check_output( @@ -358,7 +358,7 @@ class TDDnodes: psCmd = "ps -ef|grep -w valgrind.bin| grep -v grep | awk '{print $2}'" processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") while(processID): - killCmd = "kill -KILL %s > /dev/null 2>&1" % processID + killCmd = "kill -TERM %s > /dev/null 2>&1" % processID os.system(killCmd) time.sleep(1) processID = subprocess.check_output( @@ -465,7 +465,7 @@ class TDDnodes: psCmd = "ps -ef|grep -w taosd| grep -v grep | awk '{print $2}'" processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") while(processID): - killCmd = "kill -KILL %s > /dev/null 2>&1" % processID + killCmd = "kill -TERM %s > /dev/null 2>&1" % processID os.system(killCmd) time.sleep(1) processID = subprocess.check_output( @@ -474,7 +474,7 @@ class TDDnodes: psCmd = "ps -ef|grep -w valgrind.bin| grep -v grep | awk '{print $2}'" processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") while(processID): - killCmd = "kill -KILL %s > /dev/null 2>&1" % processID + killCmd = "kill -TERM %s > /dev/null 2>&1" % processID os.system(killCmd) time.sleep(1) processID = subprocess.check_output( diff --git a/tests/script/sh/exec-default.sh b/tests/script/sh/exec-default.sh new file mode 100755 index 0000000000..1c84a6b10e --- /dev/null +++ b/tests/script/sh/exec-default.sh @@ -0,0 +1,115 @@ +#!/bin/bash + +# if [ $# != 4 || $# != 5 ]; then + # echo "argument list need input : " + # echo " -n nodeName" + # echo " -s start/stop" + # echo " -c clear" + # exit 1 +# fi + +NODE_NAME= +EXEC_OPTON= +CLEAR_OPTION="false" +while getopts "n:s:u:x:ct" arg +do + case $arg in + n) + NODE_NAME=$OPTARG + ;; + s) + EXEC_OPTON=$OPTARG + ;; + c) + CLEAR_OPTION="clear" + ;; + t) + SHELL_OPTION="true" + ;; + u) + USERS=$OPTARG + ;; + x) + SIGNAL=$OPTARG + ;; + ?) + echo "unkown argument" + ;; + esac +done + +SCRIPT_DIR=`dirname $0` +cd $SCRIPT_DIR/../ +SCRIPT_DIR=`pwd` + +IN_TDINTERNAL="community" +if [[ "$SCRIPT_DIR" == *"$IN_TDINTERNAL"* ]]; then + cd ../../.. +else + cd ../../ +fi + +TAOS_DIR=`pwd` +TAOSD_DIR=`find . -name "taosd"|grep bin|head -n1` + +if [[ "$TAOSD_DIR" == *"$IN_TDINTERNAL"* ]]; then + BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' --fields=2,3` +else + BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' --fields=2` +fi + +BUILD_DIR=$TAOS_DIR/$BIN_DIR/build + +SIM_DIR=$TAOS_DIR/sim +NODE_DIR=$SIM_DIR/$NODE_NAME +EXE_DIR=$BUILD_DIR/bin +CFG_DIR=$NODE_DIR/cfg +LOG_DIR=$NODE_DIR/log +DATA_DIR=$NODE_DIR/data +MGMT_DIR=$NODE_DIR/data/mgmt +TSDB_DIR=$NODE_DIR/data/tsdb + +TAOS_CFG=$NODE_DIR/cfg/taos.cfg + +echo ------------ $EXEC_OPTON $NODE_NAME + +TAOS_FLAG=$SIM_DIR/tsim/flag +if [ -f "$TAOS_FLAG" ]; then + EXE_DIR=/usr/local/bin/taos +fi + +if [ "$CLEAR_OPTION" = "clear" ]; then + echo rm -rf $MGMT_DIR $TSDB_DIR + rm -rf $TSDB_DIR + rm -rf $MGMT_DIR +fi + +if [ "$EXEC_OPTON" = "start" ]; then + echo "ExcuteCmd:" $EXE_DIR/taosd -c $CFG_DIR + + if [ "$SHELL_OPTION" = "true" ]; then + TT=`date +%s` + mkdir ${LOG_DIR}/${TT} + nohup valgrind --log-file=${LOG_DIR}/${TT}/valgrind.log --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes $EXE_DIR/taosd -c $CFG_DIR > /dev/null 2>&1 & + else + nohup $EXE_DIR/taosd -c $CFG_DIR > /dev/null 2>&1 & + fi + +else + #relative path + RCFG_DIR=sim/$NODE_NAME/cfg + PID=`ps -ef|grep taosd | grep $RCFG_DIR | grep -v grep | awk '{print $2}'` + while [ -n "$PID" ] + do + if [ "$SIGNAL" = "SIGKILL" ]; then + echo try to kill by signal SIGKILL + kill -9 $PID + else + echo try to kill by signal SIGINT + kill -SIGINT $PID + fi + sleep 1 + PID=`ps -ef|grep taosd | grep $RCFG_DIR | grep -v grep | awk '{print $2}'` + done +fi + From cc363470bfb09d4d63d68180bec37896ef6e601d Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 25 Jul 2020 15:18:28 +0800 Subject: [PATCH 092/100] TD-974 --- src/tsdb/src/tsdbRWHelper.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index 263e1ea3d5..9ceaa08625 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -1260,13 +1260,18 @@ static int tsdbLoadBlockDataColsImpl(SRWHelper *pHelper, SCompBlock *pCompBlock, SCompCol *pCompCol = NULL; while (true) { - ASSERT(dcol < pDataCols->numOfCols); + // ASSERT(dcol < pDataCols->numOfCols); + if (dcol >= pDataCols->numOfCols) { + pDataCol = NULL; + break; + } pDataCol = &pDataCols->cols[dcol]; ASSERT(pDataCol->colId <= colId); if (pDataCol->colId == colId) break; dcol++; } + if (pDataCol == NULL) continue; ASSERT(pDataCol->colId == colId); if (colId == 0) { // load the key row From a5bf51cc05ad0d618576e941cb9421e4f853f899 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sun, 26 Jul 2020 03:59:40 +0000 Subject: [PATCH 093/100] fixbug TD-995 --- src/query/inc/qExecutor.h | 2 +- src/query/src/qResultbuf.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h index 127c38a6f8..42dfce42a6 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qExecutor.h @@ -42,7 +42,7 @@ typedef struct SSqlGroupbyExpr { } SSqlGroupbyExpr; typedef struct SPosInfo { - int16_t pageId; + int32_t pageId; int16_t rowId; } SPosInfo; diff --git a/src/query/src/qResultbuf.c b/src/query/src/qResultbuf.c index de59676e59..a3d3386a87 100644 --- a/src/query/src/qResultbuf.c +++ b/src/query/src/qResultbuf.c @@ -52,7 +52,7 @@ int32_t getResBufSize(SDiskbasedResultBuf* pResultBuf) { return pResultBuf->tota #define FILE_SIZE_ON_DISK(_r) (NUM_OF_PAGES_ON_DISK(_r) * (_r)->pageSize) static int32_t createDiskResidesBuf(SDiskbasedResultBuf* pResultBuf) { - pResultBuf->fd = open(pResultBuf->path, O_CREAT | O_RDWR, 0666); + pResultBuf->fd = open(pResultBuf->path, O_CREAT | O_RDWR | O_TRUNC, 0666); if (!FD_VALID(pResultBuf->fd)) { qError("failed to create tmp file: %s on disk. %s", pResultBuf->path, strerror(errno)); return TAOS_SYSTEM_ERROR(errno); From 73dc644b45d5c5684c49bebf9c901eb7dda851ff Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 26 Jul 2020 14:33:46 +0800 Subject: [PATCH 094/100] refactor load column data interface --- src/tsdb/src/tsdbRWHelper.c | 34 ++++++++++++++++++++++------------ 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index 9ceaa08625..28f2974573 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -1260,15 +1260,18 @@ static int tsdbLoadBlockDataColsImpl(SRWHelper *pHelper, SCompBlock *pCompBlock, SCompCol *pCompCol = NULL; while (true) { - // ASSERT(dcol < pDataCols->numOfCols); if (dcol >= pDataCols->numOfCols) { pDataCol = NULL; break; } pDataCol = &pDataCols->cols[dcol]; - ASSERT(pDataCol->colId <= colId); - if (pDataCol->colId == colId) break; - dcol++; + if (pDataCol->colId > colId) { + pDataCol = NULL; + break; + } else { + dcol++; + if (pDataCol->colId == colId) break; + } } if (pDataCol == NULL) continue; @@ -1281,15 +1284,24 @@ static int tsdbLoadBlockDataColsImpl(SRWHelper *pHelper, SCompBlock *pCompBlock, compCol.offset = TSDB_KEY_COL_OFFSET; pCompCol = &compCol; } else { // load non-key rows - while (ccol < pCompBlock->numOfCols) { - pCompCol = &pHelper->pCompData->cols[ccol]; - if (pCompCol->colId >= colId) break; - ccol++; + while (true) { + if (ccol >= pCompBlock->numOfCols) { + pCompCol = NULL; + break; + } + + pCompCol = &(pHelper->pCompData->cols[ccol]); + if (pCompCol->colId > colId) { + pCompCol = NULL; + break; + } else { + ccol++; + if (pCompCol->colId == colId) break; + } } - if (ccol >= pCompBlock->numOfCols || pCompCol->colId > colId) { + if (pCompCol == NULL) { dataColSetNEleNull(pDataCol, pCompBlock->numOfRows, pDataCols->maxPoints); - dcol++; continue; } @@ -1297,8 +1309,6 @@ static int tsdbLoadBlockDataColsImpl(SRWHelper *pHelper, SCompBlock *pCompBlock, } if (tsdbLoadColData(pHelper, pFile, pCompBlock, pCompCol, pDataCol) < 0) goto _err; - dcol++; - if (colId != 0) ccol++; } return 0; From 90e23fb8c0f10c3f8fa3ce3940af701c49903c46 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 26 Jul 2020 16:33:18 +0800 Subject: [PATCH 095/100] [TD-998] --- src/inc/taosdef.h | 1 + src/inc/taosmsg.h | 1 + src/mnode/inc/mnodeCluster.h | 38 ++++++ src/mnode/inc/mnodeDef.h | 8 ++ src/mnode/inc/mnodeSdb.h | 19 +-- src/mnode/src/mnodeAcct.c | 4 +- src/mnode/src/mnodeCluster.c | 229 +++++++++++++++++++++++++++++++++++ src/mnode/src/mnodeMain.c | 2 + src/mnode/src/mnodeSdb.c | 2 + src/mnode/src/mnodeShow.c | 2 + 10 files changed, 295 insertions(+), 11 deletions(-) create mode 100644 src/mnode/inc/mnodeCluster.h create mode 100644 src/mnode/src/mnodeCluster.c diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index 30523dc25c..1bb185e448 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -377,6 +377,7 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size); #define TSDB_ORDER_ASC 1 #define TSDB_ORDER_DESC 2 +#define TSDB_DEFAULT_CLUSTER_HASH_SIZE 1 #define TSDB_DEFAULT_MNODES_HASH_SIZE 5 #define TSDB_DEFAULT_DNODES_HASH_SIZE 10 #define TSDB_DEFAULT_ACCOUNTS_HASH_SIZE 10 diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index e095567d75..de7c6c8268 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -139,6 +139,7 @@ enum _mgmt_table { TSDB_MGMT_TABLE_GRANTS, TSDB_MGMT_TABLE_VNODES, TSDB_MGMT_TABLE_STREAMTABLES, + TSDB_MGMT_TABLE_CLUSTER, TSDB_MGMT_TABLE_MAX, }; diff --git a/src/mnode/inc/mnodeCluster.h b/src/mnode/inc/mnodeCluster.h new file mode 100644 index 0000000000..3be5d601af --- /dev/null +++ b/src/mnode/inc/mnodeCluster.h @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_MNODE_CLUSTER_H +#define TDENGINE_MNODE_CLUSTER_H + +#ifdef __cplusplus +extern "C" { +#endif + +struct SClusterObj; + +int32_t mnodeInitCluster(); +void mnodeCleanupCluster(); +int32_t mnodeGetClusterId(); +void mnodeUpdateClusterId(); +void * mnodeGetCluster(int32_t clusterId); +void * mnodeGetNextCluster(void *pIter, struct SClusterObj **pCluster); +void mnodeIncClusterRef(struct SClusterObj *pCluster); +void mnodeDecClusterRef(struct SClusterObj *pCluster); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/mnode/inc/mnodeDef.h b/src/mnode/inc/mnodeDef.h index a454f413f0..62028bc820 100644 --- a/src/mnode/inc/mnodeDef.h +++ b/src/mnode/inc/mnodeDef.h @@ -36,6 +36,14 @@ struct define notes: 3. The fields behind the updataEnd field can be changed; */ +typedef struct SClusterObj { + int32_t clusterId; + int64_t createdTime; + int8_t reserved[36]; + int8_t updateEnd[4]; + int32_t refCount; +} SClusterObj; + typedef struct SDnodeObj { int32_t dnodeId; int32_t openVnodes; diff --git a/src/mnode/inc/mnodeSdb.h b/src/mnode/inc/mnodeSdb.h index 0c47f684f8..cadd1b1f3d 100644 --- a/src/mnode/inc/mnodeSdb.h +++ b/src/mnode/inc/mnodeSdb.h @@ -23,15 +23,16 @@ extern "C" { struct SMnodeMsg; typedef enum { - SDB_TABLE_DNODE = 0, - SDB_TABLE_MNODE = 1, - SDB_TABLE_ACCOUNT = 2, - SDB_TABLE_USER = 3, - SDB_TABLE_DB = 4, - SDB_TABLE_VGROUP = 5, - SDB_TABLE_STABLE = 6, - SDB_TABLE_CTABLE = 7, - SDB_TABLE_MAX = 8 + SDB_TABLE_CLUSTER = 0, + SDB_TABLE_DNODE = 1, + SDB_TABLE_MNODE = 2, + SDB_TABLE_ACCOUNT = 3, + SDB_TABLE_USER = 4, + SDB_TABLE_DB = 5, + SDB_TABLE_VGROUP = 6, + SDB_TABLE_STABLE = 7, + SDB_TABLE_CTABLE = 8, + SDB_TABLE_MAX = 9 } ESdbTable; typedef enum { diff --git a/src/mnode/src/mnodeAcct.c b/src/mnode/src/mnodeAcct.c index 2d6e1ae007..287e8bb723 100644 --- a/src/mnode/src/mnodeAcct.c +++ b/src/mnode/src/mnodeAcct.c @@ -64,7 +64,7 @@ static int32_t mnodeAcctActionUpdate(SSdbOper *pOper) { return TSDB_CODE_SUCCESS; } -static int32_t mnodeActionActionEncode(SSdbOper *pOper) { +static int32_t mnodeAcctActionEncode(SSdbOper *pOper) { SAcctObj *pAcct = pOper->pObj; memcpy(pOper->rowData, pAcct, tsAcctUpdateSize); pOper->rowSize = tsAcctUpdateSize; @@ -109,7 +109,7 @@ int32_t mnodeInitAccts() { .insertFp = mnodeAcctActionInsert, .deleteFp = mnodeAcctActionDelete, .updateFp = mnodeAcctActionUpdate, - .encodeFp = mnodeActionActionEncode, + .encodeFp = mnodeAcctActionEncode, .decodeFp = mnodeAcctActionDecode, .destroyFp = mnodeAcctActionDestroy, .restoredFp = mnodeAcctActionRestored diff --git a/src/mnode/src/mnodeCluster.c b/src/mnode/src/mnodeCluster.c new file mode 100644 index 0000000000..e83f31cc44 --- /dev/null +++ b/src/mnode/src/mnodeCluster.c @@ -0,0 +1,229 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include "taoserror.h" +#include "ttime.h" +#include "dnode.h" +#include "mnodeDef.h" +#include "mnodeInt.h" +#include "mnodeCluster.h" +#include "mnodeSdb.h" +#include "mnodeShow.h" +#include "tglobal.h" + +static void * tsClusterSdb = NULL; +static int32_t tsClusterUpdateSize; +static int32_t tsClusterId; +static int32_t mnodeCreateCluster(); + +static int32_t mnodeGetClusterMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); +static int32_t mnodeRetrieveClusters(SShowObj *pShow, char *data, int32_t rows, void *pConn); + +static int32_t mnodeClusterActionDestroy(SSdbOper *pOper) { + tfree(pOper->pObj); + return TSDB_CODE_SUCCESS; +} + +static int32_t mnodeClusterActionInsert(SSdbOper *pOper) { + return TSDB_CODE_SUCCESS; +} + +static int32_t mnodeClusterActionDelete(SSdbOper *pOper) { + return TSDB_CODE_SUCCESS; +} + +static int32_t mnodeClusterActionUpdate(SSdbOper *pOper) { + return TSDB_CODE_SUCCESS; +} + +static int32_t mnodeClusterActionEncode(SSdbOper *pOper) { + SClusterObj *pCluster = pOper->pObj; + memcpy(pOper->rowData, pCluster, tsClusterUpdateSize); + pOper->rowSize = tsClusterUpdateSize; + return TSDB_CODE_SUCCESS; +} + +static int32_t mnodeClusterActionDecode(SSdbOper *pOper) { + SClusterObj *pCluster = (SClusterObj *) calloc(1, sizeof(SClusterObj)); + if (pCluster == NULL) return TSDB_CODE_MND_OUT_OF_MEMORY; + + memcpy(pCluster, pOper->rowData, tsClusterUpdateSize); + pOper->pObj = pCluster; + return TSDB_CODE_SUCCESS; +} + +static int32_t mnodeClusterActionRestored() { + int32_t numOfRows = sdbGetNumOfRows(tsClusterSdb); + if (numOfRows <= 0 && dnodeIsFirstDeploy()) { + mInfo("dnode first deploy, create cluster"); + int32_t code = mnodeCreateCluster(); + if (code != TSDB_CODE_SUCCESS) { + mError("failed to create cluster, reason:%s", tstrerror(code)); + return code; + } + } + + return TSDB_CODE_SUCCESS; +} + +int32_t mnodeInitCluster() { + SClusterObj tObj; + tsClusterUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj; + + SSdbTableDesc tableDesc = { + .tableId = SDB_TABLE_CLUSTER, + .tableName = "cluster", + .hashSessions = TSDB_DEFAULT_CLUSTER_HASH_SIZE, + .maxRowSize = tsClusterUpdateSize, + .refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj, + .keyType = SDB_KEY_INT, + .insertFp = mnodeClusterActionInsert, + .deleteFp = mnodeClusterActionDelete, + .updateFp = mnodeClusterActionUpdate, + .encodeFp = mnodeClusterActionEncode, + .decodeFp = mnodeClusterActionDecode, + .destroyFp = mnodeClusterActionDestroy, + .restoredFp = mnodeClusterActionRestored + }; + + tsClusterSdb = sdbOpenTable(&tableDesc); + if (tsClusterSdb == NULL) { + mError("table:%s, failed to create hash", tableDesc.tableName); + return -1; + } + + mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_CLUSTER, mnodeGetClusterMeta); + mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_CLUSTER, mnodeRetrieveClusters); + + mDebug("table:%s, hash is created", tableDesc.tableName); + return TSDB_CODE_SUCCESS; +} + +void mnodeCleanupCluster() { + sdbCloseTable(tsClusterSdb); + tsClusterSdb = NULL; +} + +void *mnodeGetCluster(int32_t clusterId) { + return sdbGetRow(tsClusterSdb, &clusterId); +} + +void *mnodeGetNextCluster(void *pIter, SClusterObj **pCluster) { + return sdbFetchRow(tsClusterSdb, pIter, (void **)pCluster); +} + +void mnodeIncClusterRef(SClusterObj *pCluster) { + sdbIncRef(tsClusterSdb, pCluster); +} + +void mnodeDecClusterRef(SClusterObj *pCluster) { + sdbDecRef(tsClusterSdb, pCluster); +} + +static int32_t mnodeCreateCluster() { + int32_t numOfClusters = sdbGetNumOfRows(tsClusterSdb); + if (numOfClusters != 0) return TSDB_CODE_SUCCESS; + + SClusterObj *pCluster = malloc(sizeof(SClusterObj)); + memset(pCluster, 0, sizeof(SClusterObj)); + pCluster->createdTime = taosGetTimestampMs(); + pCluster->clusterId = (pCluster->createdTime >> 32) & (pCluster->createdTime) & (*(int32_t*)tsFirst); + + SSdbOper oper = { + .type = SDB_OPER_GLOBAL, + .table = tsClusterSdb, + .pObj = pCluster, + }; + + return sdbInsertRow(&oper); +} + +int32_t mnodeGetClusterId() { + return tsClusterId; +} + +void mnodeUpdateClusterId() { + SClusterObj *pCluster = NULL; + mnodeGetNextCluster(NULL, &pCluster); + if (pCluster != NULL) { + tsClusterId = pCluster->clusterId; + mnodeDecClusterRef(pCluster); + mInfo("cluster id is %d", tsClusterId); + } else { + //assert(false); + } +} + + +static int32_t mnodeGetClusterMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { + int32_t cols = 0; + SSchema *pSchema = pMeta->schema; + + pShow->bytes[cols] = 4; + pSchema[cols].type = TSDB_DATA_TYPE_INT; + strcpy(pSchema[cols].name, "clusterId"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 8; + pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; + strcpy(pSchema[cols].name, "create_time"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pMeta->numOfColumns = htons(cols); + strcpy(pMeta->tableId, "show cluster"); + pShow->numOfColumns = cols; + + pShow->offset[0] = 0; + for (int32_t i = 1; i < cols; ++i) { + pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1]; + } + + pShow->numOfRows = 1; + pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; + + return 0; +} + +static int32_t mnodeRetrieveClusters(SShowObj *pShow, char *data, int32_t rows, void *pConn) { + int32_t numOfRows = 0; + int32_t cols = 0; + char * pWrite; + SClusterObj *pCluster = NULL; + + while (numOfRows < rows) { + pShow->pIter = mnodeGetNextCluster(pShow->pIter, &pCluster); + if (pCluster == NULL) break; + + cols = 0; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int32_t *) pWrite = pCluster->clusterId; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int32_t *) pWrite = pCluster->createdTime; + cols++; + + mnodeDecClusterRef(pCluster); + numOfRows++; + } + + pShow->numOfReads += numOfRows; + return numOfRows; +} diff --git a/src/mnode/src/mnodeMain.c b/src/mnode/src/mnodeMain.c index 042e356442..53f3474c3c 100644 --- a/src/mnode/src/mnodeMain.c +++ b/src/mnode/src/mnodeMain.c @@ -32,6 +32,7 @@ #include "mnodeVgroup.h" #include "mnodeUser.h" #include "mnodeTable.h" +#include "mnodeCluster.h" #include "mnodeShow.h" #include "mnodeProfile.h" @@ -46,6 +47,7 @@ static bool tsMgmtIsRunning = false; static const SMnodeComponent tsMnodeComponents[] = { {"profile", mnodeInitProfile, mnodeCleanupProfile}, + {"cluster", mnodeInitCluster, mnodeCleanupCluster}, {"accts", mnodeInitAccts, mnodeCleanupAccts}, {"users", mnodeInitUsers, mnodeCleanupUsers}, {"dnodes", mnodeInitDnodes, mnodeCleanupDnodes}, diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index 4b2945152b..26efcfeac0 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -29,6 +29,7 @@ #include "mnodeInt.h" #include "mnodeMnode.h" #include "mnodeDnode.h" +#include "mnodeCluster.h" #include "mnodeSdb.h" #define SDB_TABLE_LEN 12 @@ -214,6 +215,7 @@ void sdbUpdateMnodeRoles() { } } + mnodeUpdateClusterId(); mnodeUpdateMnodeEpSet(); } diff --git a/src/mnode/src/mnodeShow.c b/src/mnode/src/mnodeShow.c index 1d85a8cacd..d7f058b3a8 100644 --- a/src/mnode/src/mnodeShow.c +++ b/src/mnode/src/mnodeShow.c @@ -103,6 +103,8 @@ static char *mnodeGetShowType(int32_t showType) { case TSDB_MGMT_TABLE_SCORES: return "show scores"; case TSDB_MGMT_TABLE_GRANTS: return "show grants"; case TSDB_MGMT_TABLE_VNODES: return "show vnodes"; + case TSDB_MGMT_TABLE_CLUSTER: return "show clusters"; + case TSDB_MGMT_TABLE_STREAMTABLES : return "show streamtables"; default: return "undefined"; } } From 3c9fcc587a7d0d4e84aa9af9be4ad47e10df92b0 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 26 Jul 2020 17:03:26 +0800 Subject: [PATCH 096/100] [TD-998] --- src/dnode/src/dnodeMgmt.c | 13 ++++++++++++- src/inc/taoserror.h | 1 + src/inc/taosmsg.h | 2 ++ src/mnode/src/mnodeCluster.c | 3 ++- src/mnode/src/mnodeDnode.c | 17 +++++++++++++---- 5 files changed, 30 insertions(+), 6 deletions(-) diff --git a/src/dnode/src/dnodeMgmt.c b/src/dnode/src/dnodeMgmt.c index 9050b9c582..e2b2c52ce4 100644 --- a/src/dnode/src/dnodeMgmt.c +++ b/src/dnode/src/dnodeMgmt.c @@ -492,6 +492,7 @@ static void dnodeProcessStatusRsp(SRpcMsg *pMsg) { pCfg->numOfVnodes = htonl(pCfg->numOfVnodes); pCfg->moduleStatus = htonl(pCfg->moduleStatus); pCfg->dnodeId = htonl(pCfg->dnodeId); + pCfg->clusterId = htonl(pCfg->clusterId); for (int32_t i = 0; i < pMnodes->nodeNum; ++i) { SDMMnodeInfo *pMnodeInfo = &pMnodes->nodeInfos[i]; @@ -697,6 +698,7 @@ static void dnodeSendStatusMsg(void *handle, void *tmrId) { //strcpy(pStatus->dnodeName, tsDnodeName); pStatus->version = htonl(tsVersion); pStatus->dnodeId = htonl(tsDnodeCfg.dnodeId); + pStatus->clusterId = htonl(tsDnodeCfg.clusterId); strcpy(pStatus->dnodeEp, tsLocalEp); pStatus->lastReboot = htonl(tsRebootTime); pStatus->numOfCores = htons((uint16_t) tsNumOfCores); @@ -767,6 +769,13 @@ static bool dnodeReadDnodeCfg() { } tsDnodeCfg.dnodeId = dnodeId->valueint; + cJSON* clusterId = cJSON_GetObjectItem(root, "clusterId"); + if (!clusterId || clusterId->type != cJSON_Number) { + dError("failed to read dnodeCfg.json, clusterId not found"); + goto PARSE_CFG_OVER; + } + tsDnodeCfg.clusterId = clusterId->valueint; + ret = true; dInfo("read numOfVnodes successed, dnodeId:%d", tsDnodeCfg.dnodeId); @@ -791,6 +800,7 @@ static void dnodeSaveDnodeCfg() { len += snprintf(content + len, maxLen - len, "{\n"); len += snprintf(content + len, maxLen - len, " \"dnodeId\": %d\n", tsDnodeCfg.dnodeId); + len += snprintf(content + len, maxLen - len, " \"clusterId\": %d\n", tsDnodeCfg.clusterId); len += snprintf(content + len, maxLen - len, "}\n"); fwrite(content, 1, len, fp); @@ -803,8 +813,9 @@ static void dnodeSaveDnodeCfg() { void dnodeUpdateDnodeCfg(SDMDnodeCfg *pCfg) { if (tsDnodeCfg.dnodeId == 0) { - dInfo("dnodeId is set to %d", pCfg->dnodeId); + dInfo("dnodeId is set to %d, clusterId is set to %d", pCfg->dnodeId, pCfg->clusterId); tsDnodeCfg.dnodeId = pCfg->dnodeId; + tsDnodeCfg.clusterId = pCfg->clusterId; dnodeSaveDnodeCfg(); } } diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index c440fd3c06..1f03f1fe09 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -131,6 +131,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_BALANCE_ENABLED, 0, 0x0337, "balance al TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_NOT_IN_DNODE, 0, 0x0338, "vgroup not in dnode") TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_ALREADY_IN_DNODE, 0, 0x0339, "vgroup already in dnode") TAOS_DEFINE_ERROR(TSDB_CODE_MND_DNODE_NOT_FREE, 0, 0x033A, "dnode not avaliable") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_CLUSTER_ID, 0, 0x033B, "cluster id not match") TAOS_DEFINE_ERROR(TSDB_CODE_MND_ACCT_ALREADY_EXIST, 0, 0x0340, "mnode accounts already exist") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_ACCT, 0, 0x0341, "mnode invalid account") diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index de7c6c8268..ad2be97946 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -547,6 +547,7 @@ typedef struct { typedef struct { int32_t dnodeId; + int32_t clusterId; uint32_t moduleStatus; uint32_t numOfVnodes; uint32_t reserved; @@ -587,6 +588,7 @@ typedef struct { uint16_t openVnodes; uint16_t numOfCores; float diskAvailable; // GB + int32_t clusterId; uint8_t alternativeRole; uint8_t reserve2[15]; SClusterCfg clusterCfg; diff --git a/src/mnode/src/mnodeCluster.c b/src/mnode/src/mnodeCluster.c index e83f31cc44..2c78a32d40 100644 --- a/src/mnode/src/mnodeCluster.c +++ b/src/mnode/src/mnodeCluster.c @@ -77,6 +77,7 @@ static int32_t mnodeClusterActionRestored() { } } + mnodeUpdateClusterId(); return TSDB_CODE_SUCCESS; } @@ -141,7 +142,7 @@ static int32_t mnodeCreateCluster() { SClusterObj *pCluster = malloc(sizeof(SClusterObj)); memset(pCluster, 0, sizeof(SClusterObj)); pCluster->createdTime = taosGetTimestampMs(); - pCluster->clusterId = (pCluster->createdTime >> 32) & (pCluster->createdTime) & (*(int32_t*)tsFirst); + pCluster->clusterId = abs(((pCluster->createdTime >> 32) & (pCluster->createdTime)) | (*(int32_t*)tsFirst)); SSdbOper oper = { .type = SDB_OPER_GLOBAL, diff --git a/src/mnode/src/mnodeDnode.c b/src/mnode/src/mnodeDnode.c index 687bb1873b..4f9421efde 100644 --- a/src/mnode/src/mnodeDnode.c +++ b/src/mnode/src/mnodeDnode.c @@ -37,6 +37,7 @@ #include "mnodeVgroup.h" #include "mnodeWrite.h" #include "mnodePeer.h" +#include "mnodeCluster.h" int32_t tsAccessSquence = 0; static void *tsDnodeSdb = NULL; @@ -355,6 +356,7 @@ static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg) { pStatus->moduleStatus = htonl(pStatus->moduleStatus); pStatus->lastReboot = htonl(pStatus->lastReboot); pStatus->numOfCores = htons(pStatus->numOfCores); + pStatus->clusterId = htonl(pStatus->clusterId); uint32_t version = htonl(pStatus->version); if (version != tsVersion) { @@ -382,13 +384,19 @@ static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg) { pDnode->diskAvailable = pStatus->diskAvailable; pDnode->alternativeRole = pStatus->alternativeRole; pDnode->moduleStatus = pStatus->moduleStatus; - + if (pStatus->dnodeId == 0) { - mDebug("dnode:%d %s, first access", pDnode->dnodeId, pDnode->dnodeEp); + mDebug("dnode:%d %s, first access, set clusterId %d", pDnode->dnodeId, pDnode->dnodeEp, mnodeGetClusterId()); } else { - mTrace("dnode:%d, status received, access times %d", pDnode->dnodeId, pDnode->lastAccess); + if (pStatus->clusterId != mnodeGetClusterId()) { + mError("dnode:%d, input clusterId %d not match with exist %d", pDnode->dnodeId, pStatus->clusterId, + mnodeGetClusterId()); + return TSDB_CODE_MND_INVALID_CLUSTER_ID; + } else { + mTrace("dnode:%d, status received, access times %d", pDnode->dnodeId, pDnode->lastAccess); + } } - + int32_t openVnodes = htons(pStatus->openVnodes); int32_t contLen = sizeof(SDMStatusRsp) + openVnodes * sizeof(SDMVgroupAccess); SDMStatusRsp *pRsp = rpcMallocCont(contLen); @@ -400,6 +408,7 @@ static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg) { pRsp->dnodeCfg.dnodeId = htonl(pDnode->dnodeId); pRsp->dnodeCfg.moduleStatus = htonl((int32_t)pDnode->isMgmt); pRsp->dnodeCfg.numOfVnodes = htonl(openVnodes); + pRsp->dnodeCfg.clusterId = htonl(mnodeGetClusterId()); SDMVgroupAccess *pAccess = (SDMVgroupAccess *)((char *)pRsp + sizeof(SDMStatusRsp)); for (int32_t j = 0; j < openVnodes; ++j) { From 102d1c1c4bce9aabfea551edfd91225abe85fc90 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 26 Jul 2020 17:03:51 +0800 Subject: [PATCH 097/100] [TD-998] --- src/mnode/inc/mnodeSdb.h | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/mnode/inc/mnodeSdb.h b/src/mnode/inc/mnodeSdb.h index cadd1b1f3d..0d1f8e54df 100644 --- a/src/mnode/inc/mnodeSdb.h +++ b/src/mnode/inc/mnodeSdb.h @@ -23,15 +23,15 @@ extern "C" { struct SMnodeMsg; typedef enum { - SDB_TABLE_CLUSTER = 0, - SDB_TABLE_DNODE = 1, - SDB_TABLE_MNODE = 2, - SDB_TABLE_ACCOUNT = 3, - SDB_TABLE_USER = 4, - SDB_TABLE_DB = 5, - SDB_TABLE_VGROUP = 6, - SDB_TABLE_STABLE = 7, - SDB_TABLE_CTABLE = 8, + SDB_TABLE_DNODE = 0, + SDB_TABLE_MNODE = 1, + SDB_TABLE_ACCOUNT = 2, + SDB_TABLE_USER = 3, + SDB_TABLE_DB = 4, + SDB_TABLE_VGROUP = 5, + SDB_TABLE_STABLE = 6, + SDB_TABLE_CTABLE = 7, + SDB_TABLE_CLUSTER = 8, SDB_TABLE_MAX = 9 } ESdbTable; From 1bc3bf8f94886bc224170ca08fe96c441df986b0 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 26 Jul 2020 17:11:33 +0800 Subject: [PATCH 098/100] [TD-998] --- src/mnode/inc/mnodeSdb.h | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/mnode/inc/mnodeSdb.h b/src/mnode/inc/mnodeSdb.h index 0d1f8e54df..cadd1b1f3d 100644 --- a/src/mnode/inc/mnodeSdb.h +++ b/src/mnode/inc/mnodeSdb.h @@ -23,15 +23,15 @@ extern "C" { struct SMnodeMsg; typedef enum { - SDB_TABLE_DNODE = 0, - SDB_TABLE_MNODE = 1, - SDB_TABLE_ACCOUNT = 2, - SDB_TABLE_USER = 3, - SDB_TABLE_DB = 4, - SDB_TABLE_VGROUP = 5, - SDB_TABLE_STABLE = 6, - SDB_TABLE_CTABLE = 7, - SDB_TABLE_CLUSTER = 8, + SDB_TABLE_CLUSTER = 0, + SDB_TABLE_DNODE = 1, + SDB_TABLE_MNODE = 2, + SDB_TABLE_ACCOUNT = 3, + SDB_TABLE_USER = 4, + SDB_TABLE_DB = 5, + SDB_TABLE_VGROUP = 6, + SDB_TABLE_STABLE = 7, + SDB_TABLE_CTABLE = 8, SDB_TABLE_MAX = 9 } ESdbTable; From 6ef759ceadb6721b7e866bca8f76a1334d6553ad Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 26 Jul 2020 19:24:19 +0800 Subject: [PATCH 099/100] compile error for clang C --- src/mnode/src/mnodeCluster.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mnode/src/mnodeCluster.c b/src/mnode/src/mnodeCluster.c index 2c78a32d40..8933107dd1 100644 --- a/src/mnode/src/mnodeCluster.c +++ b/src/mnode/src/mnodeCluster.c @@ -142,7 +142,7 @@ static int32_t mnodeCreateCluster() { SClusterObj *pCluster = malloc(sizeof(SClusterObj)); memset(pCluster, 0, sizeof(SClusterObj)); pCluster->createdTime = taosGetTimestampMs(); - pCluster->clusterId = abs(((pCluster->createdTime >> 32) & (pCluster->createdTime)) | (*(int32_t*)tsFirst)); + pCluster->clusterId = labs((pCluster->createdTime >> 32) & (pCluster->createdTime)) | (*(int32_t*)tsFirst); SSdbOper oper = { .type = SDB_OPER_GLOBAL, From 06fbef7afb2bfce8b3ffc5e6fb05324be99fd3e9 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 26 Jul 2020 20:12:24 +0800 Subject: [PATCH 100/100] [TD-998] invalid json format --- src/dnode/src/dnodeMgmt.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dnode/src/dnodeMgmt.c b/src/dnode/src/dnodeMgmt.c index e2b2c52ce4..ec35475d73 100644 --- a/src/dnode/src/dnodeMgmt.c +++ b/src/dnode/src/dnodeMgmt.c @@ -799,7 +799,7 @@ static void dnodeSaveDnodeCfg() { char * content = calloc(1, maxLen + 1); len += snprintf(content + len, maxLen - len, "{\n"); - len += snprintf(content + len, maxLen - len, " \"dnodeId\": %d\n", tsDnodeCfg.dnodeId); + len += snprintf(content + len, maxLen - len, " \"dnodeId\": %d,\n", tsDnodeCfg.dnodeId); len += snprintf(content + len, maxLen - len, " \"clusterId\": %d\n", tsDnodeCfg.clusterId); len += snprintf(content + len, maxLen - len, "}\n");