From c877418df00f36c1621a6ccc2e2adf65e0457611 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 17 Jul 2020 00:31:23 +0000 Subject: [PATCH 01/34] support multi thread update mnode and dnode --- src/client/inc/tsclient.h | 2 +- src/client/src/tscServer.c | 78 ++++++++++++++++++++++++++------------ src/client/src/tscSql.c | 2 +- src/client/src/tscSystem.c | 3 +- src/client/src/tscUtil.c | 17 +++++---- src/inc/trpc.h | 5 +++ 6 files changed, 72 insertions(+), 35 deletions(-) diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index be82eb64a8..e1180c4143 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -457,7 +457,7 @@ extern void * tscQhandle; extern int tscKeepConn[]; extern int tsInsertHeadSize; extern int tscNumOfThreads; -extern SRpcIpSet tscMgmtIpSet; +extern SRpcCorIpSet tscMgmtIpSet; extern int (*tscBuildMsg[TSDB_SQL_MAX])(SSqlObj *pSql, SSqlInfo *pInfo); diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index d8af6d5c87..e8a03a6482 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -29,7 +29,7 @@ #define TSC_MGMT_VNODE 999 -SRpcIpSet tscMgmtIpSet; +SRpcCorIpSet tscMgmtIpSet; SRpcIpSet tscDnodeIpSet; int (*tscBuildMsg[TSDB_SQL_MAX])(SSqlObj *pSql, SSqlInfo *pInfo) = {0}; @@ -44,6 +44,42 @@ void tscSaveSubscriptionProgress(void* sub); static int32_t minMsgSize() { return tsRpcHeadSize + 100; } +static void tscDumpMgmtIpSet(SRpcIpSet *ipSet) { + taosCorBeginRead(&tscMgmtIpSet.version); + *ipSet = tscMgmtIpSet.ipSet; + taosCorEndRead(&tscMgmtIpSet.version); +} + +bool tscIpSetIsEqual(SRpcIpSet *s1, SRpcIpSet *s2) { + if (s1->numOfIps != s2->numOfIps + || s1->inUse != s1->inUse) { + return false; + } + for (int32_t i = 0; i < s1->numOfIps; i++) { + if (s1->port[i] != s2->port[i] + || strncmp(s1->fqdn[i], s2->fqdn[i], TSDB_FQDN_LEN) != 0) + return false; + } + return true; +} +void tscSetMgmtIpList(SRpcIpSet *pIpSet) { + // no need to update if equal + SRpcIpSet dump; + tscDumpMgmtIpSet(&dump); + if (tscIpSetIsEqual(&dump, pIpSet)) { + return; + } + + taosCorBeginWrite(&tscMgmtIpSet.version); + SRpcIpSet *mgmtIpSet = &tscMgmtIpSet.ipSet; + mgmtIpSet->numOfIps = pIpSet->numOfIps; + mgmtIpSet->inUse = pIpSet->inUse; + for (int32_t i = 0; i < mgmtIpSet->numOfIps; ++i) { + mgmtIpSet->port[i] = htons(pIpSet->port[i]); + strncpy(mgmtIpSet->fqdn[i], pIpSet->fqdn[i], TSDB_FQDN_LEN); + } + taosCorEndWrite(&tscMgmtIpSet.version); +} static void tscSetDnodeIpList(SSqlObj* pSql, SCMVgroupInfo* pVgroupInfo) { SRpcIpSet* pIpList = &pSql->ipList; pIpList->inUse = 0; @@ -60,31 +96,17 @@ static void tscSetDnodeIpList(SSqlObj* pSql, SCMVgroupInfo* pVgroupInfo) { } void tscPrintMgmtIp() { - if (tscMgmtIpSet.numOfIps <= 0) { - tscError("invalid mnode IP list:%d", tscMgmtIpSet.numOfIps); + SRpcIpSet dump; + tscDumpMgmtIpSet(&dump); + if (dump.numOfIps <= 0) { + tscError("invalid mnode IP list:%d", dump.numOfIps); } else { - for (int i = 0; i < tscMgmtIpSet.numOfIps; ++i) { - tscDebug("mnode index:%d %s:%d", i, tscMgmtIpSet.fqdn[i], tscMgmtIpSet.port[i]); + for (int i = 0; i < dump.numOfIps; ++i) { + tscDebug("mnode index:%d %s:%d", i, dump.fqdn[i], dump.port[i]); } } } -void tscSetMgmtIpList(SRpcIpSet *pIpList) { - tscMgmtIpSet.numOfIps = pIpList->numOfIps; - tscMgmtIpSet.inUse = pIpList->inUse; - for (int32_t i = 0; i < tscMgmtIpSet.numOfIps; ++i) { - tscMgmtIpSet.port[i] = htons(pIpList->port[i]); - } -} - -void tscUpdateIpSet(void *ahandle, SRpcIpSet *pIpSet) { - tscMgmtIpSet = *pIpSet; - tscDebug("mnode IP list is changed for ufp is called, numOfIps:%d inUse:%d", tscMgmtIpSet.numOfIps, tscMgmtIpSet.inUse); - for (int32_t i = 0; i < tscMgmtIpSet.numOfIps; ++i) { - tscDebug("index:%d fqdn:%s port:%d", i, tscMgmtIpSet.fqdn[i], tscMgmtIpSet.port[i]); - } -} - /* * For each management node, try twice at least in case of poor network situation. * If the client start to connect to a non-management node from the client, and the first retry may fail due to @@ -95,7 +117,9 @@ void tscUpdateIpSet(void *ahandle, SRpcIpSet *pIpSet) { UNUSED_FUNC static int32_t tscGetMgmtConnMaxRetryTimes() { int32_t factor = 2; - return tscMgmtIpSet.numOfIps * factor; + SRpcIpSet dump; + tscDumpMgmtIpSet(&dump); + return dump.numOfIps * factor; } void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) { @@ -185,7 +209,9 @@ int tscSendMsgToServer(SSqlObj *pSql) { // set the mgmt ip list if (pSql->cmd.command >= TSDB_SQL_MGMT) { - pSql->ipList = tscMgmtIpSet; + SRpcIpSet dump; + tscDumpMgmtIpSet(&dump); + pSql->ipList = dump; } memcpy(pMsg, pSql->cmd.payload, pSql->cmd.payloadLen); @@ -239,7 +265,7 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcIpSet *pIpSet) { if (pCmd->command < TSDB_SQL_MGMT) { if (pIpSet) pSql->ipList = *pIpSet; } else { - if (pIpSet) tscMgmtIpSet = *pIpSet; + if (pIpSet) tscSetMgmtIpList(pIpSet); } if (rpcMsg->pCont == NULL) { @@ -421,7 +447,9 @@ int tscProcessSql(SSqlObj *pSql) { return pSql->res.code; } } else if (pCmd->command < TSDB_SQL_LOCAL) { - pSql->ipList = tscMgmtIpSet; + SRpcIpSet dump; + tscDumpMgmtIpSet(&dump); + pSql->ipList = dump; } else { // local handler return (*tscProcessMsgRsp[pCmd->command])(pSql); } diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index 0677463d8d..b24cfd9b27 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -63,7 +63,7 @@ SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pass, con if (ip) { if (tscSetMgmtIpListFromCfg(ip, NULL) < 0) return NULL; - if (port) tscMgmtIpSet.port[0] = port; + if (port) tscMgmtIpSet.ipSet.port[0] = port; } void *pDnodeConn = NULL; diff --git a/src/client/src/tscSystem.c b/src/client/src/tscSystem.c index 82cc8cc225..1a78ed87bd 100644 --- a/src/client/src/tscSystem.c +++ b/src/client/src/tscSystem.c @@ -41,7 +41,8 @@ int tscNumOfThreads; static pthread_once_t tscinit = PTHREAD_ONCE_INIT; void taosInitNote(int numOfNoteLines, int maxNotes, char* lable); -void tscUpdateIpSet(void *ahandle, SRpcIpSet *pIpSet); +//void tscUpdateIpSet(void *ahandle, SRpcIpSet *pIpSet); + void tscCheckDiskUsage(void *UNUSED_PARAM(para), void* UNUSED_PARAM(param)) { taosGetDisk(); diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 3e0fe0b4be..b0e6c727eb 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -2146,16 +2146,19 @@ char* strdup_throw(const char* str) { } int tscSetMgmtIpListFromCfg(const char *first, const char *second) { - tscMgmtIpSet.numOfIps = 0; - tscMgmtIpSet.inUse = 0; + // init mgmt ip set + tscMgmtIpSet.version = 0; + SRpcIpSet *mgmtIpSet = &(tscMgmtIpSet.ipSet); + mgmtIpSet->numOfIps = 0; + mgmtIpSet->inUse = 0; if (first && first[0] != 0) { if (strlen(first) >= TSDB_EP_LEN) { terrno = TSDB_CODE_TSC_INVALID_FQDN; return -1; } - taosGetFqdnPortFromEp(first, tscMgmtIpSet.fqdn[tscMgmtIpSet.numOfIps], &tscMgmtIpSet.port[tscMgmtIpSet.numOfIps]); - tscMgmtIpSet.numOfIps++; + taosGetFqdnPortFromEp(first, mgmtIpSet->fqdn[mgmtIpSet->numOfIps], &(mgmtIpSet->port[mgmtIpSet->numOfIps])); + mgmtIpSet->numOfIps++; } if (second && second[0] != 0) { @@ -2163,11 +2166,11 @@ int tscSetMgmtIpListFromCfg(const char *first, const char *second) { terrno = TSDB_CODE_TSC_INVALID_FQDN; return -1; } - taosGetFqdnPortFromEp(second, tscMgmtIpSet.fqdn[tscMgmtIpSet.numOfIps], &tscMgmtIpSet.port[tscMgmtIpSet.numOfIps]); - tscMgmtIpSet.numOfIps++; + taosGetFqdnPortFromEp(second, mgmtIpSet->fqdn[mgmtIpSet->numOfIps], &(mgmtIpSet->port[mgmtIpSet->numOfIps])); + mgmtIpSet->numOfIps++; } - if ( tscMgmtIpSet.numOfIps == 0) { + if (mgmtIpSet->numOfIps == 0) { terrno = TSDB_CODE_TSC_INVALID_FQDN; return -1; } diff --git a/src/inc/trpc.h b/src/inc/trpc.h index d1adfb7494..b159155e9d 100644 --- a/src/inc/trpc.h +++ b/src/inc/trpc.h @@ -35,6 +35,11 @@ typedef struct SRpcIpSet { char fqdn[TSDB_MAX_REPLICA][TSDB_FQDN_LEN]; } SRpcIpSet; +typedef struct SRpcCorIpSet { + int32_t version; + SRpcIpSet ipSet; +} SRpcCorIpSet; + typedef struct SRpcConnInfo { uint32_t clientIp; uint16_t clientPort; From 3d10b6f81d7d0f301bc3dd42405bf4f3869ab723 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 17 Jul 2020 04:25:37 +0000 Subject: [PATCH 02/34] support multi thread update dnode --- src/client/src/tscSchemaUtil.c | 3 ++ src/client/src/tscServer.c | 67 +++++++++++++++++++++------------- src/inc/taosmsg.h | 1 + 3 files changed, 45 insertions(+), 26 deletions(-) diff --git a/src/client/src/tscSchemaUtil.c b/src/client/src/tscSchemaUtil.c index 934a562387..b391a5760e 100644 --- a/src/client/src/tscSchemaUtil.c +++ b/src/client/src/tscSchemaUtil.c @@ -157,6 +157,9 @@ STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg, size_t* size pTableMeta->sid = pTableMetaMsg->sid; pTableMeta->uid = pTableMetaMsg->uid; pTableMeta->vgroupInfo = pTableMetaMsg->vgroup; + //init version here + pTableMeta->vgroupInfo.version = 0; + pTableMeta->sversion = pTableMetaMsg->sversion; pTableMeta->tversion = pTableMetaMsg->tversion; diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index e8a03a6482..4a11288a85 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -51,8 +51,7 @@ static void tscDumpMgmtIpSet(SRpcIpSet *ipSet) { } bool tscIpSetIsEqual(SRpcIpSet *s1, SRpcIpSet *s2) { - if (s1->numOfIps != s2->numOfIps - || s1->inUse != s1->inUse) { + if (s1->numOfIps != s2->numOfIps /*|| s1->inUse != s1->inUse*/) { return false; } for (int32_t i = 0; i < s1->numOfIps; i++) { @@ -80,21 +79,38 @@ void tscSetMgmtIpList(SRpcIpSet *pIpSet) { } taosCorEndWrite(&tscMgmtIpSet.version); } -static void tscSetDnodeIpList(SSqlObj* pSql, SCMVgroupInfo* pVgroupInfo) { - SRpcIpSet* pIpList = &pSql->ipList; - pIpList->inUse = 0; - if (pVgroupInfo == NULL) { - pIpList->numOfIps = 0; +static void tscDumpIpSetFromVgroupInfo(SCMVgroupInfo *pVgroupInfo, SRpcIpSet *pIpSet, int32_t *vgId) { + if (pVgroupInfo == NULL) { return;} + taosCorBeginRead(&pVgroupInfo->version); + if (vgId) { + *vgId = pVgroupInfo->vgId; + } + pIpSet->inUse = 0; + pIpSet->numOfIps = pVgroupInfo->numOfIps; + for (int32_t i = 0; i < pVgroupInfo->numOfIps; ++i) { + strncpy(pIpSet->fqdn[i], pVgroupInfo->ipAddr[i].fqdn, TSDB_FQDN_LEN); + pIpSet->port[i] = pVgroupInfo->ipAddr[i].port; + } + taosCorEndRead(&pVgroupInfo->version); +} +static void tscSetVgroupInfoWithIpSet(SCMVgroupInfo *pVgroupInfo, SRpcIpSet *pIpSet) { + taosCorBeginWrite(&pVgroupInfo->version); + //TODO(dengyihao), dont care vgid + pVgroupInfo->numOfIps = pIpSet->numOfIps; + for (int32_t i = 0; pVgroupInfo->numOfIps; i++) { + strncpy(pVgroupInfo->ipAddr[i].fqdn, pIpSet->fqdn[i], TSDB_FQDN_LEN); + pVgroupInfo->ipAddr[i].port = pIpSet->port[i]; + } + taosCorEndWrite(&pVgroupInfo->version); +} +static void tscSetVgroupInfo(SSqlObj *pObj, SRpcIpSet *pIpSet) { + if (tscIpSetIsEqual(&pObj->ipList, pIpSet)) { return; } - - pIpList->numOfIps = pVgroupInfo->numOfIps; - for(int32_t i = 0; i < pVgroupInfo->numOfIps; ++i) { - strcpy(pIpList->fqdn[i], pVgroupInfo->ipAddr[i].fqdn); - pIpList->port[i] = pVgroupInfo->ipAddr[i].port; - } + SSqlCmd *pCmd = &pObj->cmd; + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); + tscSetVgroupInfoWithIpSet(&pTableMetaInfo->pTableMeta->vgroupInfo, pIpSet); } - void tscPrintMgmtIp() { SRpcIpSet dump; tscDumpMgmtIpSet(&dump); @@ -263,7 +279,7 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcIpSet *pIpSet) { } if (pCmd->command < TSDB_SQL_MGMT) { - if (pIpSet) pSql->ipList = *pIpSet; + if (pIpSet) tscSetVgroupInfo(pSql, pIpSet); } else { if (pIpSet) tscSetMgmtIpList(pIpSet); } @@ -553,7 +569,7 @@ int tscBuildSubmitMsg(SSqlObj *pSql, SSqlInfo *pInfo) { // pSql->cmd.payloadLen is set during copying data into payload pSql->cmd.msgType = TSDB_MSG_TYPE_SUBMIT; - tscSetDnodeIpList(pSql, &pTableMeta->vgroupInfo); + tscDumpIpSetFromVgroupInfo(&pTableMeta->vgroupInfo, &pSql->ipList, NULL); tscDebug("%p build submit msg, vgId:%d numOfTables:%d numberOfIP:%d", pSql, vgId, pSql->cmd.numOfTablesInSubmit, pSql->ipList.numOfIps); @@ -595,11 +611,9 @@ static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char } else { pVgroupInfo = &pTableMeta->vgroupInfo; } - - tscSetDnodeIpList(pSql, pVgroupInfo); - if (pVgroupInfo != NULL) { - pQueryMsg->head.vgId = htonl(pVgroupInfo->vgId); - } + int32_t vgId = 0; + tscDumpIpSetFromVgroupInfo(pVgroupInfo, &pSql->ipList, &vgId); + pQueryMsg->head.vgId = htonl(vgId); STableIdInfo *pTableIdInfo = (STableIdInfo *)pMsg; pTableIdInfo->tid = htonl(pTableMeta->sid); @@ -618,8 +632,9 @@ static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char SVgroupTableInfo* pTableIdList = taosArrayGet(pTableMetaInfo->pVgroupTables, index); // set the vgroup info - tscSetDnodeIpList(pSql, &pTableIdList->vgInfo); - pQueryMsg->head.vgId = htonl(pTableIdList->vgInfo.vgId); + int32_t vgId = 0; + tscDumpIpSetFromVgroupInfo(&pTableIdList->vgInfo, &pSql->ipList, &vgId); + pQueryMsg->head.vgId = htonl(vgId); int32_t numOfTables = taosArrayGetSize(pTableIdList->itemList); pQueryMsg->numOfTables = htonl(numOfTables); // set the number of tables @@ -1351,7 +1366,7 @@ int tscBuildUpdateTagMsg(SSqlObj* pSql, SSqlInfo *pInfo) { SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - tscSetDnodeIpList(pSql, &pTableMetaInfo->pTableMeta->vgroupInfo); + tscDumpIpSetFromVgroupInfo(&pTableMetaInfo->pTableMeta->vgroupInfo, &pSql->ipList, NULL); return TSDB_CODE_SUCCESS; } @@ -1875,11 +1890,11 @@ int tscProcessSTableVgroupRsp(SSqlObj *pSql) { memcpy(pInfo->vgroupList, pVgroupInfo, size); for (int32_t j = 0; j < pInfo->vgroupList->numOfVgroups; ++j) { + //just init, no need to lock SCMVgroupInfo *pVgroups = &pInfo->vgroupList->vgroups[j]; - + pVgroups->version = 0; pVgroups->vgId = htonl(pVgroups->vgId); assert(pVgroups->numOfIps >= 1); - for (int32_t k = 0; k < pVgroups->numOfIps; ++k) { pVgroups->ipAddr[k].port = htons(pVgroups->ipAddr[k].port); } diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index b7afaf1e06..c12aed2bcc 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -647,6 +647,7 @@ typedef struct SCMSTableVgroupMsg { } SCMSTableVgroupMsg, SCMSTableVgroupRspMsg; typedef struct { + int32_t version; int32_t vgId; int8_t numOfIps; SIpAddr ipAddr[TSDB_MAX_REPLICA]; From dd45e955dc5ed8c2a9862abbcda319ced4531f68 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 17 Jul 2020 06:53:56 +0000 Subject: [PATCH 03/34] fix bug --- src/client/src/tscServer.c | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 4a11288a85..045aab61d6 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -44,9 +44,18 @@ void tscSaveSubscriptionProgress(void* sub); static int32_t minMsgSize() { return tsRpcHeadSize + 100; } +void tscIpSetCopy(SRpcIpSet *dst, SRpcIpSet *src) { + dst->numOfIps = src->numOfIps; + dst->inUse = src->inUse; + for (int32_t i = 0; i < src->numOfIps; ++i) { + dst->port[i] = htons(dst->port[i]); + strncpy(dst->fqdn[i], src->fqdn[i], TSDB_FQDN_LEN); + } +} static void tscDumpMgmtIpSet(SRpcIpSet *ipSet) { taosCorBeginRead(&tscMgmtIpSet.version); - *ipSet = tscMgmtIpSet.ipSet; + SRpcIpSet* src = &tscMgmtIpSet.ipSet; + tscIpSetCopy(ipSet, src); taosCorEndRead(&tscMgmtIpSet.version); } @@ -71,12 +80,7 @@ void tscSetMgmtIpList(SRpcIpSet *pIpSet) { taosCorBeginWrite(&tscMgmtIpSet.version); SRpcIpSet *mgmtIpSet = &tscMgmtIpSet.ipSet; - mgmtIpSet->numOfIps = pIpSet->numOfIps; - mgmtIpSet->inUse = pIpSet->inUse; - for (int32_t i = 0; i < mgmtIpSet->numOfIps; ++i) { - mgmtIpSet->port[i] = htons(pIpSet->port[i]); - strncpy(mgmtIpSet->fqdn[i], pIpSet->fqdn[i], TSDB_FQDN_LEN); - } + tscIpSetCopy(mgmtIpSet, pIpSet); taosCorEndWrite(&tscMgmtIpSet.version); } static void tscDumpIpSetFromVgroupInfo(SCMVgroupInfo *pVgroupInfo, SRpcIpSet *pIpSet, int32_t *vgId) { @@ -227,7 +231,7 @@ int tscSendMsgToServer(SSqlObj *pSql) { if (pSql->cmd.command >= TSDB_SQL_MGMT) { SRpcIpSet dump; tscDumpMgmtIpSet(&dump); - pSql->ipList = dump; + tscIpSetCopy(&pSql->ipList, &dump); } memcpy(pMsg, pSql->cmd.payload, pSql->cmd.payloadLen); From 6163e2401f5d18822c206475543a14019991c500 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 17 Jul 2020 07:42:04 +0000 Subject: [PATCH 04/34] refor code --- src/client/src/tscServer.c | 41 ++++++++++++++++++-------------------- 1 file changed, 19 insertions(+), 22 deletions(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 045aab61d6..c82773a125 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -70,15 +70,10 @@ bool tscIpSetIsEqual(SRpcIpSet *s1, SRpcIpSet *s2) { } return true; } -void tscSetMgmtIpList(SRpcIpSet *pIpSet) { +void tscUpdateMgmtIpList(SRpcIpSet *pIpSet) { // no need to update if equal - SRpcIpSet dump; - tscDumpMgmtIpSet(&dump); - if (tscIpSetIsEqual(&dump, pIpSet)) { - return; - } - taosCorBeginWrite(&tscMgmtIpSet.version); + // or copy directly, tscMgmtIpSet.ipSet = *pIpSet SRpcIpSet *mgmtIpSet = &tscMgmtIpSet.ipSet; tscIpSetCopy(mgmtIpSet, pIpSet); taosCorEndWrite(&tscMgmtIpSet.version); @@ -97,7 +92,12 @@ static void tscDumpIpSetFromVgroupInfo(SCMVgroupInfo *pVgroupInfo, SRpcIpSet *pI } taosCorEndRead(&pVgroupInfo->version); } -static void tscSetVgroupInfoWithIpSet(SCMVgroupInfo *pVgroupInfo, SRpcIpSet *pIpSet) { + +static void tscUpdateVgroupInfo(SSqlObj *pObj, SRpcIpSet *pIpSet) { + SSqlCmd *pCmd = &pObj->cmd; + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); + SCMVgroupInfo *pVgroupInfo = &pTableMetaInfo->pTableMeta->vgroupInfo; + taosCorBeginWrite(&pVgroupInfo->version); //TODO(dengyihao), dont care vgid pVgroupInfo->numOfIps = pIpSet->numOfIps; @@ -107,14 +107,6 @@ static void tscSetVgroupInfoWithIpSet(SCMVgroupInfo *pVgroupInfo, SRpcIpSet *pIp } taosCorEndWrite(&pVgroupInfo->version); } -static void tscSetVgroupInfo(SSqlObj *pObj, SRpcIpSet *pIpSet) { - if (tscIpSetIsEqual(&pObj->ipList, pIpSet)) { - return; - } - SSqlCmd *pCmd = &pObj->cmd; - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); - tscSetVgroupInfoWithIpSet(&pTableMetaInfo->pTableMeta->vgroupInfo, pIpSet); -} void tscPrintMgmtIp() { SRpcIpSet dump; tscDumpMgmtIpSet(&dump); @@ -157,7 +149,7 @@ void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) { SCMHeartBeatRsp *pRsp = (SCMHeartBeatRsp *)pRes->pRsp; SRpcIpSet * pIpList = &pRsp->ipList; if (pIpList->numOfIps > 0) - tscSetMgmtIpList(pIpList); + tscUpdateMgmtIpList(pIpList); pSql->pTscObj->connId = htonl(pRsp->connId); @@ -231,6 +223,7 @@ int tscSendMsgToServer(SSqlObj *pSql) { if (pSql->cmd.command >= TSDB_SQL_MGMT) { SRpcIpSet dump; tscDumpMgmtIpSet(&dump); + // no need to update pSql->ipList tscIpSetCopy(&pSql->ipList, &dump); } @@ -282,10 +275,14 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcIpSet *pIpSet) { return; } - if (pCmd->command < TSDB_SQL_MGMT) { - if (pIpSet) tscSetVgroupInfo(pSql, pIpSet); - } else { - if (pIpSet) tscSetMgmtIpList(pIpSet); + if (pIpSet) { + if (!tscIpSetIsEqual(&pSql->ipList, pIpSet)) { + if (pCmd->command < TSDB_SQL_MGMT) { + tscUpdateVgroupInfo(pSql, pIpSet); + } else { + tscUpdateMgmtIpList(pIpSet); + } + } } if (rpcMsg->pCont == NULL) { @@ -1994,7 +1991,7 @@ int tscProcessConnectRsp(SSqlObj *pSql) { tstrncpy(pObj->db, temp, sizeof(pObj->db)); if (pConnect->ipList.numOfIps > 0) - tscSetMgmtIpList(&pConnect->ipList); + tscUpdateMgmtIpList(&pConnect->ipList); strcpy(pObj->sversion, pConnect->serverVersion); pObj->writeAuth = pConnect->writeAuth; From 5f55393464f57939a17043d3a4a23d3f81ad8353 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 17 Jul 2020 21:54:16 +0000 Subject: [PATCH 05/34] fixbug --- src/client/src/tscServer.c | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index c82773a125..ced6914df2 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -48,7 +48,7 @@ void tscIpSetCopy(SRpcIpSet *dst, SRpcIpSet *src) { dst->numOfIps = src->numOfIps; dst->inUse = src->inUse; for (int32_t i = 0; i < src->numOfIps; ++i) { - dst->port[i] = htons(dst->port[i]); + dst->port[i] = src->port[i]; strncpy(dst->fqdn[i], src->fqdn[i], TSDB_FQDN_LEN); } } @@ -58,7 +58,11 @@ static void tscDumpMgmtIpSet(SRpcIpSet *ipSet) { tscIpSetCopy(ipSet, src); taosCorEndRead(&tscMgmtIpSet.version); } - +static void tscIpSetHtons(SRpcIpSet *s) { + for (int32_t i = 0; i < s->numOfIps; i++) { + s->port[i] = htons(s->port[i]); + } +} bool tscIpSetIsEqual(SRpcIpSet *s1, SRpcIpSet *s2) { if (s1->numOfIps != s2->numOfIps /*|| s1->inUse != s1->inUse*/) { return false; @@ -149,6 +153,7 @@ void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) { SCMHeartBeatRsp *pRsp = (SCMHeartBeatRsp *)pRes->pRsp; SRpcIpSet * pIpList = &pRsp->ipList; if (pIpList->numOfIps > 0) + tscIpSetHtons(pIpList); tscUpdateMgmtIpList(pIpList); pSql->pTscObj->connId = htonl(pRsp->connId); @@ -275,13 +280,15 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcIpSet *pIpSet) { return; } - if (pIpSet) { - if (!tscIpSetIsEqual(&pSql->ipList, pIpSet)) { - if (pCmd->command < TSDB_SQL_MGMT) { + if (pIpSet) { + //SRpcIpSet dump; + tscIpSetHtons(pIpSet); + if (tscIpSetIsEqual(&pSql->ipList, pIpSet)) { + if(pCmd->command < TSDB_SQL_MGMT) { tscUpdateVgroupInfo(pSql, pIpSet); } else { tscUpdateMgmtIpList(pIpSet); - } + } } } @@ -464,10 +471,8 @@ int tscProcessSql(SSqlObj *pSql) { return pSql->res.code; } } else if (pCmd->command < TSDB_SQL_LOCAL) { - SRpcIpSet dump; - tscDumpMgmtIpSet(&dump); - pSql->ipList = dump; - } else { // local handler + tscDumpMgmtIpSet(&pSql->ipList); + } else { return (*tscProcessMsgRsp[pCmd->command])(pSql); } @@ -1991,6 +1996,7 @@ int tscProcessConnectRsp(SSqlObj *pSql) { tstrncpy(pObj->db, temp, sizeof(pObj->db)); if (pConnect->ipList.numOfIps > 0) + tscIpSetHtons(&pConnect->ipList); tscUpdateMgmtIpList(&pConnect->ipList); strcpy(pObj->sversion, pConnect->serverVersion); From d2823d60e2bc30dffa04970bf970de53761f076a Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 17 Jul 2020 22:17:23 +0000 Subject: [PATCH 06/34] modify code style --- src/client/src/tscServer.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index ced6914df2..b162387685 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -226,10 +226,7 @@ int tscSendMsgToServer(SSqlObj *pSql) { // set the mgmt ip list if (pSql->cmd.command >= TSDB_SQL_MGMT) { - SRpcIpSet dump; - tscDumpMgmtIpSet(&dump); - // no need to update pSql->ipList - tscIpSetCopy(&pSql->ipList, &dump); + tscDumpMgmtIpSet(&pSql->ipList); } memcpy(pMsg, pSql->cmd.payload, pSql->cmd.payloadLen); @@ -471,7 +468,7 @@ int tscProcessSql(SSqlObj *pSql) { return pSql->res.code; } } else if (pCmd->command < TSDB_SQL_LOCAL) { - tscDumpMgmtIpSet(&pSql->ipList); + //tscDumpMgmtIpSet(&pSql->ipList); } else { return (*tscProcessMsgRsp[pCmd->command])(pSql); } From 65b32c3855ecfd7caf2aef977ada081e5e3b1b66 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 17 Jul 2020 22:35:57 +0000 Subject: [PATCH 07/34] modify code style --- src/client/src/tscServer.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index b162387685..3b3884eb07 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -100,6 +100,7 @@ static void tscDumpIpSetFromVgroupInfo(SCMVgroupInfo *pVgroupInfo, SRpcIpSet *pI static void tscUpdateVgroupInfo(SSqlObj *pObj, SRpcIpSet *pIpSet) { SSqlCmd *pCmd = &pObj->cmd; STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); + if (pTableMetaInfo == NULL || pTableMetaInfo->pTableMeta == NULL) { return;} SCMVgroupInfo *pVgroupInfo = &pTableMetaInfo->pTableMeta->vgroupInfo; taosCorBeginWrite(&pVgroupInfo->version); From 0ac592fbfd44686a875629a93760d6e48e24b8bf Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sat, 18 Jul 2020 01:04:02 +0000 Subject: [PATCH 08/34] add inuse to vgroupinfo --- src/client/src/tscServer.c | 9 ++++++--- src/inc/taosmsg.h | 1 + 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 3b3884eb07..d0c9d3636b 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -64,7 +64,7 @@ static void tscIpSetHtons(SRpcIpSet *s) { } } bool tscIpSetIsEqual(SRpcIpSet *s1, SRpcIpSet *s2) { - if (s1->numOfIps != s2->numOfIps /*|| s1->inUse != s1->inUse*/) { + if (s1->numOfIps != s2->numOfIps || s1->inUse != s2->inUse) { return false; } for (int32_t i = 0; i < s1->numOfIps; i++) { @@ -88,7 +88,8 @@ static void tscDumpIpSetFromVgroupInfo(SCMVgroupInfo *pVgroupInfo, SRpcIpSet *pI if (vgId) { *vgId = pVgroupInfo->vgId; } - pIpSet->inUse = 0; + int8_t inUse = pVgroupInfo->inUse; + pIpSet->inUse = (inUse >= 0 && inUse < TSDB_MAX_REPLICA) ? inUse: 0; pIpSet->numOfIps = pVgroupInfo->numOfIps; for (int32_t i = 0; i < pVgroupInfo->numOfIps; ++i) { strncpy(pIpSet->fqdn[i], pVgroupInfo->ipAddr[i].fqdn, TSDB_FQDN_LEN); @@ -104,7 +105,8 @@ static void tscUpdateVgroupInfo(SSqlObj *pObj, SRpcIpSet *pIpSet) { SCMVgroupInfo *pVgroupInfo = &pTableMetaInfo->pTableMeta->vgroupInfo; taosCorBeginWrite(&pVgroupInfo->version); - //TODO(dengyihao), dont care vgid + //TODO(dengyihao), dont care vgid + pVgroupInfo->inUse = pIpSet->inUse; pVgroupInfo->numOfIps = pIpSet->numOfIps; for (int32_t i = 0; pVgroupInfo->numOfIps; i++) { strncpy(pVgroupInfo->ipAddr[i].fqdn, pIpSet->fqdn[i], TSDB_FQDN_LEN); @@ -1897,6 +1899,7 @@ int tscProcessSTableVgroupRsp(SSqlObj *pSql) { //just init, no need to lock SCMVgroupInfo *pVgroups = &pInfo->vgroupList->vgroups[j]; pVgroups->version = 0; + pVgroups->inUse = 0; pVgroups->vgId = htonl(pVgroups->vgId); assert(pVgroups->numOfIps >= 1); for (int32_t k = 0; k < pVgroups->numOfIps; ++k) { diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index c12aed2bcc..fec3d77d8e 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -649,6 +649,7 @@ typedef struct SCMSTableVgroupMsg { typedef struct { int32_t version; int32_t vgId; + int8_t inUse; int8_t numOfIps; SIpAddr ipAddr[TSDB_MAX_REPLICA]; } SCMVgroupInfo; From f569bb4babb1341c4128e24cfe2c20005cca011e Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Fri, 17 Jul 2020 14:27:02 +0800 Subject: [PATCH 09/34] td-909: fix subscription bugs --- src/client/src/tscSub.c | 164 ++++++++++++++++++++++---------------- src/query/src/qExecutor.c | 2 +- 2 files changed, 97 insertions(+), 69 deletions(-) diff --git a/src/client/src/tscSub.c b/src/client/src/tscSub.c index 2c5035c2ef..73614fa0a5 100644 --- a/src/client/src/tscSub.c +++ b/src/client/src/tscSub.c @@ -34,6 +34,7 @@ typedef struct SSubscriptionProgress { typedef struct SSub { void * signature; char topic[32]; + sem_t sem; int64_t lastSyncTime; int64_t lastConsumeTime; TAOS * taos; @@ -83,84 +84,107 @@ void tscUpdateSubscriptionProgress(void* sub, int64_t uid, TSKEY ts) { static void asyncCallback(void *param, TAOS_RES *tres, int code) { assert(param != NULL); - SSqlObj *pSql = ((SSqlObj *)param); - - pSql->res.code = code; - sem_post(&pSql->rspSem); + SSub *pSub = ((SSub *)param); + pSub->pSql->res.code = code; + sem_post(&pSub->sem); } static SSub* tscCreateSubscription(STscObj* pObj, const char* topic, const char* sql) { - SSub* pSub = NULL; + int code = TSDB_CODE_SUCCESS, line = __LINE__; - TRY( 8 ) { - SSqlObj* pSql = calloc_throw(1, sizeof(SSqlObj)); - CLEANUP_PUSH_FREE(true, pSql); - SSqlCmd *pCmd = &pSql->cmd; - SSqlRes *pRes = &pSql->res; + SSub* pSub = calloc(1, sizeof(SSub)); + if (pSub == NULL) { + line = __LINE__; + code = TSDB_CODE_TSC_OUT_OF_MEMORY; + goto fail; + } + pSub->signature = pSub; + if (tsem_init(&pSub->sem, 0, 0) == -1) { + line = __LINE__; + code = TAOS_SYSTEM_ERROR(errno); + goto fail; + } + tstrncpy(pSub->topic, topic, sizeof(pSub->topic)); + pSub->progress = taosArrayInit(32, sizeof(SSubscriptionProgress)); + if (pSub->progress == NULL) { + line = __LINE__; + code = TSDB_CODE_TSC_OUT_OF_MEMORY; + goto fail; + } - if (tsem_init(&pSql->rspSem, 0, 0) == -1) { - THROW(TAOS_SYSTEM_ERROR(errno)); - } - CLEANUP_PUSH_INT_PTR(true, tsem_destroy, &pSql->rspSem); + SSqlObj* pSql = calloc(1, sizeof(SSqlObj)); + if (pSql == NULL) { + line = __LINE__; + code = TSDB_CODE_TSC_OUT_OF_MEMORY; + goto fail; + } + pSql->signature = pSql; + pSql->pTscObj = pObj; + pSql->pSubscription = pSub; + pSub->pSql = pSql; - pSql->signature = pSql; - pSql->param = pSql; - pSql->pTscObj = pObj; - pSql->maxRetry = TSDB_MAX_REPLICA; - pSql->fp = asyncCallback; + SSqlCmd* pCmd = &pSql->cmd; + SSqlRes* pRes = &pSql->res; + if (tsem_init(&pSql->rspSem, 0, 0) == -1) { + line = __LINE__; + code = TAOS_SYSTEM_ERROR(errno); + goto fail; + } - int code = tscAllocPayload(pCmd, TSDB_DEFAULT_PAYLOAD_SIZE); - if (code != TSDB_CODE_SUCCESS) { - THROW(code); - } - CLEANUP_PUSH_FREE(true, pCmd->payload); + pSql->param = pSub; + pSql->maxRetry = TSDB_MAX_REPLICA; + pSql->fp = asyncCallback; + pSql->fetchFp = asyncCallback; + pSql->sqlstr = strdup(sql); + if (pSql->sqlstr == NULL) { + line = __LINE__; + code = TSDB_CODE_TSC_OUT_OF_MEMORY; + goto fail; + } + strtolower(pSql->sqlstr, pSql->sqlstr); + pRes->qhandle = 0; + pRes->numOfRows = 1; - pRes->qhandle = 0; - pRes->numOfRows = 1; + code = tscAllocPayload(pCmd, TSDB_DEFAULT_PAYLOAD_SIZE); + if (code != TSDB_CODE_SUCCESS) { + line = __LINE__; + goto fail; + } - pSql->sqlstr = strdup_throw(sql); - CLEANUP_PUSH_FREE(true, pSql->sqlstr); - strtolower(pSql->sqlstr, pSql->sqlstr); + code = tsParseSql(pSql, false); + if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { + sem_wait(&pSub->sem); + code = pSql->res.code; + } + if (code != TSDB_CODE_SUCCESS) { + line = __LINE__; + goto fail; + } - code = tsParseSql(pSql, false); - if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { - // wait for the callback function to post the semaphore - sem_wait(&pSql->rspSem); - code = pSql->res.code; - } - if (code != TSDB_CODE_SUCCESS) { - tscError("failed to parse sql statement: %s, error: %s", pSub->topic, tstrerror(code)); - THROW( code ); - } - - if (pSql->cmd.command != TSDB_SQL_SELECT) { - tscError("only 'select' statement is allowed in subscription: %s", pSub->topic); - THROW( -1 ); // TODO - } - - pSub = calloc_throw(1, sizeof(SSub)); - CLEANUP_PUSH_FREE(true, pSub); - pSql->pSubscription = pSub; - pSub->pSql = pSql; - pSub->signature = pSub; - strncpy(pSub->topic, topic, sizeof(pSub->topic)); - pSub->topic[sizeof(pSub->topic) - 1] = 0; - pSub->progress = taosArrayInit(32, sizeof(SSubscriptionProgress)); - if (pSub->progress == NULL) { - THROW(TSDB_CODE_TSC_OUT_OF_MEMORY); - } - - CLEANUP_EXECUTE(); - - } CATCH( code ) { - tscError("failed to create subscription object: %s", tstrerror(code)); - CLEANUP_EXECUTE(); - pSub = NULL; - - } END_TRY + if (pSql->cmd.command != TSDB_SQL_SELECT) { + line = __LINE__; + code = TSDB_CODE_TSC_INVALID_SQL; + goto fail; + } return pSub; + +fail: + tscError("tscCreateSubscription failed at line %d, reason: %s", line, tstrerror(code)); + if (pSql != NULL) { + tscFreeSqlObj(pSql); + pSql = NULL; + } + if (pSub != NULL) { + taosArrayDestroy(pSub->progress); + tsem_destroy(&pSub->sem); + free(pSub); + pSub = NULL; + } + + terrno = code; + return NULL; } @@ -405,9 +429,10 @@ TAOS_RES *taos_consume(TAOS_SUB *tsub) { tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0)->vgroupIndex = 0; pSql->fp = asyncCallback; - pSql->param = pSql; + pSql->fetchFp = asyncCallback; + pSql->param = pSub; tscDoQuery(pSql); - sem_wait(&pSql->rspSem); + sem_wait(&pSub->sem); if (pRes->code != TSDB_CODE_SUCCESS) { continue; @@ -437,7 +462,9 @@ void taos_unsubscribe(TAOS_SUB *tsub, int keepProgress) { } if (keepProgress) { - tscSaveSubscriptionProgress(pSub); + if (pSub->progress != NULL) { + tscSaveSubscriptionProgress(pSub); + } } else { char path[256]; sprintf(path, "%s/subscribe/%s", tsDataDir, pSub->topic); @@ -448,6 +475,7 @@ void taos_unsubscribe(TAOS_SUB *tsub, int keepProgress) { tscFreeSqlObj(pSub->pSql); taosArrayDestroy(pSub->progress); + tsem_destroy(&pSub->sem); memset(pSub, 0, sizeof(*pSub)); free(pSub); } diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index cf13ace11f..3b10967e33 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -4918,7 +4918,7 @@ static void tableMultiOutputProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) pQuery->current->lastKey, pQuery->window.ekey); } else if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { STableIdInfo tidInfo; - STableId* id = TSDB_TABLEID(pQuery->current); + STableId* id = TSDB_TABLEID(pQuery->current->pTable); tidInfo.uid = id->uid; tidInfo.tid = id->tid; From f9cb99e6f79e0cab9a40ad4840c49f348f2392b8 Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Fri, 17 Jul 2020 15:47:20 +0800 Subject: [PATCH 10/34] td-909: make taos_validate_sql work --- src/client/src/tscSql.c | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index d5160cd3c6..49e4e027a0 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -724,6 +724,13 @@ int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields) return len; } +static void asyncCallback(void *param, TAOS_RES *tres, int code) { + assert(param != NULL); + SSqlObj *pSql = ((SSqlObj *)param); + pSql->res.code = code; + sem_post(&pSql->rspSem); +} + int taos_validate_sql(TAOS *taos, const char *sql) { STscObj *pObj = (STscObj *)taos; if (pObj == NULL || pObj->signature != pObj) { @@ -732,7 +739,8 @@ int taos_validate_sql(TAOS *taos, const char *sql) { } SSqlObj* pSql = calloc(1, sizeof(SSqlObj)); - + pSql->pTscObj = taos; + pSql->signature = pSql; SSqlRes *pRes = &pSql->res; SSqlCmd *pCmd = &pSql->cmd; @@ -766,10 +774,17 @@ int taos_validate_sql(TAOS *taos, const char *sql) { pCmd->pTableList = NULL; } - pRes->code = (uint8_t)tsParseSql(pSql, false); - int code = pRes->code; - - tscDebug("%p Valid SQL result:%d, %s pObj:%p", pSql, pRes->code, taos_errstr(taos), pObj); + pSql->fp = asyncCallback; + pSql->fetchFp = asyncCallback; + pSql->param = pSql; + int code = tsParseSql(pSql, true); + if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { + sem_wait(&pSql->rspSem); + code = pSql->res.code; + } + if (code != TSDB_CODE_SUCCESS) { + tscDebug("%p Valid SQL result:%d, %s pObj:%p", pSql, code, taos_errstr(taos), pObj); + } taos_free_result(pSql); return code; From efe558731b14053a04cbfb84ea1e1000fce0881b Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Fri, 17 Jul 2020 15:56:49 +0800 Subject: [PATCH 11/34] td-909: add verification for some API --- tests/examples/c/apitest.c | 394 +++++++++++++++++++++++++++++++++++++ 1 file changed, 394 insertions(+) create mode 100644 tests/examples/c/apitest.c diff --git a/tests/examples/c/apitest.c b/tests/examples/c/apitest.c new file mode 100644 index 0000000000..0be1f01bbe --- /dev/null +++ b/tests/examples/c/apitest.c @@ -0,0 +1,394 @@ +// sample code to verify all TDengine API +// to compile: gcc -o apitest apitest.c -ltaos + +#include +#include +#include +#include +#include + + +static void prepare_data(TAOS* taos) { + taos_query(taos, "drop database if exists test;"); + usleep(100000); + taos_query(taos, "create database test;"); + usleep(100000); + taos_select_db(taos, "test"); + + taos_query(taos, "create table meters(ts timestamp, a int) tags(area int);"); + + taos_query(taos, "create table t0 using meters tags(0);"); + taos_query(taos, "create table t1 using meters tags(1);"); + taos_query(taos, "create table t2 using meters tags(2);"); + taos_query(taos, "create table t3 using meters tags(3);"); + taos_query(taos, "create table t4 using meters tags(4);"); + taos_query(taos, "create table t5 using meters tags(5);"); + taos_query(taos, "create table t6 using meters tags(6);"); + taos_query(taos, "create table t7 using meters tags(7);"); + taos_query(taos, "create table t8 using meters tags(8);"); + taos_query(taos, "create table t9 using meters tags(9);"); + + TAOS_RES* res = taos_query(taos, "insert into t0 values('2020-01-01 00:00:00.000', 0)" + " ('2020-01-01 00:01:00.000', 0)" + " ('2020-01-01 00:02:00.000', 0)" + " t1 values('2020-01-01 00:00:00.000', 0)" + " ('2020-01-01 00:01:00.000', 0)" + " ('2020-01-01 00:02:00.000', 0)" + " ('2020-01-01 00:03:00.000', 0)" + " t2 values('2020-01-01 00:00:00.000', 0)" + " ('2020-01-01 00:01:00.000', 0)" + " ('2020-01-01 00:01:01.000', 0)" + " ('2020-01-01 00:01:02.000', 0)" + " t3 values('2020-01-01 00:01:02.000', 0)" + " t4 values('2020-01-01 00:01:02.000', 0)" + " t5 values('2020-01-01 00:01:02.000', 0)" + " t6 values('2020-01-01 00:01:02.000', 0)" + " t7 values('2020-01-01 00:01:02.000', 0)" + " t8 values('2020-01-01 00:01:02.000', 0)" + " t9 values('2020-01-01 00:01:02.000', 0)"); + int affected = taos_affected_rows(res); + if (affected != 18) { + printf("\033[31m%d rows affected by last insert statement, but it should be 18\033[0m\n", affected); + } + // super tables subscription + usleep(1000000); +} + + +static int print_result(TAOS_RES* res, int blockFetch) { + TAOS_ROW row = NULL; + int num_fields = taos_num_fields(res); + TAOS_FIELD* fields = taos_fetch_fields(res); + int nRows = 0; + + if (blockFetch) { + int rows = 0; + while ((rows = taos_fetch_block(res, &row))) { + for (int i = 0; i < rows; i++) { + char temp[256]; + taos_print_row(temp, row + i, fields, num_fields); + puts(temp); + } + nRows += rows; + } + } else { + while ((row = taos_fetch_row(res))) { + char temp[256]; + taos_print_row(temp, row, fields, num_fields); + puts(temp); + nRows++; + } + } + + printf("%d rows consumed.\n", nRows); + return nRows; +} + + +static void check_row_count(int line, TAOS_RES* res, int expected) { + int actual = print_result(res, expected % 2); + if (actual != expected) { + printf("\033[31mline %d: row count mismatch, expected: %d, actual: %d\033[0m\n", line, expected, actual); + } else { + printf("line %d: %d rows consumed as expected\n", line, actual); + } +} + + +static void verify_query(TAOS* taos) { + prepare_data(taos); + + int code = taos_validate_sql(taos, "select * from nonexisttable"); + if (code == 0) { + printf("\033[31mimpossible, the table does not exists\033[0m\n"); + } + + code = taos_validate_sql(taos, "select * from meters"); + if (code != 0) { + printf("\033[31mimpossible, the table does exists: 0x%08x\033[0m\n", code); + } + + TAOS_RES* res = taos_query(taos, "select * from meters"); + check_row_count(__LINE__, res, 18); + printf("result precision is: %d\n", taos_result_precision(res)); + int c = taos_field_count(res); + printf("field count is: %d\n", c); + int* lengths = taos_fetch_lengths(res); + for (int i = 0; i < c; i++) { + printf("length of column %d is %d\n", i, lengths[i]); + } + taos_free_result(res); + + res = taos_query(taos, "select * from t0"); + check_row_count(__LINE__, res, 3); + taos_free_result(res); + + res = taos_query(taos, "select * from nonexisttable"); + code = taos_errno(res); + printf("code=%d, error msg=%s\n", code, taos_errstr(res)); + taos_free_result(res); +} + + +static void verify_subscribe(TAOS* taos) { + prepare_data(taos); + + TAOS_SUB* tsub = taos_subscribe(taos, 0, "test", "select * from meters;", NULL, NULL, 0); + TAOS_RES* res = taos_consume(tsub); + check_row_count(__LINE__, res, 18); + + res = taos_consume(tsub); + check_row_count(__LINE__, res, 0); + + taos_query(taos, "insert into t0 values('2020-01-01 00:02:00.001', 0);"); + taos_query(taos, "insert into t8 values('2020-01-01 00:01:03.000', 0);"); + res = taos_consume(tsub); + check_row_count(__LINE__, res, 2); + + taos_query(taos, "insert into t2 values('2020-01-01 00:01:02.001', 0);"); + taos_query(taos, "insert into t1 values('2020-01-01 00:03:00.001', 0);"); + res = taos_consume(tsub); + check_row_count(__LINE__, res, 2); + + taos_query(taos, "insert into t1 values('2020-01-01 00:03:00.002', 0);"); + res = taos_consume(tsub); + check_row_count(__LINE__, res, 1); + + // keep progress information and restart subscription + taos_unsubscribe(tsub, 1); + taos_query(taos, "insert into t0 values('2020-01-01 00:04:00.000', 0);"); + tsub = taos_subscribe(taos, 1, "test", "select * from meters;", NULL, NULL, 0); + res = taos_consume(tsub); + check_row_count(__LINE__, res, 24); + + // keep progress information and continue previous subscription + taos_unsubscribe(tsub, 1); + tsub = taos_subscribe(taos, 0, "test", "select * from meters;", NULL, NULL, 0); + res = taos_consume(tsub); + check_row_count(__LINE__, res, 0); + + // don't keep progress information and continue previous subscription + taos_unsubscribe(tsub, 0); + tsub = taos_subscribe(taos, 0, "test", "select * from meters;", NULL, NULL, 0); + res = taos_consume(tsub); + check_row_count(__LINE__, res, 24); + + // single meter subscription + + taos_unsubscribe(tsub, 0); + tsub = taos_subscribe(taos, 0, "test", "select * from t0;", NULL, NULL, 0); + res = taos_consume(tsub); + check_row_count(__LINE__, res, 5); + + res = taos_consume(tsub); + check_row_count(__LINE__, res, 0); + + taos_query(taos, "insert into t0 values('2020-01-01 00:04:00.001', 0);"); + res = taos_consume(tsub); + check_row_count(__LINE__, res, 1); + + taos_unsubscribe(tsub, 0); +} + + +void verify_prepare(TAOS* taos) { + TAOS_RES* result = taos_query(taos, "drop database if exists test;"); + usleep(100000); + taos_query(taos, "create database test;"); + + int code = taos_errno(result); + if (code != 0) { + printf("failed to create database, reason:%s\n", taos_errstr(result)); + taos_free_result(result); + exit(1); + } + taos_free_result(result); + + usleep(100000); + taos_select_db(taos, "test"); + + // create table + const char* sql = "create table m1 (ts timestamp, b bool, v1 tinyint, v2 smallint, v4 int, v8 bigint, f4 float, f8 double, bin binary(40), blob nchar(10))"; + result = taos_query(taos, sql); + code = taos_errno(result); + if (code != 0) { + printf("failed to create table, reason:%s\n", taos_errstr(result)); + taos_free_result(result); + exit(1); + } + taos_free_result(result); + + // insert 10 records + struct { + int64_t ts; + int8_t b; + int8_t v1; + int16_t v2; + int32_t v4; + int64_t v8; + float f4; + double f8; + char bin[40]; + char blob[80]; + } v = {0}; + + TAOS_STMT* stmt = taos_stmt_init(taos); + TAOS_BIND params[10]; + params[0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; + params[0].buffer_length = sizeof(v.ts); + params[0].buffer = &v.ts; + params[0].length = ¶ms[0].buffer_length; + params[0].is_null = NULL; + + params[1].buffer_type = TSDB_DATA_TYPE_BOOL; + params[1].buffer_length = sizeof(v.b); + params[1].buffer = &v.b; + params[1].length = ¶ms[1].buffer_length; + params[1].is_null = NULL; + + params[2].buffer_type = TSDB_DATA_TYPE_TINYINT; + params[2].buffer_length = sizeof(v.v1); + params[2].buffer = &v.v1; + params[2].length = ¶ms[2].buffer_length; + params[2].is_null = NULL; + + params[3].buffer_type = TSDB_DATA_TYPE_SMALLINT; + params[3].buffer_length = sizeof(v.v2); + params[3].buffer = &v.v2; + params[3].length = ¶ms[3].buffer_length; + params[3].is_null = NULL; + + params[4].buffer_type = TSDB_DATA_TYPE_INT; + params[4].buffer_length = sizeof(v.v4); + params[4].buffer = &v.v4; + params[4].length = ¶ms[4].buffer_length; + params[4].is_null = NULL; + + params[5].buffer_type = TSDB_DATA_TYPE_BIGINT; + params[5].buffer_length = sizeof(v.v8); + params[5].buffer = &v.v8; + params[5].length = ¶ms[5].buffer_length; + params[5].is_null = NULL; + + params[6].buffer_type = TSDB_DATA_TYPE_FLOAT; + params[6].buffer_length = sizeof(v.f4); + params[6].buffer = &v.f4; + params[6].length = ¶ms[6].buffer_length; + params[6].is_null = NULL; + + params[7].buffer_type = TSDB_DATA_TYPE_DOUBLE; + params[7].buffer_length = sizeof(v.f8); + params[7].buffer = &v.f8; + params[7].length = ¶ms[7].buffer_length; + params[7].is_null = NULL; + + params[8].buffer_type = TSDB_DATA_TYPE_BINARY; + params[8].buffer_length = sizeof(v.bin); + params[8].buffer = v.bin; + params[8].length = ¶ms[8].buffer_length; + params[8].is_null = NULL; + + strcpy(v.blob, "一二三四五六七八九十"); + params[9].buffer_type = TSDB_DATA_TYPE_NCHAR; + params[9].buffer_length = strlen(v.blob); + params[9].buffer = v.blob; + params[9].length = ¶ms[9].buffer_length; + params[9].is_null = NULL; + + int is_null = 1; + + sql = "insert into m1 values(?,?,?,?,?,?,?,?,?,?)"; + code = taos_stmt_prepare(stmt, sql, 0); + if (code != 0){ + printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); + } + v.ts = 1591060628000; + for (int i = 0; i < 10; ++i) { + v.ts += 1; + for (int j = 1; j < 10; ++j) { + params[j].is_null = ((i == j) ? &is_null : 0); + } + v.b = (int8_t)i % 2; + v.v1 = (int8_t)i; + v.v2 = (int16_t)(i * 2); + v.v4 = (int32_t)(i * 4); + v.v8 = (int64_t)(i * 8); + v.f4 = (float)(i * 40); + v.f8 = (double)(i * 80); + for (int j = 0; j < sizeof(v.bin) - 1; ++j) { + v.bin[j] = (char)(i + '0'); + } + + taos_stmt_bind_param(stmt, params); + taos_stmt_add_batch(stmt); + } + if (taos_stmt_execute(stmt) != 0) { + printf("failed to execute insert statement.\n"); + exit(1); + } + taos_stmt_close(stmt); + + // query the records + stmt = taos_stmt_init(taos); + taos_stmt_prepare(stmt, "SELECT * FROM m1 WHERE v1 > ? AND v2 < ?", 0); + v.v1 = 5; + v.v2 = 15; + taos_stmt_bind_param(stmt, params + 2); + if (taos_stmt_execute(stmt) != 0) { + printf("failed to execute select statement.\n"); + exit(1); + } + + result = taos_stmt_use_result(stmt); + + TAOS_ROW row; + int rows = 0; + int num_fields = taos_num_fields(result); + TAOS_FIELD *fields = taos_fetch_fields(result); + char temp[256]; + + // fetch the records row by row + while ((row = taos_fetch_row(result))) { + rows++; + taos_print_row(temp, row, fields, num_fields); + printf("%s\n", temp); + } + + taos_free_result(result); + taos_stmt_close(stmt); +} + +void verify_stream(TAOS* taos) { +} + +int main(int argc, char *argv[]) { + const char* host = "127.0.0.1"; + const char* user = "root"; + const char* passwd = "taosdata"; + + taos_options(TSDB_OPTION_TIMEZONE, "GMT-8"); + taos_init(); + + TAOS* taos = taos_connect(host, user, passwd, "", 0); + if (taos == NULL) { + printf("failed to connect to db, reason:%s\n", taos_errstr(taos)); + exit(1); + } + + char* info = taos_get_server_info(taos); + printf("server info: %s\n", info); + info = taos_get_client_info(taos); + printf("client info: %s\n", info); + + printf("************ verify query *************\n"); + verify_query(taos); + printf("*********** verify subscribe ************\n"); + verify_subscribe(taos); + printf("************ verify prepare *************\n"); + verify_prepare(taos); + printf("************ verify stream *************\n"); + verify_stream(taos); + + taos_close(taos); + taos_cleanup(); +} \ No newline at end of file From dbd22da833284b240dda7fe32935c1fb9a5009d0 Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Fri, 17 Jul 2020 18:28:33 +0800 Subject: [PATCH 12/34] td-909: fix stream start time issue fix crash in taos_load_table_info --- src/client/src/tscSql.c | 2 + src/client/src/tscStream.c | 6 ++- tests/examples/c/apitest.c | 82 +++++++++++++++++++++++++++++++++++++- 3 files changed, 88 insertions(+), 2 deletions(-) diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index 49e4e027a0..b1c85dd662 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -880,6 +880,8 @@ int taos_load_table_info(TAOS *taos, const char *tableNameList) { } SSqlObj* pSql = calloc(1, sizeof(SSqlObj)); + pSql->pTscObj = taos; + pSql->signature = pSql; SSqlRes *pRes = &pSql->res; pRes->numOfTotal = 0; // the number of getting table meta from server diff --git a/src/client/src/tscStream.c b/src/client/src/tscStream.c index 7c188ec969..b07627c87b 100644 --- a/src/client/src/tscStream.c +++ b/src/client/src/tscStream.c @@ -122,7 +122,7 @@ static void tscProcessStreamTimer(void *handle, void *tmrId) { pQueryInfo->window.ekey = pStream->etime; } } else { - pQueryInfo->window.skey = pStream->stime - pStream->interval; + pQueryInfo->window.skey = pStream->stime; int64_t etime = taosGetTimestamp(pStream->precision); // delay to wait all data in last time window if (pStream->precision == TSDB_TIME_PRECISION_MICRO) { @@ -232,6 +232,9 @@ static void tscProcessStreamRetrieveResult(void *param, TAOS_RES *res, int numOf (*pStream->fp)(pStream->param, res, row); } + if (!pStream->isProject) { + pStream->stime += pStream->slidingTime; + } // actually only one row is returned. this following is not necessary taos_fetch_rows_a(res, tscProcessStreamRetrieveResult, pStream); } else { // numOfRows == 0, all data has been retrieved @@ -432,6 +435,7 @@ static int64_t tscGetStreamStartTimestamp(SSqlObj *pSql, SSqlStream *pStream, in } else { // timewindow based aggregation stream if (stime == 0) { // no data in meter till now stime = ((int64_t)taosGetTimestamp(pStream->precision) / pStream->interval) * pStream->interval; + stime -= pStream->interval; tscWarn("%p stream:%p, last timestamp:0, reset to:%" PRId64, pSql, pStream, stime); } else { int64_t newStime = (stime / pStream->interval) * pStream->interval; diff --git a/tests/examples/c/apitest.c b/tests/examples/c/apitest.c index 0be1f01bbe..bc628fb58f 100644 --- a/tests/examples/c/apitest.c +++ b/tests/examples/c/apitest.c @@ -98,7 +98,12 @@ static void check_row_count(int line, TAOS_RES* res, int expected) { static void verify_query(TAOS* taos) { prepare_data(taos); - int code = taos_validate_sql(taos, "select * from nonexisttable"); + int code = taos_load_table_info(taos, "t0,t1,t2,t3,t4,t5,t6,t7,t8,t9"); + if (code != 0) { + printf("\033[31mfailed to load table info: 0x%08x\033[0m\n", code); + } + + code = taos_validate_sql(taos, "select * from nonexisttable"); if (code == 0) { printf("\033[31mimpossible, the table does not exists\033[0m\n"); } @@ -127,9 +132,17 @@ static void verify_query(TAOS* taos) { code = taos_errno(res); printf("code=%d, error msg=%s\n", code, taos_errstr(res)); taos_free_result(res); + + res = taos_query(taos, "select * from meters"); + taos_stop_query(res); } +void subscribe_callback(TAOS_SUB* tsub, TAOS_RES *res, void* param, int code) { + int rows = print_result(res, *(int*)param); + printf("%d rows consumed in subscribe_callback\n", rows); +} + static void verify_subscribe(TAOS* taos) { prepare_data(taos); @@ -188,6 +201,13 @@ static void verify_subscribe(TAOS* taos) { check_row_count(__LINE__, res, 1); taos_unsubscribe(tsub, 0); + + int blockFetch = 0; + tsub = taos_subscribe(taos, 1, "test", "select * from meters;", subscribe_callback, &blockFetch, 1000); + usleep(2000000); + taos_query(taos, "insert into t0 values('2020-01-01 00:05:00.001', 0);"); + usleep(2000000); + taos_unsubscribe(tsub, 0); } @@ -358,7 +378,60 @@ void verify_prepare(TAOS* taos) { taos_stmt_close(stmt); } +void retrieve_callback(void *param, TAOS_RES *tres, int numOfRows) +{ + if (numOfRows > 0) { + printf("%d rows async retrieved\n", numOfRows); + taos_fetch_rows_a(tres, retrieve_callback, param); + } else { + if (numOfRows < 0) { + printf("\033[31masync retrieve failed, code: %d\033[0m\n", numOfRows); + } else { + printf("async retrieve completed\n"); + } + taos_free_result(tres); + } +} + +void select_callback(void *param, TAOS_RES *tres, int code) +{ + if (code == 0 && tres) { + taos_fetch_rows_a(tres, retrieve_callback, param); + } else { + printf("\033[31masync select failed, code: %d\033[0m\n", code); + } +} + +void verify_async(TAOS* taos) { + prepare_data(taos); + taos_query_a(taos, "select * from meters", select_callback, NULL); + usleep(1000000); +} + +void stream_callback(void *param, TAOS_RES *res, TAOS_ROW row) { + int num_fields = taos_num_fields(res); + TAOS_FIELD* fields = taos_fetch_fields(res); + + printf("got one row from stream_callback\n"); + char temp[256]; + taos_print_row(temp, row, fields, num_fields); + puts(temp); +} + void verify_stream(TAOS* taos) { + prepare_data(taos); + TAOS_STREAM* strm = taos_open_stream( + taos, + "select count(*) from meters interval(1m)", + stream_callback, + 0, + NULL, + NULL); + printf("waiting for stream data\n"); + usleep(100000); + taos_query(taos, "insert into t0 values(now, 0)(now+5s,1)(now+10s, 2);"); + usleep(200000000); + taos_close_stream(strm); } int main(int argc, char *argv[]) { @@ -382,12 +455,19 @@ int main(int argc, char *argv[]) { printf("************ verify query *************\n"); verify_query(taos); + + printf("********* verify async query **********\n"); + verify_async(taos); + printf("*********** verify subscribe ************\n"); verify_subscribe(taos); + printf("************ verify prepare *************\n"); verify_prepare(taos); + printf("************ verify stream *************\n"); verify_stream(taos); + printf("done\n"); taos_close(taos); taos_cleanup(); From 700504900fc6419f6e87f26d2449ab2168144017 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sat, 18 Jul 2020 01:32:00 +0000 Subject: [PATCH 13/34] compile error --- src/client/src/tscServer.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index d0c9d3636b..cafdcfbf25 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -155,9 +155,10 @@ void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) { if (code == 0) { SCMHeartBeatRsp *pRsp = (SCMHeartBeatRsp *)pRes->pRsp; SRpcIpSet * pIpList = &pRsp->ipList; - if (pIpList->numOfIps > 0) + if (pIpList->numOfIps > 0) { tscIpSetHtons(pIpList); tscUpdateMgmtIpList(pIpList); + } pSql->pTscObj->connId = htonl(pRsp->connId); @@ -1996,9 +1997,10 @@ int tscProcessConnectRsp(SSqlObj *pSql) { assert(len <= sizeof(pObj->db)); tstrncpy(pObj->db, temp, sizeof(pObj->db)); - if (pConnect->ipList.numOfIps > 0) + if (pConnect->ipList.numOfIps > 0) { tscIpSetHtons(&pConnect->ipList); tscUpdateMgmtIpList(&pConnect->ipList); + } strcpy(pObj->sversion, pConnect->serverVersion); pObj->writeAuth = pConnect->writeAuth; From 54ab10737a7dfc0b433680a16751858077fb6e8b Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sat, 18 Jul 2020 01:43:22 +0000 Subject: [PATCH 14/34] compile error --- src/client/src/tscServer.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index cafdcfbf25..9425b32aab 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -26,6 +26,7 @@ #include "ttime.h" #include "ttimer.h" #include "tutil.h" +#include "tlockfree.h" #define TSC_MGMT_VNODE 999 From 7aa062c9ad5679891f0d949160146c02eaa6b71a Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sat, 18 Jul 2020 03:03:04 +0000 Subject: [PATCH 15/34] modify function --- src/util/inc/tlockfree.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/util/inc/tlockfree.h b/src/util/inc/tlockfree.h index e425d71d27..a81f597832 100644 --- a/src/util/inc/tlockfree.h +++ b/src/util/inc/tlockfree.h @@ -75,7 +75,7 @@ void taosRUnLockLatch(SRWLatch *pLatch); // copy on read #define taosCorBeginRead(x) for (uint32_t i_ = 1; 1; ++i_) { \ - int32_t old_ = atomic_load_32(x); \ + int32_t old_ = atomic_add_fetch_32((x), 0); \ if (old_ & 0x00000001) { \ if (i_ % 1000 == 0) { \ sched_yield(); \ @@ -84,7 +84,7 @@ void taosRUnLockLatch(SRWLatch *pLatch); } #define taosCorEndRead(x) \ - if (atomic_load_32(x) == old_) { \ + if (atomic_add_fetch_32((x), 0) == old_) { \ break; \ } \ } From a28e9a9402fcb802fa7a301d76385fb083d6db3d Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sat, 18 Jul 2020 03:21:16 +0000 Subject: [PATCH 16/34] test --- src/client/src/tscServer.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 9425b32aab..7269173c8c 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -85,7 +85,7 @@ void tscUpdateMgmtIpList(SRpcIpSet *pIpSet) { } static void tscDumpIpSetFromVgroupInfo(SCMVgroupInfo *pVgroupInfo, SRpcIpSet *pIpSet, int32_t *vgId) { if (pVgroupInfo == NULL) { return;} - taosCorBeginRead(&pVgroupInfo->version); + //taosCorBeginRead(&pVgroupInfo->version); if (vgId) { *vgId = pVgroupInfo->vgId; } @@ -96,7 +96,7 @@ static void tscDumpIpSetFromVgroupInfo(SCMVgroupInfo *pVgroupInfo, SRpcIpSet *pI strncpy(pIpSet->fqdn[i], pVgroupInfo->ipAddr[i].fqdn, TSDB_FQDN_LEN); pIpSet->port[i] = pVgroupInfo->ipAddr[i].port; } - taosCorEndRead(&pVgroupInfo->version); + //taosCorEndRead(&pVgroupInfo->version); } static void tscUpdateVgroupInfo(SSqlObj *pObj, SRpcIpSet *pIpSet) { @@ -105,7 +105,7 @@ static void tscUpdateVgroupInfo(SSqlObj *pObj, SRpcIpSet *pIpSet) { if (pTableMetaInfo == NULL || pTableMetaInfo->pTableMeta == NULL) { return;} SCMVgroupInfo *pVgroupInfo = &pTableMetaInfo->pTableMeta->vgroupInfo; - taosCorBeginWrite(&pVgroupInfo->version); + //taosCorBeginWrite(&pVgroupInfo->version); //TODO(dengyihao), dont care vgid pVgroupInfo->inUse = pIpSet->inUse; pVgroupInfo->numOfIps = pIpSet->numOfIps; @@ -113,7 +113,7 @@ static void tscUpdateVgroupInfo(SSqlObj *pObj, SRpcIpSet *pIpSet) { strncpy(pVgroupInfo->ipAddr[i].fqdn, pIpSet->fqdn[i], TSDB_FQDN_LEN); pVgroupInfo->ipAddr[i].port = pIpSet->port[i]; } - taosCorEndWrite(&pVgroupInfo->version); + //taosCorEndWrite(&pVgroupInfo->version); } void tscPrintMgmtIp() { SRpcIpSet dump; From 3c4e620f31257219d3871857f8c95102f87efecc Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sat, 18 Jul 2020 05:41:19 +0000 Subject: [PATCH 17/34] bugfix --- src/client/inc/tsclient.h | 10 ++++++- src/client/src/tscSchemaUtil.c | 16 ++++++++--- src/client/src/tscServer.c | 49 ++++++++++++++++++++-------------- src/inc/taosmsg.h | 2 -- 4 files changed, 50 insertions(+), 27 deletions(-) diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index e1180c4143..3a420567a9 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -52,12 +52,20 @@ typedef struct STableComInfo { int32_t rowSize; } STableComInfo; +typedef struct SCMCorVgroupInfo { + int32_t version; + int8_t inUse; + int8_t numOfIps; + SIpAddr ipAddr[TSDB_MAX_REPLICA]; +} SCMCorVgroupInfo; + typedef struct STableMeta { STableComInfo tableInfo; uint8_t tableType; int16_t sversion; int16_t tversion; - SCMVgroupInfo vgroupInfo; + SCMVgroupInfo vgroupInfo; + SCMCorVgroupInfo corVgroupInfo; int32_t sid; // the index of one table in a virtual node uint64_t uid; // unique id of a table SSchema schema[]; // if the table is TSDB_CHILD_TABLE, schema is acquired by super table meta info diff --git a/src/client/src/tscSchemaUtil.c b/src/client/src/tscSchemaUtil.c index b391a5760e..52342b3650 100644 --- a/src/client/src/tscSchemaUtil.c +++ b/src/client/src/tscSchemaUtil.c @@ -140,7 +140,15 @@ struct SSchema tscGetTbnameColumnSchema() { strcpy(s.name, TSQL_TBNAME_L); return s; } - +static void tscInitCorVgroupInfo(SCMCorVgroupInfo *corVgroupInfo, SCMVgroupInfo *vgroupInfo) { + corVgroupInfo->version = 0; + corVgroupInfo->inUse = 0; + corVgroupInfo->numOfIps = vgroupInfo->numOfIps; + for (int32_t i = 0; i < corVgroupInfo->numOfIps; i++) { + strncpy(corVgroupInfo->ipAddr[i].fqdn, vgroupInfo->ipAddr[i].fqdn, TSDB_FQDN_LEN); + corVgroupInfo->ipAddr[i].port = vgroupInfo->ipAddr[i].port; + } +} STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg, size_t* size) { assert(pTableMetaMsg != NULL); @@ -157,9 +165,9 @@ STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg, size_t* size pTableMeta->sid = pTableMetaMsg->sid; pTableMeta->uid = pTableMetaMsg->uid; pTableMeta->vgroupInfo = pTableMetaMsg->vgroup; - //init version here - pTableMeta->vgroupInfo.version = 0; - + + tscInitCorVgroupInfo(&pTableMeta->corVgroupInfo, &pTableMeta->vgroupInfo); + pTableMeta->sversion = pTableMetaMsg->sversion; pTableMeta->tversion = pTableMetaMsg->tversion; diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 7269173c8c..cb97307b04 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -45,6 +45,20 @@ void tscSaveSubscriptionProgress(void* sub); static int32_t minMsgSize() { return tsRpcHeadSize + 100; } +static void tscSetDnodeIpList(SSqlObj* pSql, SCMVgroupInfo* pVgroupInfo) { + SRpcIpSet* pIpList = &pSql->ipList; + pIpList->inUse = 0; + if (pVgroupInfo == NULL) { + pIpList->numOfIps = 0; + return; + } + + pIpList->numOfIps = pVgroupInfo->numOfIps; + for(int32_t i = 0; i < pVgroupInfo->numOfIps; ++i) { + strcpy(pIpList->fqdn[i], pVgroupInfo->ipAddr[i].fqdn); + pIpList->port[i] = pVgroupInfo->ipAddr[i].port; + } +} void tscIpSetCopy(SRpcIpSet *dst, SRpcIpSet *src) { dst->numOfIps = src->numOfIps; dst->inUse = src->inUse; @@ -83,12 +97,9 @@ void tscUpdateMgmtIpList(SRpcIpSet *pIpSet) { tscIpSetCopy(mgmtIpSet, pIpSet); taosCorEndWrite(&tscMgmtIpSet.version); } -static void tscDumpIpSetFromVgroupInfo(SCMVgroupInfo *pVgroupInfo, SRpcIpSet *pIpSet, int32_t *vgId) { +static void tscDumpIpSetFromVgroupInfo(SCMCorVgroupInfo *pVgroupInfo, SRpcIpSet *pIpSet) { if (pVgroupInfo == NULL) { return;} - //taosCorBeginRead(&pVgroupInfo->version); - if (vgId) { - *vgId = pVgroupInfo->vgId; - } + taosCorBeginRead(&pVgroupInfo->version); int8_t inUse = pVgroupInfo->inUse; pIpSet->inUse = (inUse >= 0 && inUse < TSDB_MAX_REPLICA) ? inUse: 0; pIpSet->numOfIps = pVgroupInfo->numOfIps; @@ -96,16 +107,16 @@ static void tscDumpIpSetFromVgroupInfo(SCMVgroupInfo *pVgroupInfo, SRpcIpSet *pI strncpy(pIpSet->fqdn[i], pVgroupInfo->ipAddr[i].fqdn, TSDB_FQDN_LEN); pIpSet->port[i] = pVgroupInfo->ipAddr[i].port; } - //taosCorEndRead(&pVgroupInfo->version); + taosCorEndRead(&pVgroupInfo->version); } static void tscUpdateVgroupInfo(SSqlObj *pObj, SRpcIpSet *pIpSet) { SSqlCmd *pCmd = &pObj->cmd; STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); if (pTableMetaInfo == NULL || pTableMetaInfo->pTableMeta == NULL) { return;} - SCMVgroupInfo *pVgroupInfo = &pTableMetaInfo->pTableMeta->vgroupInfo; + SCMCorVgroupInfo *pVgroupInfo = &pTableMetaInfo->pTableMeta->corVgroupInfo; - //taosCorBeginWrite(&pVgroupInfo->version); + taosCorBeginWrite(&pVgroupInfo->version); //TODO(dengyihao), dont care vgid pVgroupInfo->inUse = pIpSet->inUse; pVgroupInfo->numOfIps = pIpSet->numOfIps; @@ -113,7 +124,7 @@ static void tscUpdateVgroupInfo(SSqlObj *pObj, SRpcIpSet *pIpSet) { strncpy(pVgroupInfo->ipAddr[i].fqdn, pIpSet->fqdn[i], TSDB_FQDN_LEN); pVgroupInfo->ipAddr[i].port = pIpSet->port[i]; } - //taosCorEndWrite(&pVgroupInfo->version); + taosCorEndWrite(&pVgroupInfo->version); } void tscPrintMgmtIp() { SRpcIpSet dump; @@ -577,7 +588,7 @@ int tscBuildSubmitMsg(SSqlObj *pSql, SSqlInfo *pInfo) { // pSql->cmd.payloadLen is set during copying data into payload pSql->cmd.msgType = TSDB_MSG_TYPE_SUBMIT; - tscDumpIpSetFromVgroupInfo(&pTableMeta->vgroupInfo, &pSql->ipList, NULL); + tscDumpIpSetFromVgroupInfo(&pTableMeta->corVgroupInfo, &pSql->ipList); tscDebug("%p build submit msg, vgId:%d numOfTables:%d numberOfIP:%d", pSql, vgId, pSql->cmd.numOfTablesInSubmit, pSql->ipList.numOfIps); @@ -619,9 +630,10 @@ static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char } else { pVgroupInfo = &pTableMeta->vgroupInfo; } - int32_t vgId = 0; - tscDumpIpSetFromVgroupInfo(pVgroupInfo, &pSql->ipList, &vgId); - pQueryMsg->head.vgId = htonl(vgId); + tscSetDnodeIpList(pSql, pVgroupInfo); + if (pVgroupInfo != NULL) { + pQueryMsg->head.vgId = htonl(pVgroupInfo->vgId); + } STableIdInfo *pTableIdInfo = (STableIdInfo *)pMsg; pTableIdInfo->tid = htonl(pTableMeta->sid); @@ -639,10 +651,9 @@ static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char SVgroupTableInfo* pTableIdList = taosArrayGet(pTableMetaInfo->pVgroupTables, index); - // set the vgroup info - int32_t vgId = 0; - tscDumpIpSetFromVgroupInfo(&pTableIdList->vgInfo, &pSql->ipList, &vgId); - pQueryMsg->head.vgId = htonl(vgId); + // set the vgroup info + tscSetDnodeIpList(pSql, &pTableIdList->vgInfo); + pQueryMsg->head.vgId = htonl(pTableIdList->vgInfo.vgId); int32_t numOfTables = taosArrayGetSize(pTableIdList->itemList); pQueryMsg->numOfTables = htonl(numOfTables); // set the number of tables @@ -1374,7 +1385,7 @@ int tscBuildUpdateTagMsg(SSqlObj* pSql, SSqlInfo *pInfo) { SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - tscDumpIpSetFromVgroupInfo(&pTableMetaInfo->pTableMeta->vgroupInfo, &pSql->ipList, NULL); + tscDumpIpSetFromVgroupInfo(&pTableMetaInfo->pTableMeta->corVgroupInfo, &pSql->ipList); return TSDB_CODE_SUCCESS; } @@ -1900,8 +1911,6 @@ int tscProcessSTableVgroupRsp(SSqlObj *pSql) { for (int32_t j = 0; j < pInfo->vgroupList->numOfVgroups; ++j) { //just init, no need to lock SCMVgroupInfo *pVgroups = &pInfo->vgroupList->vgroups[j]; - pVgroups->version = 0; - pVgroups->inUse = 0; pVgroups->vgId = htonl(pVgroups->vgId); assert(pVgroups->numOfIps >= 1); for (int32_t k = 0; k < pVgroups->numOfIps; ++k) { diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index fec3d77d8e..b7afaf1e06 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -647,9 +647,7 @@ typedef struct SCMSTableVgroupMsg { } SCMSTableVgroupMsg, SCMSTableVgroupRspMsg; typedef struct { - int32_t version; int32_t vgId; - int8_t inUse; int8_t numOfIps; SIpAddr ipAddr[TSDB_MAX_REPLICA]; } SCMVgroupInfo; From 7ccba2bdf2bb85eb7e5ddafea78e1c602e59efd3 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 18 Jul 2020 15:12:08 +0800 Subject: [PATCH 18/34] [td-225] refactor codes and add some log. --- src/query/inc/qast.h | 1 - src/query/src/qExecutor.c | 106 +++++++++++++++++++------------------- src/query/src/qast.c | 4 +- src/tsdb/src/tsdbRead.c | 30 ++--------- 4 files changed, 59 insertions(+), 82 deletions(-) diff --git a/src/query/inc/qast.h b/src/query/inc/qast.h index 918604f8c9..00049b486d 100644 --- a/src/query/inc/qast.h +++ b/src/query/inc/qast.h @@ -45,7 +45,6 @@ typedef void (*__do_filter_suppl_fn_t)(void *, void *); * */ typedef struct tQueryInfo { - int32_t colIndex; // index of column in schema uint8_t optr; // expression operator SSchema sch; // schema of tags char* q; diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 6dba5cbd2a..42fc90f478 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -50,11 +50,6 @@ #define SDATA_BLOCK_INITIALIZER (SDataBlockInfo) {{0}, 0} -/* get the qinfo struct address from the query struct address */ -#define GET_COLUMN_BYTES(query, colidx) \ - ((query)->colList[(query)->pSelectExpr[colidx].base.colInfo.colIndex].bytes) -#define GET_COLUMN_TYPE(query, colidx) ((query)->colList[(query)->pSelectExpr[colidx].base.colInfo.colIndex].type) - enum { // when query starts to execute, this status will set QUERY_NOT_COMPLETED = 0x1u, @@ -394,15 +389,15 @@ static SWindowResult *doSetTimeWindowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SWin if (pWindowResInfo->size >= pWindowResInfo->capacity) { int64_t newCap = pWindowResInfo->capacity * 1.5; char *t = realloc(pWindowResInfo->pResult, newCap * sizeof(SWindowResult)); - if (t != NULL) { - pWindowResInfo->pResult = (SWindowResult *)t; - - int32_t inc = newCap - pWindowResInfo->capacity; - memset(&pWindowResInfo->pResult[pWindowResInfo->capacity], 0, sizeof(SWindowResult) * inc); - } else { - // todo + if (t == NULL) { + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } + pWindowResInfo->pResult = (SWindowResult *)t; + + int32_t inc = newCap - pWindowResInfo->capacity; + memset(&pWindowResInfo->pResult[pWindowResInfo->capacity], 0, sizeof(SWindowResult) * inc); + for (int32_t i = pWindowResInfo->capacity; i < newCap; ++i) { createQueryResultInfo(pQuery, &pWindowResInfo->pResult[i], pRuntimeEnv->stableQuery, pRuntimeEnv->interBufSize); } @@ -1053,9 +1048,9 @@ static char *getGroupbyColumnData(SQuery *pQuery, int16_t *type, int16_t *bytes, *type = pQuery->colList[colIndex].type; *bytes = pQuery->colList[colIndex].bytes; /* - * the colIndex is acquired from the first meter of all qualified meters in this vnode during query prepare - * stage, the remain meter may not have the required column in cache actually. So, the validation of required - * column in cache with the corresponding meter schema is reinforced. + * the colIndex is acquired from the first tables of all qualified tables in this vnode during query prepare + * stage, the remain tables may not have the required column in cache actually. So, the validation of required + * column in cache with the corresponding schema is reinforced. */ int32_t numOfCols = taosArrayGetSize(pDataBlock); @@ -2364,6 +2359,18 @@ static void doSetTagValueInParam(void *tsdb, void* pTable, int32_t tagColId, tVa } } +static SColumnInfo* doGetTagColumnInfoById(SColumnInfo* pTagColList, int32_t numOfTags, int16_t colId) { + assert(pTagColList != NULL && numOfTags > 0); + + for(int32_t i = 0; i < numOfTags; ++i) { + if (pTagColList[i].colId == colId) { + return &pTagColList[i]; + } + } + + return NULL; +} + void setTagVal(SQueryRuntimeEnv *pRuntimeEnv, void *pTable, void *tsdb) { SQuery *pQuery = pRuntimeEnv->pQuery; SQInfo* pQInfo = GET_QINFO_ADDR(pRuntimeEnv); @@ -2372,16 +2379,10 @@ void setTagVal(SQueryRuntimeEnv *pRuntimeEnv, void *pTable, void *tsdb) { if (pQuery->numOfOutput == 1 && pExprInfo->base.functionId == TSDB_FUNC_TS_COMP) { assert(pExprInfo->base.numOfParams == 1); - // todo refactor extract function. - int16_t type = -1, bytes = -1; - for(int32_t i = 0; i < pQuery->numOfTags; ++i) { - if (pQuery->tagColList[i].colId == pExprInfo->base.arg->argValue.i64) { - type = pQuery->tagColList[i].type; - bytes = pQuery->tagColList[i].bytes; - } - } + int16_t tagColId = pExprInfo->base.arg->argValue.i64; + SColumnInfo* pColInfo = doGetTagColumnInfoById(pQuery->tagColList, pQuery->numOfTags, tagColId); - doSetTagValueInParam(tsdb, pTable, pExprInfo->base.arg->argValue.i64, &pRuntimeEnv->pCtx[0].tag, type, bytes); + doSetTagValueInParam(tsdb, pTable, tagColId, &pRuntimeEnv->pCtx[0].tag, pColInfo->type, pColInfo->bytes); } else { // set tag value, by which the results are aggregated. for (int32_t idx = 0; idx < pQuery->numOfOutput; ++idx) { @@ -2399,20 +2400,14 @@ void setTagVal(SQueryRuntimeEnv *pRuntimeEnv, void *pTable, void *tsdb) { // set the join tag for first column SSqlFuncMsg *pFuncMsg = &pExprInfo->base; - if ((pFuncMsg->functionId == TSDB_FUNC_TS || pFuncMsg->functionId == TSDB_FUNC_PRJ) && pFuncMsg->colInfo.colIndex == PRIMARYKEY_TIMESTAMP_COL_INDEX && - pRuntimeEnv->pTSBuf != NULL) { + if ((pFuncMsg->functionId == TSDB_FUNC_TS || pFuncMsg->functionId == TSDB_FUNC_PRJ) && pRuntimeEnv->pTSBuf != NULL && + pFuncMsg->colInfo.colIndex == PRIMARYKEY_TIMESTAMP_COL_INDEX) { assert(pFuncMsg->numOfParams == 1); - // todo refactor - int16_t type = -1, bytes = -1; - for(int32_t i = 0; i < pQuery->numOfTags; ++i) { - if (pQuery->tagColList[i].colId == pExprInfo->base.arg->argValue.i64) { - type = pQuery->tagColList[i].type; - bytes = pQuery->tagColList[i].bytes; - } - } + int16_t tagColId = pExprInfo->base.arg->argValue.i64; + SColumnInfo* pColInfo = doGetTagColumnInfoById(pQuery->tagColList, pQuery->numOfTags, tagColId); - doSetTagValueInParam(tsdb, pTable, pExprInfo->base.arg->argValue.i64, &pRuntimeEnv->pCtx[0].tag, type, bytes); + doSetTagValueInParam(tsdb, pTable, tagColId, &pRuntimeEnv->pCtx[0].tag, pColInfo->type, pColInfo->bytes); qDebug("QInfo:%p set tag value for join comparison, colId:%" PRId64 ", val:%"PRId64, pQInfo, pExprInfo->base.arg->argValue.i64, pRuntimeEnv->pCtx[0].tag.i64Key) } @@ -4149,6 +4144,7 @@ static int32_t setupQueryHandle(void* tsdb, SQInfo* pQInfo, bool isSTableQuery) } else { pRuntimeEnv->pQueryHandle = tsdbQueryTables(tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo); } + return terrno; } @@ -4174,10 +4170,10 @@ static SFillColInfo* taosCreateFillColInfo(SQuery* pQuery) { } int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bool isSTableQuery) { - int32_t code = TSDB_CODE_SUCCESS; - + qDebug("QInfo:%p start to init qhandle", pQInfo); SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; + int32_t code = TSDB_CODE_SUCCESS; SQuery *pQuery = pQInfo->runtimeEnv.pQuery; pQuery->precision = tsdbGetCfg(tsdb)->precision; @@ -4186,6 +4182,7 @@ int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bo setScanLimitationByResultBuffer(pQuery); changeExecuteScanOrder(pQInfo, false); + code = setupQueryHandle(tsdb, pQInfo, isSTableQuery); if (code != TSDB_CODE_SUCCESS) { return code; @@ -5693,7 +5690,6 @@ static void doUpdateExprColumnIndex(SQuery *pQuery) { } } - static int compareTableIdInfo(const void* a, const void* b) { const STableIdInfo* x = (const STableIdInfo*)a; const STableIdInfo* y = (const STableIdInfo*)b; @@ -5926,6 +5922,7 @@ static int32_t initQInfo(SQueryTableMsg *pQueryMsg, void *tsdb, int32_t vgId, SQ pQuery->window.ekey, pQuery->order.order); setQueryStatus(pQuery, QUERY_COMPLETED); pQInfo->tableqinfoGroupInfo.numOfTables = 0; + sem_post(&pQInfo->dataReady); return TSDB_CODE_SUCCESS; } @@ -6136,16 +6133,17 @@ int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, vo int32_t code = TSDB_CODE_SUCCESS; - char * tagCond = NULL, *tbnameCond = NULL; - SArray * pTableIdList = NULL; - SSqlFuncMsg **pExprMsg = NULL; - SColIndex * pGroupColIndex = NULL; - SColumnInfo* pTagColumnInfo = NULL; - SExprInfo *pExprs = NULL; - SSqlGroupbyExpr *pGroupbyExpr = NULL; + char *tagCond = NULL; + char *tbnameCond = NULL; + SArray *pTableIdList = NULL; + SSqlFuncMsg **pExprMsg = NULL; + SExprInfo *pExprs = NULL; + SColIndex *pGroupColIndex = NULL; + SColumnInfo *pTagColumnInfo = NULL; + SSqlGroupbyExpr *pGroupbyExpr = NULL; - if ((code = convertQueryMsg(pQueryMsg, &pTableIdList, &pExprMsg, &tagCond, &tbnameCond, &pGroupColIndex, &pTagColumnInfo)) != - TSDB_CODE_SUCCESS) { + code = convertQueryMsg(pQueryMsg, &pTableIdList, &pExprMsg, &tagCond, &tbnameCond, &pGroupColIndex, &pTagColumnInfo); + if (code != TSDB_CODE_SUCCESS) { goto _over; } @@ -6172,7 +6170,8 @@ int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, vo bool isSTableQuery = false; STableGroupInfo tableGroupInfo = {0}; - + int64_t st = taosGetTimestampUs(); + if (TSDB_QUERY_HAS_TYPE(pQueryMsg->queryType, TSDB_QUERY_TYPE_TABLE_QUERY)) { STableIdInfo *id = taosArrayGet(pTableIdList, 0); @@ -6182,7 +6181,6 @@ int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, vo } } else if (TSDB_QUERY_HAS_TYPE(pQueryMsg->queryType, TSDB_QUERY_TYPE_MULTITABLE_QUERY|TSDB_QUERY_TYPE_STABLE_QUERY)) { isSTableQuery = true; - // TODO: need a macro from TSDB to check if table is super table // also note there's possibility that only one table in the super table if (!TSDB_QUERY_HAS_TYPE(pQueryMsg->queryType, TSDB_QUERY_TYPE_MULTITABLE_QUERY)) { @@ -6193,11 +6191,12 @@ int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, vo if (pQueryMsg->numOfGroupCols == 1 && !TSDB_COL_IS_TAG(pGroupColIndex->flag)) { numOfGroupByCols = 0; } - + + qDebug("qmsg:%p query stable, uid:%"PRId64", tid:%d", pQueryMsg, id->uid, id->tid); code = tsdbQuerySTableByTagCond(tsdb, id->uid, tagCond, pQueryMsg->tagCondLen, pQueryMsg->tagNameRelType, tbnameCond, &tableGroupInfo, pGroupColIndex, numOfGroupByCols); if (code != TSDB_CODE_SUCCESS) { - qError("qmsg:%p failed to QueryStable, reason: %s", pQueryMsg, tstrerror(code)); + qError("qmsg:%p failed to query stable, reason: %s", pQueryMsg, tstrerror(code)); goto _over; } } else { @@ -6208,6 +6207,9 @@ int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, vo qDebug("qmsg:%p query on %zu tables in one group from client", pQueryMsg, tableGroupInfo.numOfTables); } + + int64_t el = taosGetTimestampUs() - st; + qDebug("qmsg:%p tag filter completed, elapsed time:%"PRId64"us", pQueryMsg, el); } else { assert(0); } @@ -6247,7 +6249,7 @@ _over: *pQInfo = NULL; } - // if failed to add ref for all meters in this query, abort current query + // if failed to add ref for all tables in this query, abort current query return code; } diff --git a/src/query/src/qast.c b/src/query/src/qast.c index da4eb8f3ba..42f9f214f5 100644 --- a/src/query/src/qast.c +++ b/src/query/src/qast.c @@ -678,7 +678,7 @@ static void tQueryIndexlessColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, tstr *name = (tstr*) tsdbGetTableName(*(void**) pData); // todo speed up by using hash - if (pQueryInfo->colIndex == TSDB_TBNAME_COLUMN_INDEX) { + if (pQueryInfo->sch.colId == TSDB_TBNAME_COLUMN_INDEX) { if (pQueryInfo->optr == TSDB_RELATION_IN) { addToResult = pQueryInfo->compare(name, pQueryInfo->q); } else if (pQueryInfo->optr == TSDB_RELATION_LIKE) { @@ -716,7 +716,7 @@ void tExprTreeTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, S } tQueryInfo *pQueryInfo = pExpr->_node.info; - if (pQueryInfo->colIndex == 0 && pQueryInfo->optr != TSDB_RELATION_LIKE) { + if (pQueryInfo->sch.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX && pQueryInfo->optr != TSDB_RELATION_LIKE) { tQueryIndexColumn(pSkipList, pQueryInfo, result); } else { tQueryIndexlessColumn(pSkipList, pQueryInfo, result, param->nodeFilterFn); diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index eb34805de4..8463e791db 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -240,7 +240,7 @@ TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STab pQueryHandle->defaultLoadColumn = getDefaultLoadColumns(pQueryHandle, true); - tsdbDebug("%p total numOfTable:%zu in query", pQueryHandle, taosArrayGetSize(pQueryHandle->pTableCheckInfo)); + tsdbDebug("%p total numOfTable:%zu in query, %p", pQueryHandle, taosArrayGetSize(pQueryHandle->pTableCheckInfo), pQueryHandle->qinfo); tsdbInitDataBlockLoadInfo(&pQueryHandle->dataBlockLoadInfo); tsdbInitCompBlockLoadInfo(&pQueryHandle->compBlockLoadInfo); @@ -1869,7 +1869,6 @@ int32_t tsdbRetrieveDataBlockStatisInfo(TsdbQueryHandleT* pQueryHandle, SDataSta pHandle->statis[i].numOfNull = pBlockInfo->compBlock->numOfRows; } - // todo opt perf SColumnInfo* pColInfo = taosArrayGet(pHandle->pColumns, i); if (pColInfo->type == TSDB_DATA_TYPE_TIMESTAMP) { pHandle->statis[i].min = pBlockInfo->compBlock->keyFirst; @@ -1961,43 +1960,20 @@ static void destroyHelper(void* param) { free(param); } -#define TAG_INVALID_COLUMN_INDEX -2 -static int32_t getTagColumnIndex(STSchema* pTSchema, SSchema* pSchema) { - // filter on table name(TBNAME) - if (strcasecmp(pSchema->name, TSQL_TBNAME_L) == 0) { - return TSDB_TBNAME_COLUMN_INDEX; - } - - for(int32_t i = 0; i < schemaNCols(pTSchema); ++i) { - STColumn* pColumn = &pTSchema->columns[i]; - if (pColumn->bytes == pSchema->bytes && pColumn->type == pSchema->type && pColumn->colId == pSchema->colId) { - return i; - } - } - - return -2; -} - void filterPrepare(void* expr, void* param) { tExprNode* pExpr = (tExprNode*)expr; if (pExpr->_node.info != NULL) { return; } - int32_t i = 0; pExpr->_node.info = calloc(1, sizeof(tQueryInfo)); - STSchema* pTSSchema = (STSchema*) param; - + STSchema* pTSSchema = (STSchema*) param; tQueryInfo* pInfo = pExpr->_node.info; tVariant* pCond = pExpr->_node.pRight->pVal; SSchema* pSchema = pExpr->_node.pLeft->pSchema; - int32_t index = getTagColumnIndex(pTSSchema, pSchema); - assert((index >= 0 && i < TSDB_MAX_TAGS) || (index == TSDB_TBNAME_COLUMN_INDEX) || index == TAG_INVALID_COLUMN_INDEX); - pInfo->sch = *pSchema; - pInfo->colIndex = index; pInfo->optr = pExpr->_node.optr; pInfo->compare = getComparFunc(pSchema->type, pInfo->optr); pInfo->param = pTSSchema; @@ -2143,7 +2119,7 @@ bool indexedNodeFilterFp(const void* pNode, void* param) { char* val = NULL; - if (pInfo->colIndex == TSDB_TBNAME_COLUMN_INDEX) { + if (pInfo->sch.colId == TSDB_TBNAME_COLUMN_INDEX) { val = (char*) TABLE_NAME(pTable); } else { val = tdGetKVRowValOfCol(pTable->tagVal, pInfo->sch.colId); From 246ea1df248e8721bad64f40fd57ed0ec0db263c Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sat, 18 Jul 2020 10:31:31 +0000 Subject: [PATCH 19/34] modify from ip to ep --- src/client/inc/tsclient.h | 4 +- src/client/src/tscSchemaUtil.c | 8 +-- src/client/src/tscServer.c | 95 +++++++++++++++------------------- src/client/src/tscSql.c | 2 +- src/client/src/tscUtil.c | 2 +- src/inc/trpc.h | 6 +-- 6 files changed, 53 insertions(+), 64 deletions(-) diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index 83f9165d35..b5455ed1fb 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -55,8 +55,8 @@ typedef struct STableComInfo { typedef struct SCMCorVgroupInfo { int32_t version; int8_t inUse; - int8_t numOfIps; - SIpAddr ipAddr[TSDB_MAX_REPLICA]; + int8_t numOfEps; + SEpAddr epAddr[TSDB_MAX_REPLICA]; } SCMCorVgroupInfo; typedef struct STableMeta { diff --git a/src/client/src/tscSchemaUtil.c b/src/client/src/tscSchemaUtil.c index 52342b3650..9b8f48b109 100644 --- a/src/client/src/tscSchemaUtil.c +++ b/src/client/src/tscSchemaUtil.c @@ -143,10 +143,10 @@ struct SSchema tscGetTbnameColumnSchema() { static void tscInitCorVgroupInfo(SCMCorVgroupInfo *corVgroupInfo, SCMVgroupInfo *vgroupInfo) { corVgroupInfo->version = 0; corVgroupInfo->inUse = 0; - corVgroupInfo->numOfIps = vgroupInfo->numOfIps; - for (int32_t i = 0; i < corVgroupInfo->numOfIps; i++) { - strncpy(corVgroupInfo->ipAddr[i].fqdn, vgroupInfo->ipAddr[i].fqdn, TSDB_FQDN_LEN); - corVgroupInfo->ipAddr[i].port = vgroupInfo->ipAddr[i].port; + corVgroupInfo->numOfEps = vgroupInfo->numOfEps; + for (int32_t i = 0; i < corVgroupInfo->numOfEps; i++) { + strncpy(corVgroupInfo->epAddr[i].fqdn, vgroupInfo->epAddr[i].fqdn, TSDB_FQDN_LEN); + corVgroupInfo->epAddr[i].port = vgroupInfo->epAddr[i].port; } } STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg, size_t* size) { diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 3d78063bed..d9922b8718 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -53,64 +53,53 @@ static void tscSetDnodeEpSet(SSqlObj* pSql, SCMVgroupInfo* pVgroupInfo) { return; } - pEpList->numOfEps = pVgroupInfo->numOfEps; + pEpSet->numOfEps = pVgroupInfo->numOfEps; for(int32_t i = 0; i < pVgroupInfo->numOfEps; ++i) { - strcpy(pEpList->fqdn[i], pVgroupInfo->epAddr[i].fqdn); - pEpList->port[i] = pVgroupInfo->epAddr[i].port; + strcpy(pEpSet->fqdn[i], pVgroupInfo->epAddr[i].fqdn); + pEpSet->port[i] = pVgroupInfo->epAddr[i].port; } } -void tscIpSetCopy(SRpcIpSet *dst, SRpcIpSet *src) { - dst->numOfIps = src->numOfIps; - dst->inUse = src->inUse; - for (int32_t i = 0; i < src->numOfIps; ++i) { - dst->port[i] = src->port[i]; - strncpy(dst->fqdn[i], src->fqdn[i], TSDB_FQDN_LEN); - } -} -static void tscDumpMgmtIpSet(SRpcIpSet *ipSet) { - taosCorBeginRead(&tscMgmtIpSet.version); - SRpcIpSet* src = &tscMgmtIpSet.ipSet; - tscIpSetCopy(ipSet, src); - taosCorEndRead(&tscMgmtIpSet.version); +static void tscDumpMgmtEpSet(SRpcEpSet *epSet) { + taosCorBeginRead(&tscMgmtEpSet.version); + *epSet = tscMgmtEpSet.epSet; + taosCorEndRead(&tscMgmtEpSet.version); } -static void tscIpSetHtons(SRpcIpSet *s) { - for (int32_t i = 0; i < s->numOfIps; i++) { +static void tscEpSetHtons(SRpcEpSet *s) { + for (int32_t i = 0; i < s->numOfEps; i++) { s->port[i] = htons(s->port[i]); } } -bool tscIpSetIsEqual(SRpcIpSet *s1, SRpcIpSet *s2) { - if (s1->numOfIps != s2->numOfIps || s1->inUse != s2->inUse) { +bool tscEpSetIsEqual(SRpcEpSet *s1, SRpcEpSet *s2) { + if (s1->numOfEps != s2->numOfEps || s1->inUse != s2->inUse) { return false; } - for (int32_t i = 0; i < s1->numOfIps; i++) { + for (int32_t i = 0; i < s1->numOfEps; i++) { if (s1->port[i] != s2->port[i] || strncmp(s1->fqdn[i], s2->fqdn[i], TSDB_FQDN_LEN) != 0) return false; } return true; } -void tscUpdateMgmtIpList(SRpcIpSet *pIpSet) { +void tscUpdateMgmtEpSet(SRpcEpSet *pEpSet) { // no need to update if equal - taosCorBeginWrite(&tscMgmtIpSet.version); - // or copy directly, tscMgmtIpSet.ipSet = *pIpSet - SRpcIpSet *mgmtIpSet = &tscMgmtIpSet.ipSet; - tscIpSetCopy(mgmtIpSet, pIpSet); - taosCorEndWrite(&tscMgmtIpSet.version); + taosCorBeginWrite(&tscMgmtEpSet.version); + tscMgmtEpSet.epSet = *pEpSet; + taosCorEndWrite(&tscMgmtEpSet.version); } -static void tscDumpIpSetFromVgroupInfo(SCMCorVgroupInfo *pVgroupInfo, SRpcIpSet *pIpSet) { +static void tscDumpEpSetFromVgroupInfo(SCMCorVgroupInfo *pVgroupInfo, SRpcEpSet *pEpSet) { if (pVgroupInfo == NULL) { return;} taosCorBeginRead(&pVgroupInfo->version); int8_t inUse = pVgroupInfo->inUse; - pIpSet->inUse = (inUse >= 0 && inUse < TSDB_MAX_REPLICA) ? inUse: 0; - pIpSet->numOfIps = pVgroupInfo->numOfIps; - for (int32_t i = 0; i < pVgroupInfo->numOfIps; ++i) { - strncpy(pIpSet->fqdn[i], pVgroupInfo->ipAddr[i].fqdn, TSDB_FQDN_LEN); - pIpSet->port[i] = pVgroupInfo->ipAddr[i].port; + pEpSet->inUse = (inUse >= 0 && inUse < TSDB_MAX_REPLICA) ? inUse: 0; + pEpSet->numOfEps = pVgroupInfo->numOfEps; + for (int32_t i = 0; i < pVgroupInfo->numOfEps; ++i) { + strncpy(pEpSet->fqdn[i], pVgroupInfo->epAddr[i].fqdn, TSDB_FQDN_LEN); + pEpSet->port[i] = pVgroupInfo->epAddr[i].port; } taosCorEndRead(&pVgroupInfo->version); } -static void tscUpdateVgroupInfo(SSqlObj *pObj, SRpcIpSet *pIpSet) { +static void tscUpdateVgroupInfo(SSqlObj *pObj, SRpcEpSet *pEpSet) { SSqlCmd *pCmd = &pObj->cmd; STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); if (pTableMetaInfo == NULL || pTableMetaInfo->pTableMeta == NULL) { return;} @@ -130,7 +119,7 @@ void tscPrintMgmtEp() { SRpcEpSet dump; tscDumpMgmtEpSet(&dump); if (dump.numOfEps <= 0) { - tscError("invalid mnode EP list:%d", dump.numOfEPs); + tscError("invalid mnode EP list:%d", dump.numOfEps); } else { for (int i = 0; i < dump.numOfEps; ++i) { tscDebug("mnode index:%d %s:%d", i, dump.fqdn[i], dump.port[i]); @@ -166,10 +155,10 @@ void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) { if (code == 0) { SCMHeartBeatRsp *pRsp = (SCMHeartBeatRsp *)pRes->pRsp; - SRpcEpSet * pEpList = &pRsp->epList; - if (pEpList->numOfEps > 0) { - tscEpSetHtons(pEpList); - tscUpdateMgmtEpList(pEpList); + SRpcEpSet * epSet = &pRsp->epSet; + if (epSet->numOfEps > 0) { + tscEpSetHtons(epSet); + tscUpdateMgmtEpSet(epSet); } pSql->pTscObj->connId = htonl(pRsp->connId); @@ -242,7 +231,7 @@ int tscSendMsgToServer(SSqlObj *pSql) { // set the mgmt ip list if (pSql->cmd.command >= TSDB_SQL_MGMT) { - tscDumpMgmtEpSet(&pSql->epList); + tscDumpMgmtEpSet(&pSql->epSet); } memcpy(pMsg, pSql->cmd.payload, pSql->cmd.payloadLen); @@ -296,11 +285,11 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) { if (pEpSet) { //SRpcEpSet dump; tscEpSetHtons(pEpSet); - if (tscEpSetIsEqual(&pSql->epList, pEpSet)) { + if (tscEpSetIsEqual(&pSql->epSet, pEpSet)) { if(pCmd->command < TSDB_SQL_MGMT) { tscUpdateVgroupInfo(pSql, pEpSet); } else { - tscUpdateMgmtEpList(pEpSet); + tscUpdateMgmtEpSet(pEpSet); } } } @@ -589,7 +578,7 @@ int tscBuildSubmitMsg(SSqlObj *pSql, SSqlInfo *pInfo) { // pSql->cmd.payloadLen is set during copying data into payload pSql->cmd.msgType = TSDB_MSG_TYPE_SUBMIT; - tscDumpEpSetFromVgroupInfo(&pTableMeta->corVgroupInfo, &pSql->epList); + tscDumpEpSetFromVgroupInfo(&pTableMeta->corVgroupInfo, &pSql->epSet); tscDebug("%p build submit msg, vgId:%d numOfTables:%d numberOfEP:%d", pSql, vgId, pSql->cmd.numOfTablesInSubmit, pSql->epSet.numOfEps); @@ -631,7 +620,7 @@ static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char } else { pVgroupInfo = &pTableMeta->vgroupInfo; } - tscSetDnodeEpList(pSql, pVgroupInfo); + tscSetDnodeEpSet(pSql, pVgroupInfo); if (pVgroupInfo != NULL) { pQueryMsg->head.vgId = htonl(pVgroupInfo->vgId); @@ -654,7 +643,7 @@ static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char SVgroupTableInfo* pTableIdList = taosArrayGet(pTableMetaInfo->pVgroupTables, index); // set the vgroup info - tscSetDnodeEpList(pSql, &pTableIdList->vgInfo); + tscSetDnodeEpSet(pSql, &pTableIdList->vgInfo); pQueryMsg->head.vgId = htonl(pTableIdList->vgInfo.vgId); int32_t numOfTables = taosArrayGetSize(pTableIdList->itemList); @@ -1200,11 +1189,11 @@ int32_t tscBuildShowMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pShowMsg->payloadLen = htons(pPattern->n); } } else { - SSQLToken *pIpAddr = &pShowInfo->prefix; - assert(pIpAddr->n > 0 && pIpAddr->type > 0); + SSQLToken *pEpAddr = &pShowInfo->prefix; + assert(pEpAddr->n > 0 && pEpAddr->type > 0); - strncpy(pShowMsg->payload, pIpAddr->z, pIpAddr->n); - pShowMsg->payloadLen = htons(pIpAddr->n); + strncpy(pShowMsg->payload, pEpAddr->z, pEpAddr->n); + pShowMsg->payloadLen = htons(pEpAddr->n); } pCmd->payloadLen = sizeof(SCMShowMsg) + pShowMsg->payloadLen; @@ -1387,7 +1376,7 @@ int tscBuildUpdateTagMsg(SSqlObj* pSql, SSqlInfo *pInfo) { SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - tscDumpEpSetFromVgroupInfo(&pTableMetaInfo->pTableMeta->corVgroupInfo, &pSql->epList); + tscDumpEpSetFromVgroupInfo(&pTableMetaInfo->pTableMeta->corVgroupInfo, &pSql->epSet); return TSDB_CODE_SUCCESS; } @@ -2011,9 +2000,9 @@ int tscProcessConnectRsp(SSqlObj *pSql) { assert(len <= sizeof(pObj->db)); tstrncpy(pObj->db, temp, sizeof(pObj->db)); - if (pConnect->epList.numOfEps > 0) { - tscEpSetHtons(&pConnect->epList); - tscUpdateMgmtEpList(&pConnect->epList); + if (pConnect->epSet.numOfEps > 0) { + tscEpSetHtons(&pConnect->epSet); + tscUpdateMgmtEpSet(&pConnect->epSet); } strcpy(pObj->sversion, pConnect->serverVersion); diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index 59b506a454..5848b7b82f 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -62,7 +62,7 @@ SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pass, con } if (ip) { - if (tscSetMgmtEpListFromCfg(ip, NULL) < 0) return NULL; + if (tscSetMgmtEpSetFromCfg(ip, NULL) < 0) return NULL; if (port) tscMgmtEpSet.epSet.port[0] = port; } diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index c3cae58d8a..957bdeeb7f 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -2145,7 +2145,7 @@ char* strdup_throw(const char* str) { return p; } -int tscSetMgmtEpListFromCfg(const char *first, const char *second) { +int tscSetMgmtEpSetFromCfg(const char *first, const char *second) { // init mgmt ip set tscMgmtEpSet.version = 0; SRpcEpSet *mgmtEpSet = &(tscMgmtEpSet.epSet); diff --git a/src/inc/trpc.h b/src/inc/trpc.h index 41ed98fd70..bdee917b5e 100644 --- a/src/inc/trpc.h +++ b/src/inc/trpc.h @@ -35,10 +35,10 @@ typedef struct SRpcEpSet { char fqdn[TSDB_MAX_REPLICA][TSDB_FQDN_LEN]; } SRpcEpSet; -typedef struct SRpcCorIpSet { +typedef struct SRpcCorEpSet { int32_t version; - SRpcIpSet ipSet; -} SRpcCorIpSet; + SRpcEpSet epSet; +} SRpcCorEpSet; typedef struct SRpcConnInfo { uint32_t clientIp; From b40c7018e3fe9b0b0525c4b2fdbb587179e89c5f Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Mon, 20 Jul 2020 09:28:13 +0800 Subject: [PATCH 20/34] print messages in red on error --- tests/examples/c/apitest.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/examples/c/apitest.c b/tests/examples/c/apitest.c index bc628fb58f..759e16d1de 100644 --- a/tests/examples/c/apitest.c +++ b/tests/examples/c/apitest.c @@ -218,9 +218,9 @@ void verify_prepare(TAOS* taos) { int code = taos_errno(result); if (code != 0) { - printf("failed to create database, reason:%s\n", taos_errstr(result)); + printf("\033[31mfailed to create database, reason:%s\033[0m\n", taos_errstr(result)); taos_free_result(result); - exit(1); + return; } taos_free_result(result); @@ -232,9 +232,9 @@ void verify_prepare(TAOS* taos) { result = taos_query(taos, sql); code = taos_errno(result); if (code != 0) { - printf("failed to create table, reason:%s\n", taos_errstr(result)); + printf("\033[31mfailed to create table, reason:%s\033[0m\n", taos_errstr(result)); taos_free_result(result); - exit(1); + return; } taos_free_result(result); @@ -320,7 +320,7 @@ void verify_prepare(TAOS* taos) { sql = "insert into m1 values(?,?,?,?,?,?,?,?,?,?)"; code = taos_stmt_prepare(stmt, sql, 0); if (code != 0){ - printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); + printf("\033[31mfailed to execute taos_stmt_prepare. code:0x%x\033[0m\n", code); } v.ts = 1591060628000; for (int i = 0; i < 10; ++i) { @@ -343,8 +343,8 @@ void verify_prepare(TAOS* taos) { taos_stmt_add_batch(stmt); } if (taos_stmt_execute(stmt) != 0) { - printf("failed to execute insert statement.\n"); - exit(1); + printf("\033[31mfailed to execute insert statement.\033[0m\n"); + return; } taos_stmt_close(stmt); @@ -355,8 +355,8 @@ void verify_prepare(TAOS* taos) { v.v2 = 15; taos_stmt_bind_param(stmt, params + 2); if (taos_stmt_execute(stmt) != 0) { - printf("failed to execute select statement.\n"); - exit(1); + printf("\033[31mfailed to execute select statement.\033[0m\n"); + return; } result = taos_stmt_use_result(stmt); @@ -444,7 +444,7 @@ int main(int argc, char *argv[]) { TAOS* taos = taos_connect(host, user, passwd, "", 0); if (taos == NULL) { - printf("failed to connect to db, reason:%s\n", taos_errstr(taos)); + printf("\033[31mfailed to connect to db, reason:%s\033[0m\n", taos_errstr(taos)); exit(1); } From 1067175746851366b658b0b2b590f77ff915642f Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Mon, 20 Jul 2020 11:15:32 +0800 Subject: [PATCH 21/34] fix compiler error --- src/client/src/tscSub.c | 3 ++- tests/examples/c/makefile | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/client/src/tscSub.c b/src/client/src/tscSub.c index 73614fa0a5..526f380475 100644 --- a/src/client/src/tscSub.c +++ b/src/client/src/tscSub.c @@ -92,6 +92,7 @@ static void asyncCallback(void *param, TAOS_RES *tres, int code) { static SSub* tscCreateSubscription(STscObj* pObj, const char* topic, const char* sql) { int code = TSDB_CODE_SUCCESS, line = __LINE__; + SSqlObj* pSql = NULL; SSub* pSub = calloc(1, sizeof(SSub)); if (pSub == NULL) { @@ -113,7 +114,7 @@ static SSub* tscCreateSubscription(STscObj* pObj, const char* topic, const char* goto fail; } - SSqlObj* pSql = calloc(1, sizeof(SSqlObj)); + pSql = calloc(1, sizeof(SSqlObj)); if (pSql == NULL) { line = __LINE__; code = TSDB_CODE_TSC_OUT_OF_MEMORY; diff --git a/tests/examples/c/makefile b/tests/examples/c/makefile index 0a4b8ee9d2..f9653c9c96 100644 --- a/tests/examples/c/makefile +++ b/tests/examples/c/makefile @@ -4,7 +4,6 @@ ROOT=./ TARGET=exe LFLAGS = '-Wl,-rpath,/usr/local/taos/driver/' -ltaos -lpthread -lm -lrt -#LFLAGS = '-Wl,-rpath,/home/zbm/project/td/debug/build/lib/' -L/home/zbm/project/td/debug/build/lib -ltaos -lpthread -lm -lrt CFLAGS = -O3 -g -Wall -Wno-deprecated -fPIC -Wno-unused-result -Wconversion -Wno-char-subscripts -D_REENTRANT -Wno-format -D_REENTRANT -DLINUX -msse4.2 -Wno-unused-function -D_M_X64 \ -I/usr/local/taos/include -std=gnu99 @@ -16,6 +15,7 @@ exe: gcc $(CFLAGS) ./prepare.c -o $(ROOT)/prepare $(LFLAGS) gcc $(CFLAGS) ./stream.c -o $(ROOT)/stream $(LFLAGS) gcc $(CFLAGS) ./subscribe.c -o $(ROOT)subscribe $(LFLAGS) + gcc $(CFLAGS) ./apitest.c -o $(ROOT)apitest $(LFLAGS) clean: rm $(ROOT)/asyncdemo From caa5442005f28a0e4c22b538ff64bfc6ffa124a2 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sun, 19 Jul 2020 12:27:45 +0000 Subject: [PATCH 22/34] src/inc/taos.h --- src/inc/taos.h | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/inc/taos.h b/src/inc/taos.h index d6f1883572..1d609bc7db 100644 --- a/src/inc/taos.h +++ b/src/inc/taos.h @@ -67,8 +67,6 @@ DLL_EXPORT void taos_init(); DLL_EXPORT void taos_cleanup(); DLL_EXPORT int taos_options(TSDB_OPTION option, const void *arg, ...); DLL_EXPORT TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port); -DLL_EXPORT TAOS *taos_connect_c(const char *ip, uint8_t ipLen, const char *user, uint8_t userLen, - const char *pass, uint8_t passLen, const char *db, uint8_t dbLen, uint16_t port); DLL_EXPORT void taos_close(TAOS *taos); typedef struct TAOS_BIND { @@ -90,7 +88,6 @@ TAOS_RES * taos_stmt_use_result(TAOS_STMT *stmt); int taos_stmt_close(TAOS_STMT *stmt); DLL_EXPORT TAOS_RES *taos_query(TAOS *taos, const char *sql); -DLL_EXPORT TAOS_RES *taos_query_c(TAOS *taos, const char *sql, uint32_t sqlLen); DLL_EXPORT TAOS_ROW taos_fetch_row(TAOS_RES *res); DLL_EXPORT int taos_result_precision(TAOS_RES *res); // get the time precision of result DLL_EXPORT void taos_free_result(TAOS_RES *res); From 55e3352d7be4c3739bc6355f41c143bef0c37d33 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 19 Jul 2020 23:29:17 +0800 Subject: [PATCH 23/34] [td-225] refactor codes. opt resbuf perf. rename sources file --- src/client/inc/tsclient.h | 2 +- src/client/src/tscFunctionImpl.c | 8 +- src/client/src/tscServer.c | 63 ++--- src/query/inc/{qExecutor.h => qexecutor.h} | 6 +- src/query/inc/{qresultBuf.h => qresultbuf.h} | 24 +- src/query/inc/{qUtil.h => qutil.h} | 4 +- src/query/inc/tsqlfunction.h | 10 +- src/query/src/{qExecutor.c => qexecutor.c} | 132 +++++++---- .../src/{qFilterFunc.c => qfilterfunc.c} | 4 +- src/query/src/qresultBuf.c | 184 --------------- src/query/src/qresultbuf.c | 218 ++++++++++++++++++ src/query/src/{qUtil.c => qutil.c} | 8 +- src/query/tests/resultBufferTest.cpp | 2 +- src/tsdb/src/tsdbRead.c | 2 +- src/util/src/tutil.c | 2 +- 15 files changed, 365 insertions(+), 304 deletions(-) rename src/query/inc/{qExecutor.h => qexecutor.h} (99%) rename src/query/inc/{qresultBuf.h => qresultbuf.h} (75%) rename src/query/inc/{qUtil.h => qutil.h} (95%) rename src/query/src/{qExecutor.c => qexecutor.c} (98%) rename src/query/src/{qFilterFunc.c => qfilterfunc.c} (99%) delete mode 100644 src/query/src/qresultBuf.c create mode 100644 src/query/src/qresultbuf.c rename src/query/src/{qUtil.c => qutil.c} (98%) diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index ef8184990d..3f548c18ca 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -30,7 +30,7 @@ extern "C" { #include "tsqlfunction.h" #include "tutil.h" -#include "qExecutor.h" +#include "qexecutor.h" #include "qsqlparser.h" #include "qtsbuf.h" #include "tcmdtype.h" diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c index 262b7ab3f6..fb15e4efd5 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/client/src/tscFunctionImpl.c @@ -74,7 +74,7 @@ for (int32_t i = 0; i < (ctx)->tagInfo.numOfTagCols; ++i) { \ void noop1(SQLFunctionCtx *UNUSED_PARAM(pCtx)) {} void noop2(SQLFunctionCtx *UNUSED_PARAM(pCtx), int32_t UNUSED_PARAM(index)) {} -void doFinalizer(SQLFunctionCtx *pCtx) { resetResultInfo(GET_RES_INFO(pCtx)); } +void doFinalizer(SQLFunctionCtx *pCtx) { RESET_RESULT_INFO(GET_RES_INFO(pCtx)); } typedef struct tValuePair { tVariant v; @@ -330,12 +330,6 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI return TSDB_CODE_SUCCESS; } -/** - * the numOfRes should be kept, since it may be used later - * and allow the ResultInfo to be re initialized - */ -void resetResultInfo(SResultInfo *pResInfo) { pResInfo->initialized = false; } - void setResultInfoBuf(SResultInfo *pResInfo, int32_t size, bool superTable, char* buf) { assert(pResInfo->interResultBuf == NULL); diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 398dea9d09..2bc41a834d 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -242,52 +242,31 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) { if (pEpSet) tscMgmtEpSet = *pEpSet; } - if (rpcMsg->pCont == NULL) { - rpcMsg->code = TSDB_CODE_RPC_NETWORK_UNAVAIL; - } else { - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); - // if (rpcMsg->code != TSDB_CODE_RPC_NETWORK_UNAVAIL) { - // if (pCmd->command == TSDB_SQL_CONNECT) { - // rpcMsg->code = TSDB_CODE_RPC_NETWORK_UNAVAIL; - // rpcFreeCont(rpcMsg->pCont); - // return; - // } + STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); - // if (pCmd->command == TSDB_SQL_HB) { - // rpcMsg->code = TSDB_CODE_RPC_NOT_READY; - // rpcFreeCont(rpcMsg->pCont); - // return; - // } + int32_t cmd = pCmd->command; + if ((cmd == TSDB_SQL_SELECT || cmd == TSDB_SQL_FETCH || cmd == TSDB_SQL_INSERT || cmd == TSDB_SQL_UPDATE_TAGS_VAL) && + (rpcMsg->code == TSDB_CODE_TDB_INVALID_TABLE_ID || + rpcMsg->code == TSDB_CODE_VND_INVALID_VGROUP_ID || + rpcMsg->code == TSDB_CODE_RPC_NETWORK_UNAVAIL || + rpcMsg->code == TSDB_CODE_TDB_TABLE_RECONFIGURE)) { + tscWarn("%p it shall renew table meta, code:%s, retry:%d", pSql, tstrerror(rpcMsg->code), ++pSql->retry); - // if (pCmd->command == TSDB_SQL_META || pCmd->command == TSDB_SQL_DESCRIBE_TABLE || - // pCmd->command == TSDB_SQL_STABLEVGROUP || pCmd->command == TSDB_SQL_SHOW || - // pCmd->command == TSDB_SQL_RETRIEVE) { - // // get table meta/vgroup query will not retry, do nothing - // } - // } + // set the flag to denote that sql string needs to be re-parsed and build submit block with table schema + if (rpcMsg->code == TSDB_CODE_TDB_TABLE_RECONFIGURE) { + pSql->cmd.submitSchema = 1; + } - if ((pCmd->command == TSDB_SQL_SELECT || pCmd->command == TSDB_SQL_FETCH || pCmd->command == TSDB_SQL_INSERT || - pCmd->command == TSDB_SQL_UPDATE_TAGS_VAL) && - (rpcMsg->code == TSDB_CODE_TDB_INVALID_TABLE_ID || rpcMsg->code == TSDB_CODE_VND_INVALID_VGROUP_ID || - rpcMsg->code == TSDB_CODE_RPC_NETWORK_UNAVAIL || rpcMsg->code == TSDB_CODE_TDB_TABLE_RECONFIGURE)) { - tscWarn("%p it shall renew table meta, code:%s, retry:%d", pSql, tstrerror(rpcMsg->code), ++pSql->retry); - // set the flag to denote that sql string needs to be re-parsed and build submit block with table schema - if (rpcMsg->code == TSDB_CODE_TDB_TABLE_RECONFIGURE) { - pSql->cmd.submitSchema = 1; - } + pSql->res.code = rpcMsg->code; // keep the previous error code + if (pSql->retry > pSql->maxRetry) { + tscError("%p max retry %d reached, give up", pSql, pSql->maxRetry); + } else { + rpcMsg->code = tscRenewTableMeta(pSql, pTableMetaInfo->name); - pSql->res.code = rpcMsg->code; // keep the previous error code - if (pSql->retry > pSql->maxRetry) { - tscError("%p max retry %d reached, give up", pSql, pSql->maxRetry); - } else { - rpcMsg->code = tscRenewTableMeta(pSql, pTableMetaInfo->name); - - // if there is an error occurring, proceed to the following error handling procedure. - // todo add test cases - if (rpcMsg->code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { - rpcFreeCont(rpcMsg->pCont); - return; - } + // if there is an error occurring, proceed to the following error handling procedure. + if (rpcMsg->code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { + rpcFreeCont(rpcMsg->pCont); + return; } } } diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qexecutor.h similarity index 99% rename from src/query/inc/qExecutor.h rename to src/query/inc/qexecutor.h index fb8750323f..97a7565de8 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qexecutor.h @@ -19,15 +19,15 @@ #include "hash.h" #include "qfill.h" -#include "qresultBuf.h" +#include "qresultbuf.h" #include "qsqlparser.h" #include "qtsbuf.h" +#include "query.h" #include "taosdef.h" #include "tarray.h" #include "tlockfree.h" #include "tsdb.h" #include "tsqlfunction.h" -#include "query.h" struct SColumnFilterElem; typedef bool (*__filter_func_t)(struct SColumnFilterElem* pFilter, char* val1, char* val2); @@ -158,7 +158,7 @@ typedef struct SQueryRuntimeEnv { SResultInfo* resultInfo; // todo refactor to merge with SWindowResInfo SQuery* pQuery; SQLFunctionCtx* pCtx; - int16_t numOfRowsPerPage; + int32_t numOfRowsPerPage; int16_t offset[TSDB_MAX_COLUMNS]; uint16_t scanFlag; // denotes reversed scan of data or not SFillInfo* pFillInfo; diff --git a/src/query/inc/qresultBuf.h b/src/query/inc/qresultbuf.h similarity index 75% rename from src/query/inc/qresultBuf.h rename to src/query/inc/qresultbuf.h index a323d530a2..f901474cc1 100644 --- a/src/query/inc/qresultBuf.h +++ b/src/query/inc/qresultbuf.h @@ -33,14 +33,19 @@ typedef struct SDiskbasedResultBuf { int32_t fd; // data file fd int32_t allocateId; // allocated page id int32_t incStep; // minimum allocated pages - char* pBuf; // mmap buffer pointer + void* pBuf; // mmap buffer pointer char* path; // file path - + int32_t pageSize; // current used page size + int32_t inMemPages; // numOfPages that are allocated in memory SHashObj* idsTable; // id hash table SIDList list; // for each id, there is a page id list + + void* iBuf; // inmemory buf + void* handle; // for debug purpose } SDiskbasedResultBuf; -#define DEFAULT_INTERN_BUF_PAGE_SIZE (8192L*5) +#define DEFAULT_INTERN_BUF_PAGE_SIZE (1024L) +#define DEFAULT_INMEM_BUF_PAGES 10 /** * create disk-based result buffer @@ -49,7 +54,8 @@ typedef struct SDiskbasedResultBuf { * @param rowSize * @return */ -int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t size, int32_t rowSize, void* handle); +int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t numOfPages, int32_t rowSize, int32_t pagesize, + int32_t inMemPages, void* handle); /** * @@ -81,8 +87,14 @@ SIDList getDataBufPagesIdList(SDiskbasedResultBuf* pResultBuf, int32_t groupId); * @param id * @return */ -#define GET_RES_BUF_PAGE_BY_ID(buf, id) ((tFilePage*)((buf)->pBuf + DEFAULT_INTERN_BUF_PAGE_SIZE*(id))) - +//#define getResBufPage(buf, id) ((tFilePage*)((buf)->pBuf + (buf)->pageSize * (id))) +static FORCE_INLINE tFilePage* getResBufPage(SDiskbasedResultBuf* pResultBuf, int32_t id) { + if (id < pResultBuf->inMemPages) { + return pResultBuf->iBuf + id * pResultBuf->pageSize; + } else { + return pResultBuf->pBuf + (id - pResultBuf->inMemPages) * pResultBuf->pageSize; + } +} /** * get the total buffer size in the format of disk file * @param pResultBuf diff --git a/src/query/inc/qUtil.h b/src/query/inc/qutil.h similarity index 95% rename from src/query/inc/qUtil.h rename to src/query/inc/qutil.h index 78ae7be030..7119cb75fe 100644 --- a/src/query/inc/qUtil.h +++ b/src/query/inc/qutil.h @@ -49,7 +49,7 @@ static FORCE_INLINE char *getPosInResultPage(SQueryRuntimeEnv *pRuntimeEnv, int3 assert(pResult != NULL && pRuntimeEnv != NULL); SQuery *pQuery = pRuntimeEnv->pQuery; - tFilePage *page = GET_RES_BUF_PAGE_BY_ID(pRuntimeEnv->pResultBuf, pResult->pos.pageId); + tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pResult->pos.pageId); int32_t realRowId = pResult->pos.rowId * GET_ROW_PARAM_FOR_MULTIOUTPUT(pQuery, pRuntimeEnv->topBotQuery, pRuntimeEnv->stableQuery); return ((char *)page->data) + pRuntimeEnv->offset[columnIndex] * pRuntimeEnv->numOfRowsPerPage + @@ -59,6 +59,4 @@ static FORCE_INLINE char *getPosInResultPage(SQueryRuntimeEnv *pRuntimeEnv, int3 __filter_func_t *getRangeFilterFuncArray(int32_t type); __filter_func_t *getValueFilterFuncArray(int32_t type); -bool supportPrefilter(int32_t type); - #endif // TDENGINE_QUERYUTIL_H diff --git a/src/query/inc/tsqlfunction.h b/src/query/inc/tsqlfunction.h index c687f01cbc..5ce9121cf1 100644 --- a/src/query/inc/tsqlfunction.h +++ b/src/query/inc/tsqlfunction.h @@ -255,7 +255,15 @@ extern int32_t functionCompatList[]; // compatible check array list bool topbot_datablock_filter(SQLFunctionCtx *pCtx, int32_t functionId, const char *minval, const char *maxval); -void resetResultInfo(SResultInfo *pResInfo); +/** + * the numOfRes should be kept, since it may be used later + * and allow the ResultInfo to be re initialized + */ +#define RESET_RESULT_INFO(_r) \ + do { \ + (_r)->initialized = false; \ + } while (0) + void setResultInfoBuf(SResultInfo *pResInfo, int32_t size, bool superTable, char* buf); static FORCE_INLINE void initResultInfo(SResultInfo *pResInfo) { diff --git a/src/query/src/qExecutor.c b/src/query/src/qexecutor.c similarity index 98% rename from src/query/src/qExecutor.c rename to src/query/src/qexecutor.c index 42fc90f478..9348f702d0 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qexecutor.c @@ -20,12 +20,12 @@ #include "exception.h" #include "hash.h" -#include "qExecutor.h" -#include "qUtil.h" -#include "qresultBuf.h" +#include "qast.h" +#include "qexecutor.h" +#include "qresultbuf.h" #include "query.h" #include "queryLog.h" -#include "qast.h" +#include "qutil.h" #include "tlosertree.h" #include "tscompression.h" #include "ttime.h" @@ -233,9 +233,7 @@ bool isGroupbyNormalCol(SSqlGroupbyExpr *pGroupbyExpr) { for (int32_t i = 0; i < pGroupbyExpr->numOfGroupCols; ++i) { SColIndex *pColIndex = taosArrayGet(pGroupbyExpr->columnInfo, i); if (pColIndex->flag == TSDB_COL_NORMAL) { - /* - * make sure the normal column locates at the second position if tbname exists in group by clause - */ + //make sure the normal column locates at the second position if tbname exists in group by clause if (pGroupbyExpr->numOfGroupCols > 1) { assert(pColIndex->colIndex > 0); } @@ -294,6 +292,17 @@ bool isSelectivityWithTagsQuery(SQuery *pQuery) { return false; } +bool isProjQuery(SQuery *pQuery) { + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { + int32_t functId = pQuery->pSelectExpr[i].base.functionId; + if (functId != TSDB_FUNC_PRJ && functId != TSDB_FUNC_TAGPRJ) { + return false; + } + } + + return true; +} + bool isTSCompQuery(SQuery *pQuery) { return pQuery->pSelectExpr[0].base.functionId == TSDB_FUNC_TS_COMP; } static bool limitResults(SQueryRuntimeEnv* pRuntimeEnv) { @@ -470,7 +479,7 @@ static int32_t addNewWindowResultBuf(SWindowResult *pWindowRes, SDiskbasedResult pData = getNewDataBuf(pResultBuf, sid, &pageId); } else { pageId = getLastPageId(list); - pData = GET_RES_BUF_PAGE_BY_ID(pResultBuf, pageId); + pData = getResBufPage(pResultBuf, pageId); if (pData->num >= numOfRowsPerPage) { pData = getNewDataBuf(pResultBuf, sid, &pageId); @@ -1003,7 +1012,6 @@ static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, char *pDat case TSDB_DATA_TYPE_BIGINT: v = GET_INT64_VAL(pData); break; } -// assert(pRuntimeEnv->windowResInfo.hashList->size <= 2); SWindowResult *pWindowRes = doSetTimeWindowFromKey(pRuntimeEnv, &pRuntimeEnv->windowResInfo, pData, bytes, true); if (pWindowRes == NULL) { return -1; @@ -1201,9 +1209,8 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS continue; } - // interval window query + // interval window query, decide the time window according to the primary timestamp if (QUERY_IS_INTERVAL_QUERY(pQuery)) { - // decide the time window according to the primary timestamp int64_t ts = tsCols[offset]; STimeWindow win = getActiveTimeWindow(pWindowResInfo, ts, pQuery); @@ -1225,8 +1232,7 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS while (1) { GET_NEXT_TIMEWINDOW(pQuery, &nextWin); - if (/*pWindowResInfo->startTime > nextWin.skey ||*/ - (nextWin.skey > pQuery->window.ekey && QUERY_IS_ASC_QUERY(pQuery)) || + if ((nextWin.skey > pQuery->window.ekey && QUERY_IS_ASC_QUERY(pQuery)) || (nextWin.skey < pQuery->window.ekey && !QUERY_IS_ASC_QUERY(pQuery))) { break; } @@ -1484,6 +1490,8 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order goto _clean; } + qDebug("QInfo:%p setup runtime env1", GET_QINFO_ADDR(pRuntimeEnv)); + pRuntimeEnv->offset[0] = 0; for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { SSqlFuncMsg *pSqlFuncMsg = &pQuery->pSelectExpr[i].base; @@ -1528,6 +1536,8 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order } } + qDebug("QInfo:%p setup runtime env2", GET_QINFO_ADDR(pRuntimeEnv)); + // set the order information for top/bottom query int32_t functionId = pCtx->functionId; @@ -1548,17 +1558,25 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order } } + qDebug("QInfo:%p setup runtime env3", GET_QINFO_ADDR(pRuntimeEnv)); + char* buf = (char*) pRuntimeEnv->resultInfo + sizeof(SResultInfo) * pQuery->numOfOutput; // set the intermediate result output buffer setWindowResultInfo(pRuntimeEnv->resultInfo, pQuery, pRuntimeEnv->stableQuery, buf); + qDebug("QInfo:%p setup runtime env4", GET_QINFO_ADDR(pRuntimeEnv)); + // if it is group by normal column, do not set output buffer, the output buffer is pResult - if (!isGroupbyNormalCol(pQuery->pGroupbyExpr) && !pRuntimeEnv->stableQuery) { + if (!pRuntimeEnv->groupbyNormalCol && !pRuntimeEnv->stableQuery) { resetCtxOutputBuf(pRuntimeEnv); } + qDebug("QInfo:%p setup runtime env5", GET_QINFO_ADDR(pRuntimeEnv)); + setCtxTagColumnInfo(pRuntimeEnv, pRuntimeEnv->pCtx); + + qDebug("QInfo:%p init completed", GET_QINFO_ADDR(pRuntimeEnv)); return TSDB_CODE_SUCCESS; _clean: @@ -1910,9 +1928,20 @@ static int32_t getInitialPageNum(SQInfo *pQInfo) { return num; } -static FORCE_INLINE int32_t getNumOfRowsInResultPage(SQuery *pQuery, bool topBotQuery, bool isSTableQuery) { - int32_t rowSize = pQuery->rowSize * GET_ROW_PARAM_FOR_MULTIOUTPUT(pQuery, topBotQuery, isSTableQuery); - return (DEFAULT_INTERN_BUF_PAGE_SIZE - sizeof(tFilePage)) / rowSize; +static void getIntermediateBufInfo(SQueryRuntimeEnv* pRuntimeEnv, int32_t* ps, int32_t* rowsize) { + SQuery* pQuery = pRuntimeEnv->pQuery; + + *rowsize = pQuery->rowSize * GET_ROW_PARAM_FOR_MULTIOUTPUT(pQuery, pRuntimeEnv->topBotQuery, pRuntimeEnv->stableQuery); + int32_t overhead = sizeof(tFilePage); + + // one page contains at least two rows + *ps = DEFAULT_INTERN_BUF_PAGE_SIZE; + while(((*rowsize) * 2) > (*ps) - overhead) { + *ps = (*ps << 1u); + } + + pRuntimeEnv->numOfRowsPerPage = ((*ps) - sizeof(tFilePage)) / (*rowsize); + } #define IS_PREFILTER_TYPE(_t) ((_t) != TSDB_DATA_TYPE_BINARY && (_t) != TSDB_DATA_TYPE_NCHAR) @@ -2038,8 +2067,7 @@ int32_t loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, void* pQueryHandle, } else { // check if this data block is required to load // Calculate all time windows that are overlapping or contain current data block. - // If current data block is contained by all possible time window, loading current - // data block is not needed. + // If current data block is contained by all possible time window, do not load current data block. if (QUERY_IS_INTERVAL_QUERY(pQuery) && overlapWithTimeWindow(pQuery, pBlockInfo)) { status = BLK_DATA_ALL_NEEDED; } @@ -2424,7 +2452,7 @@ static void doMerge(SQueryRuntimeEnv *pRuntimeEnv, int64_t timestamp, SWindowRes pCtx[i].aOutputBuf = pCtx[i].aOutputBuf + pCtx[i].outputBytes; pCtx[i].currentStage = FIRST_STAGE_MERGE; - resetResultInfo(pCtx[i].resultInfo); + RESET_RESULT_INFO(pCtx[i].resultInfo); aAggs[functionId].init(&pCtx[i]); } @@ -2661,7 +2689,7 @@ void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) { int32_t size = taosArrayGetSize(list); for (int32_t i = 0; i < size; ++i) { int32_t* pgId = taosArrayGet(list, i); - tFilePage *pData = GET_RES_BUF_PAGE_BY_ID(pResultBuf, *pgId); + tFilePage *pData = getResBufPage(pResultBuf, *pgId); total += pData->num; } @@ -2670,7 +2698,7 @@ void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) { int32_t offset = 0; for (int32_t j = 0; j < size; ++j) { int32_t* pgId = taosArrayGet(list, j); - tFilePage *pData = GET_RES_BUF_PAGE_BY_ID(pResultBuf, *pgId); + tFilePage *pData = getResBufPage(pResultBuf, *pgId); for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { int32_t bytes = pRuntimeEnv->pCtx[i].outputBytes; @@ -2860,10 +2888,10 @@ int32_t flushFromResultBuf(SQInfo *pQInfo) { SQuery * pQuery = pRuntimeEnv->pQuery; SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf; - int32_t capacity = (DEFAULT_INTERN_BUF_PAGE_SIZE - sizeof(tFilePage)) / pQuery->rowSize; // the base value for group result, since the maximum number of table for each vnode will not exceed 100,000. int32_t pageId = -1; + int32_t capacity = pResultBuf->numOfRowsPerPage; int32_t remain = pQuery->sdata[0]->num; int32_t offset = 0; @@ -3033,7 +3061,7 @@ void resetCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv) { * set the output buffer information and intermediate buffer * not all queries require the interResultBuf, such as COUNT/TAGPRJ/PRJ/TAG etc. */ - resetResultInfo(&pRuntimeEnv->resultInfo[i]); + RESET_RESULT_INFO(&pRuntimeEnv->resultInfo[i]); pCtx->resultInfo = &pRuntimeEnv->resultInfo[i]; // set the timestamp output buffer for top/bottom/diff query @@ -3072,7 +3100,7 @@ void forwardCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, int64_t output) { pRuntimeEnv->pCtx[j].ptsOutputBuf += TSDB_KEYSIZE * output; } - resetResultInfo(pRuntimeEnv->pCtx[j].resultInfo); + RESET_RESULT_INFO(pRuntimeEnv->pCtx[j].resultInfo); } } @@ -3326,8 +3354,8 @@ void scanOneTableDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start) { if (pRuntimeEnv->pSecQueryHandle == NULL) { longjmp(pRuntimeEnv->env, terrno); } - pRuntimeEnv->windowResInfo.curIndex = qstatus.windowIndex; + pRuntimeEnv->windowResInfo.curIndex = qstatus.windowIndex; setQueryStatus(pQuery, QUERY_NOT_COMPLETED); pRuntimeEnv->scanFlag = REPEAT_SCAN; @@ -3462,7 +3490,6 @@ void setExecutionContext(SQInfo *pQInfo, int32_t groupIndex, TSKEY nextKey) { return; } - int32_t GROUPRESULTID = 1; SWindowResult *pWindowRes = doSetTimeWindowFromKey(pRuntimeEnv, pWindowResInfo, (char *)&groupIndex, sizeof(groupIndex), true); if (pWindowRes == NULL) { @@ -3474,7 +3501,7 @@ void setExecutionContext(SQInfo *pQInfo, int32_t groupIndex, TSKEY nextKey) { * all group belong to one result set, and each group result has different group id so set the id to be one */ if (pWindowRes->pos.pageId == -1) { - if (addNewWindowResultBuf(pWindowRes, pRuntimeEnv->pResultBuf, GROUPRESULTID, pRuntimeEnv->numOfRowsPerPage) != + if (addNewWindowResultBuf(pWindowRes, pRuntimeEnv->pResultBuf, groupIndex, pRuntimeEnv->numOfRowsPerPage) != TSDB_CODE_SUCCESS) { return; } @@ -4170,7 +4197,6 @@ static SFillColInfo* taosCreateFillColInfo(SQuery* pQuery) { } int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bool isSTableQuery) { - qDebug("QInfo:%p start to init qhandle", pQInfo); SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; int32_t code = TSDB_CODE_SUCCESS; @@ -4209,33 +4235,42 @@ int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bo return code; } - pRuntimeEnv->numOfRowsPerPage = getNumOfRowsInResultPage(pQuery, pRuntimeEnv->topBotQuery, isSTableQuery); + int32_t ps = DEFAULT_PAGE_SIZE; + int32_t rowsize = 0; + getIntermediateBufInfo(pRuntimeEnv, &ps, &rowsize); if (isSTableQuery && !onlyQueryTags(pRuntimeEnv->pQuery)) { - int32_t rows = getInitialPageNum(pQInfo); - code = createDiskbasedResultBuffer(&pRuntimeEnv->pResultBuf, rows, pQuery->rowSize, pQInfo); + int32_t numOfPages = getInitialPageNum(pQInfo); + code = createDiskbasedResultBuffer(&pRuntimeEnv->pResultBuf, numOfPages, rowsize, ps, numOfPages, pQInfo); if (code != TSDB_CODE_SUCCESS) { return code; } if (!QUERY_IS_INTERVAL_QUERY(pQuery)) { int16_t type = TSDB_DATA_TYPE_NULL; + int32_t threshold = 0; if (pRuntimeEnv->groupbyNormalCol) { // group by columns not tags; type = getGroupbyColumnType(pQuery, pQuery->pGroupbyExpr); + threshold = 4000; } else { type = TSDB_DATA_TYPE_INT; // group id + threshold = GET_NUM_OF_TABLEGROUP(pQInfo); + if (threshold < 8) { + threshold = 8; + } } - code = initWindowResInfo(&pRuntimeEnv->windowResInfo, pRuntimeEnv, 32, 4096, type); + code = initWindowResInfo(&pRuntimeEnv->windowResInfo, pRuntimeEnv, 8, threshold, type); if (code != TSDB_CODE_SUCCESS) { return code; } } - } else if (pRuntimeEnv->groupbyNormalCol || QUERY_IS_INTERVAL_QUERY(pQuery)) { - int32_t rows = getInitialPageNum(pQInfo); - code = createDiskbasedResultBuffer(&pRuntimeEnv->pResultBuf, rows, pQuery->rowSize, pQInfo); + int32_t numOfResultRows = getInitialPageNum(pQInfo); + getIntermediateBufInfo(pRuntimeEnv, &ps, &rowsize); + + code = createDiskbasedResultBuffer(&pRuntimeEnv->pResultBuf, numOfResultRows, rowsize, ps, numOfResultRows, pQInfo); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -4247,7 +4282,7 @@ int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bo type = TSDB_DATA_TYPE_TIMESTAMP; } - code = initWindowResInfo(&pRuntimeEnv->windowResInfo, pRuntimeEnv, rows, 4096, type); + code = initWindowResInfo(&pRuntimeEnv->windowResInfo, pRuntimeEnv, numOfResultRows, 4096, type); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -5705,13 +5740,18 @@ static void calResultBufSize(SQuery* pQuery) { const int32_t RESULT_MSG_MIN_ROWS = 8192; const float RESULT_THRESHOLD_RATIO = 0.85; - int32_t numOfRes = RESULT_MSG_MIN_SIZE / pQuery->rowSize; - if (numOfRes < RESULT_MSG_MIN_ROWS) { - numOfRes = RESULT_MSG_MIN_ROWS; - } + if (isProjQuery(pQuery)) { + int32_t numOfRes = RESULT_MSG_MIN_SIZE / pQuery->rowSize; + if (numOfRes < RESULT_MSG_MIN_ROWS) { + numOfRes = RESULT_MSG_MIN_ROWS; + } - pQuery->rec.capacity = numOfRes; - pQuery->rec.threshold = numOfRes * RESULT_THRESHOLD_RATIO; + pQuery->rec.capacity = numOfRes; + pQuery->rec.threshold = numOfRes * RESULT_THRESHOLD_RATIO; + } else { // in case of non-prj query, a smaller output buffer will be used. + pQuery->rec.capacity = 4096; + pQuery->rec.threshold = pQuery->rec.capacity * RESULT_THRESHOLD_RATIO; + } } static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SArray* pTableIdList, SSqlGroupbyExpr *pGroupbyExpr, SExprInfo *pExprs, @@ -5723,6 +5763,7 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SArray* pTableIdList, if (pQInfo == NULL) { goto _cleanup_qinfo; } + // to make sure third party won't overwrite this structure pQInfo->signature = pQInfo; pQInfo->tableGroupInfo = *pTableGroupInfo; @@ -6041,11 +6082,10 @@ static void freeQInfo(SQInfo *pQInfo) { tfree(pQuery->sdata); tfree(pQuery); + pQInfo->signature = 0; qDebug("QInfo:%p QInfo is freed", pQInfo); - // destroy signature, in order to avoid the query process pass the object safety check - memset(pQInfo, 0, sizeof(SQInfo)); tfree(pQInfo); } @@ -6209,7 +6249,7 @@ int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, vo } int64_t el = taosGetTimestampUs() - st; - qDebug("qmsg:%p tag filter completed, elapsed time:%"PRId64"us", pQueryMsg, el); + qDebug("qmsg:%p tag filter completed, numOfTables:%zu, elapsed time:%"PRId64"us", pQueryMsg, tableGroupInfo.numOfTables, el); } else { assert(0); } diff --git a/src/query/src/qFilterFunc.c b/src/query/src/qfilterfunc.c similarity index 99% rename from src/query/src/qFilterFunc.c rename to src/query/src/qfilterfunc.c index 1a95b9fe21..6f336434e1 100644 --- a/src/query/src/qFilterFunc.c +++ b/src/query/src/qfilterfunc.c @@ -16,7 +16,7 @@ #define _DEFAULT_SOURCE #include "os.h" -#include "qExecutor.h" +#include "qexecutor.h" #include "taosmsg.h" #include "tcompare.h" #include "tsqlfunction.h" @@ -554,5 +554,3 @@ __filter_func_t* getValueFilterFuncArray(int32_t type) { default: return NULL; } } - -bool supportPrefilter(int32_t type) { return type != TSDB_DATA_TYPE_BINARY && type != TSDB_DATA_TYPE_NCHAR; } diff --git a/src/query/src/qresultBuf.c b/src/query/src/qresultBuf.c deleted file mode 100644 index ae1a95179b..0000000000 --- a/src/query/src/qresultBuf.c +++ /dev/null @@ -1,184 +0,0 @@ -#include "qresultBuf.h" -#include "hash.h" -#include "qextbuffer.h" -#include "taoserror.h" -#include "queryLog.h" - -int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t size, int32_t rowSize, void* handle) { - *pResultBuf = calloc(1, sizeof(SDiskbasedResultBuf)); - SDiskbasedResultBuf* pResBuf = *pResultBuf; - if (pResBuf == NULL) { - return TSDB_CODE_COM_OUT_OF_MEMORY; - } - - pResBuf->numOfRowsPerPage = (DEFAULT_INTERN_BUF_PAGE_SIZE - sizeof(tFilePage)) / rowSize; - pResBuf->numOfPages = size; - - pResBuf->totalBufSize = pResBuf->numOfPages * DEFAULT_INTERN_BUF_PAGE_SIZE; - pResBuf->incStep = 4; - - // init id hash table - pResBuf->idsTable = taosHashInit(size, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false); - pResBuf->list = taosArrayInit(size, POINTER_BYTES); - - char path[4096] = {0}; - getTmpfilePath("tsdb_qbuf", path); - pResBuf->path = strdup(path); - - pResBuf->fd = open(pResBuf->path, O_CREAT | O_RDWR, 0666); - if (!FD_VALID(pResBuf->fd)) { - qError("failed to create tmp file: %s on disk. %s", pResBuf->path, strerror(errno)); - return TAOS_SYSTEM_ERROR(errno); - } - - int32_t ret = ftruncate(pResBuf->fd, pResBuf->numOfPages * DEFAULT_INTERN_BUF_PAGE_SIZE); - if (ret != TSDB_CODE_SUCCESS) { - qError("failed to create tmp file: %s on disk. %s", pResBuf->path, strerror(errno)); - return TAOS_SYSTEM_ERROR(errno); - } - - pResBuf->pBuf = mmap(NULL, pResBuf->totalBufSize, PROT_READ | PROT_WRITE, MAP_SHARED, pResBuf->fd, 0); - if (pResBuf->pBuf == MAP_FAILED) { - qError("QInfo:%p failed to map temp file: %s. %s", handle, pResBuf->path, strerror(errno)); - return TAOS_SYSTEM_ERROR(errno); - } - - qDebug("QInfo:%p create tmp file for output result:%s, %" PRId64 "bytes", handle, pResBuf->path, - pResBuf->totalBufSize); - - return TSDB_CODE_SUCCESS; -} - -int32_t getNumOfResultBufGroupId(SDiskbasedResultBuf* pResultBuf) { return taosHashGetSize(pResultBuf->idsTable); } - -int32_t getResBufSize(SDiskbasedResultBuf* pResultBuf) { return pResultBuf->totalBufSize; } - -static int32_t extendDiskFileSize(SDiskbasedResultBuf* pResultBuf, int32_t numOfPages) { - assert(pResultBuf->numOfPages * DEFAULT_INTERN_BUF_PAGE_SIZE == pResultBuf->totalBufSize); - - int32_t ret = munmap(pResultBuf->pBuf, pResultBuf->totalBufSize); - pResultBuf->numOfPages += numOfPages; - - /* - * disk-based output buffer is exhausted, try to extend the disk-based buffer, the available disk space may - * be insufficient - */ - ret = ftruncate(pResultBuf->fd, pResultBuf->numOfPages * DEFAULT_INTERN_BUF_PAGE_SIZE); - if (ret != 0) { - // dError("QInfo:%p failed to create intermediate result output file:%s. %s", pQInfo, pSupporter->extBufFile, - // strerror(errno)); - return TSDB_CODE_QRY_NO_DISKSPACE; - } - - pResultBuf->totalBufSize = pResultBuf->numOfPages * DEFAULT_INTERN_BUF_PAGE_SIZE; - pResultBuf->pBuf = mmap(NULL, pResultBuf->totalBufSize, PROT_READ | PROT_WRITE, MAP_SHARED, pResultBuf->fd, 0); - - if (pResultBuf->pBuf == MAP_FAILED) { - // dError("QInfo:%p failed to map temp file: %s. %s", pQInfo, pSupporter->extBufFile, strerror(errno)); - return TSDB_CODE_QRY_OUT_OF_MEMORY; - } - - return TSDB_CODE_SUCCESS; -} - -static FORCE_INLINE bool noMoreAvailablePages(SDiskbasedResultBuf* pResultBuf) { - return (pResultBuf->allocateId == pResultBuf->numOfPages - 1); -} - -static FORCE_INLINE int32_t getGroupIndex(SDiskbasedResultBuf* pResultBuf, int32_t groupId) { - assert(pResultBuf != NULL); - - char* p = taosHashGet(pResultBuf->idsTable, (const char*)&groupId, sizeof(int32_t)); - if (p == NULL) { // it is a new group id - return -1; - } - - int32_t slot = GET_INT32_VAL(p); - assert(slot >= 0 && slot < taosHashGetSize(pResultBuf->idsTable)); - - return slot; -} - -static int32_t addNewGroupId(SDiskbasedResultBuf* pResultBuf, int32_t groupId) { - int32_t num = getNumOfResultBufGroupId(pResultBuf); // the num is the newest allocated group id slot - taosHashPut(pResultBuf->idsTable, (const char*)&groupId, sizeof(int32_t), &num, sizeof(int32_t)); - - SArray* pa = taosArrayInit(1, sizeof(int32_t)); - taosArrayPush(pResultBuf->list, &pa); - - assert(taosArrayGetSize(pResultBuf->list) == taosHashGetSize(pResultBuf->idsTable)); - return num; -} - -static void registerPageId(SDiskbasedResultBuf* pResultBuf, int32_t groupId, int32_t pageId) { - int32_t slot = getGroupIndex(pResultBuf, groupId); - if (slot < 0) { - slot = addNewGroupId(pResultBuf, groupId); - } - - SIDList pList = taosArrayGetP(pResultBuf->list, slot); - taosArrayPush(pList, &pageId); -} - -tFilePage* getNewDataBuf(SDiskbasedResultBuf* pResultBuf, int32_t groupId, int32_t* pageId) { - if (noMoreAvailablePages(pResultBuf)) { - if (extendDiskFileSize(pResultBuf, pResultBuf->incStep) != TSDB_CODE_SUCCESS) { - return NULL; - } - } - - // register new id in this group - *pageId = (pResultBuf->allocateId++); - registerPageId(pResultBuf, groupId, *pageId); - - tFilePage* page = GET_RES_BUF_PAGE_BY_ID(pResultBuf, *pageId); - - // clear memory for the new page - memset(page, 0, DEFAULT_INTERN_BUF_PAGE_SIZE); - - return page; -} - -int32_t getNumOfRowsPerPage(SDiskbasedResultBuf* pResultBuf) { return pResultBuf->numOfRowsPerPage; } - -SIDList getDataBufPagesIdList(SDiskbasedResultBuf* pResultBuf, int32_t groupId) { - int32_t slot = getGroupIndex(pResultBuf, groupId); - if (slot < 0) { - return taosArrayInit(1, sizeof(int32_t)); - } else { - return taosArrayGetP(pResultBuf->list, slot); - } -} - -void destroyResultBuf(SDiskbasedResultBuf* pResultBuf, void* handle) { - if (pResultBuf == NULL) { - return; - } - - if (FD_VALID(pResultBuf->fd)) { - close(pResultBuf->fd); - } - - qDebug("QInfo:%p disk-based output buffer closed, %" PRId64 " bytes, file:%s", handle, pResultBuf->totalBufSize, pResultBuf->path); - munmap(pResultBuf->pBuf, pResultBuf->totalBufSize); - unlink(pResultBuf->path); - - tfree(pResultBuf->path); - - size_t size = taosArrayGetSize(pResultBuf->list); - for (int32_t i = 0; i < size; ++i) { - SArray* pa = taosArrayGetP(pResultBuf->list, i); - taosArrayDestroy(pa); - } - - taosArrayDestroy(pResultBuf->list); - taosHashCleanup(pResultBuf->idsTable); - - tfree(pResultBuf); -} - -int32_t getLastPageId(SIDList pList) { - size_t size = taosArrayGetSize(pList); - return *(int32_t*) taosArrayGet(pList, size - 1); -} - diff --git a/src/query/src/qresultbuf.c b/src/query/src/qresultbuf.c new file mode 100644 index 0000000000..646d8b7986 --- /dev/null +++ b/src/query/src/qresultbuf.c @@ -0,0 +1,218 @@ +#include "qresultbuf.h" +#include "hash.h" +#include "qextbuffer.h" +#include "queryLog.h" +#include "taoserror.h" + +int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t numOfPages, int32_t rowSize, + int32_t pagesize, int32_t inMemPages, void* handle) { + + *pResultBuf = calloc(1, sizeof(SDiskbasedResultBuf)); + SDiskbasedResultBuf* pResBuf = *pResultBuf; + if (pResBuf == NULL) { + return TSDB_CODE_COM_OUT_OF_MEMORY; + } + + pResBuf->pageSize = pagesize; + pResBuf->numOfPages = inMemPages; // all pages are in buffer in the first place + pResBuf->inMemPages = inMemPages; + assert(inMemPages <= numOfPages); + + pResBuf->numOfRowsPerPage = (pagesize - sizeof(tFilePage)) / rowSize; + + pResBuf->totalBufSize = pResBuf->numOfPages * pagesize; + pResBuf->incStep = 4; + pResBuf->allocateId = -1; + + pResBuf->iBuf = calloc(pResBuf->inMemPages, pResBuf->pageSize); + + // init id hash table + pResBuf->idsTable = taosHashInit(numOfPages, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false); + pResBuf->list = taosArrayInit(numOfPages, POINTER_BYTES); + + char path[PATH_MAX] = {0}; + getTmpfilePath("tsdb_qbuf", path); + pResBuf->path = strdup(path); + + pResBuf->fd = FD_INITIALIZER; + pResBuf->pBuf = NULL; + + qDebug("QInfo:%p create resBuf for output, page size:%d, initial pages:%d, %" PRId64 "bytes", handle, + pResBuf->pageSize, pResBuf->numOfPages, pResBuf->totalBufSize); + + return TSDB_CODE_SUCCESS; +} + +int32_t getNumOfResultBufGroupId(SDiskbasedResultBuf* pResultBuf) { return taosHashGetSize(pResultBuf->idsTable); } + +int32_t getResBufSize(SDiskbasedResultBuf* pResultBuf) { return pResultBuf->totalBufSize; } + +#define NUM_OF_PAGES_ON_DISK(_r) ((_r)->numOfPages - (_r)->inMemPages) +#define FILE_SIZE_ON_DISK(_r) (NUM_OF_PAGES_ON_DISK(_r) * (_r)->pageSize) + +static int32_t createDiskResidesBuf(SDiskbasedResultBuf* pResultBuf) { + pResultBuf->fd = open(pResultBuf->path, O_CREAT | O_RDWR, 0666); + if (!FD_VALID(pResultBuf->fd)) { + qError("failed to create tmp file: %s on disk. %s", pResultBuf->path, strerror(errno)); + return TAOS_SYSTEM_ERROR(errno); + } + + assert(pResultBuf->numOfPages == pResultBuf->inMemPages); + pResultBuf->numOfPages += pResultBuf->incStep; + + int32_t ret = ftruncate(pResultBuf->fd, NUM_OF_PAGES_ON_DISK(pResultBuf) * pResultBuf->pageSize); + if (ret != TSDB_CODE_SUCCESS) { + qError("failed to create tmp file: %s on disk. %s", pResultBuf->path, strerror(errno)); + return TAOS_SYSTEM_ERROR(errno); + } + + pResultBuf->pBuf = mmap(NULL, FILE_SIZE_ON_DISK(pResultBuf), PROT_READ | PROT_WRITE, MAP_SHARED, pResultBuf->fd, 0); + if (pResultBuf->pBuf == MAP_FAILED) { + qError("QInfo:%p failed to map temp file: %s. %s", pResultBuf->handle, pResultBuf->path, strerror(errno)); + return TAOS_SYSTEM_ERROR(errno); + } + + pResultBuf->totalBufSize = pResultBuf->numOfPages * pResultBuf->pageSize; + return TSDB_CODE_SUCCESS; +} + +static int32_t extendDiskFileSize(SDiskbasedResultBuf* pResultBuf, int32_t incNumOfPages) { + assert(pResultBuf->numOfPages * pResultBuf->pageSize == pResultBuf->totalBufSize); + int32_t ret = TSDB_CODE_SUCCESS; + + if (pResultBuf->pBuf == NULL) { + assert(pResultBuf->fd == FD_INITIALIZER); + + if ((ret = createDiskResidesBuf(pResultBuf)) != TSDB_CODE_SUCCESS) { + return ret; + } + } else { + ret = munmap(pResultBuf->pBuf, FILE_SIZE_ON_DISK(pResultBuf)); + pResultBuf->numOfPages += incNumOfPages; + + /* + * disk-based output buffer is exhausted, try to extend the disk-based buffer, the available disk space may + * be insufficient + */ + ret = ftruncate(pResultBuf->fd, NUM_OF_PAGES_ON_DISK(pResultBuf) * pResultBuf->pageSize); + if (ret != TSDB_CODE_SUCCESS) { + // dError("QInfo:%p failed to create intermediate result output file:%s. %s", pQInfo, pSupporter->extBufFile, + // strerror(errno)); + return TSDB_CODE_QRY_NO_DISKSPACE; + } + + pResultBuf->totalBufSize = pResultBuf->numOfPages * pResultBuf->pageSize; + pResultBuf->pBuf = mmap(NULL, FILE_SIZE_ON_DISK(pResultBuf), PROT_READ | PROT_WRITE, MAP_SHARED, pResultBuf->fd, 0); + + if (pResultBuf->pBuf == MAP_FAILED) { + // dError("QInfo:%p failed to map temp file: %s. %s", pQInfo, pSupporter->extBufFile, strerror(errno)); + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } + } + + return TSDB_CODE_SUCCESS; +} + +#define NO_AVAILABLE_PAGES(_b) ((_b)->allocateId == (_b)->numOfPages - 1) + +static FORCE_INLINE int32_t getGroupIndex(SDiskbasedResultBuf* pResultBuf, int32_t groupId) { + assert(pResultBuf != NULL); + + char* p = taosHashGet(pResultBuf->idsTable, (const char*)&groupId, sizeof(int32_t)); + if (p == NULL) { // it is a new group id + return -1; + } + + int32_t slot = GET_INT32_VAL(p); + assert(slot >= 0 && slot < taosHashGetSize(pResultBuf->idsTable)); + + return slot; +} + +static int32_t addNewGroupId(SDiskbasedResultBuf* pResultBuf, int32_t groupId) { + int32_t num = getNumOfResultBufGroupId(pResultBuf); // the num is the newest allocated group id slot + taosHashPut(pResultBuf->idsTable, (const char*)&groupId, sizeof(int32_t), &num, sizeof(int32_t)); + + SArray* pa = taosArrayInit(1, sizeof(int32_t)); + taosArrayPush(pResultBuf->list, &pa); + + assert(taosArrayGetSize(pResultBuf->list) == taosHashGetSize(pResultBuf->idsTable)); + return num; +} + +static void registerPageId(SDiskbasedResultBuf* pResultBuf, int32_t groupId, int32_t pageId) { + int32_t slot = getGroupIndex(pResultBuf, groupId); + if (slot < 0) { + slot = addNewGroupId(pResultBuf, groupId); + } + + SIDList pList = taosArrayGetP(pResultBuf->list, slot); + taosArrayPush(pList, &pageId); +} + +tFilePage* getNewDataBuf(SDiskbasedResultBuf* pResultBuf, int32_t groupId, int32_t* pageId) { + if (NO_AVAILABLE_PAGES(pResultBuf)) { + if (extendDiskFileSize(pResultBuf, pResultBuf->incStep) != TSDB_CODE_SUCCESS) { + return NULL; + } + } + + // register new id in this group + *pageId = (++pResultBuf->allocateId); + registerPageId(pResultBuf, groupId, *pageId); + + // clear memory for the new page + tFilePage* page = getResBufPage(pResultBuf, *pageId); + memset(page, 0, pResultBuf->pageSize); + + return page; +} + +int32_t getNumOfRowsPerPage(SDiskbasedResultBuf* pResultBuf) { return pResultBuf->numOfRowsPerPage; } + +SIDList getDataBufPagesIdList(SDiskbasedResultBuf* pResultBuf, int32_t groupId) { + int32_t slot = getGroupIndex(pResultBuf, groupId); + if (slot < 0) { + return taosArrayInit(1, sizeof(int32_t)); + } else { + return taosArrayGetP(pResultBuf->list, slot); + } +} + +void destroyResultBuf(SDiskbasedResultBuf* pResultBuf, void* handle) { + if (pResultBuf == NULL) { + return; + } + + if (FD_VALID(pResultBuf->fd)) { + qDebug("QInfo:%p disk-based output buffer closed, total:%" PRId64 " bytes, file created:%s, file size:%d", handle, + pResultBuf->totalBufSize, pResultBuf->path, FILE_SIZE_ON_DISK(pResultBuf)); + + close(pResultBuf->fd); + munmap(pResultBuf->pBuf, FILE_SIZE_ON_DISK(pResultBuf)); + pResultBuf->pBuf = NULL; + } else { + qDebug("QInfo:%p disk-based output buffer closed, total:%" PRId64 " bytes, no file created", handle, + pResultBuf->totalBufSize); + } + + unlink(pResultBuf->path); + tfree(pResultBuf->path); + + size_t size = taosArrayGetSize(pResultBuf->list); + for (int32_t i = 0; i < size; ++i) { + SArray* pa = taosArrayGetP(pResultBuf->list, i); + taosArrayDestroy(pa); + } + + taosArrayDestroy(pResultBuf->list); + taosHashCleanup(pResultBuf->idsTable); + + tfree(pResultBuf); +} + +int32_t getLastPageId(SIDList pList) { + size_t size = taosArrayGetSize(pList); + return *(int32_t*) taosArrayGet(pList, size - 1); +} + diff --git a/src/query/src/qUtil.c b/src/query/src/qutil.c similarity index 98% rename from src/query/src/qUtil.c rename to src/query/src/qutil.c index 4e71de830f..4c3595b0be 100644 --- a/src/query/src/qUtil.c +++ b/src/query/src/qutil.c @@ -19,19 +19,17 @@ #include "taosmsg.h" #include "ttime.h" -#include "qExecutor.h" -#include "qUtil.h" +#include "qexecutor.h" +#include "qutil.h" int32_t getOutputInterResultBufSize(SQuery* pQuery) { int32_t size = 0; for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - assert(pQuery->pSelectExpr[i].interBytes <= DEFAULT_INTERN_BUF_PAGE_SIZE); size += pQuery->pSelectExpr[i].interBytes; } assert(size > 0); - return size; } @@ -243,7 +241,7 @@ void clearTimeWindowResBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pWindow size_t size = pRuntimeEnv->pQuery->pSelectExpr[i].bytes; memset(s, 0, size); - resetResultInfo(pResultInfo); + RESET_RESULT_INFO(pResultInfo); } pWindowRes->numOfRows = 0; diff --git a/src/query/tests/resultBufferTest.cpp b/src/query/tests/resultBufferTest.cpp index 9270c6e458..bf6f3bb50f 100644 --- a/src/query/tests/resultBufferTest.cpp +++ b/src/query/tests/resultBufferTest.cpp @@ -2,8 +2,8 @@ #include #include +#include "qresultbuf.h" #include "taos.h" -#include "qresultBuf.h" #include "tsdb.h" namespace { diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 8463e791db..5cb0dffd46 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -1376,7 +1376,7 @@ static int32_t createDataBlocksInfo(STsdbQueryHandle* pQueryHandle, int32_t numO * } */ - tsdbDebug("%p %d data blocks sort completed", pQueryHandle, cnt); + tsdbDebug("%p %d data blocks sort completed, %p", pQueryHandle, cnt, pQueryHandle->qinfo); cleanBlockOrderSupporter(&sup, numOfTables); free(pTree); diff --git a/src/util/src/tutil.c b/src/util/src/tutil.c index 3764df4afc..7b73b1e17c 100644 --- a/src/util/src/tutil.c +++ b/src/util/src/tutil.c @@ -522,7 +522,7 @@ int32_t taosFileRename(char *fullPath, char *suffix, char delimiter, char **dstP void getTmpfilePath(const char *fileNamePrefix, char *dstPath) { const char* tdengineTmpFileNamePrefix = "tdengine-"; - char tmpPath[PATH_MAX] = {0}; + char tmpPath[PATH_MAX]; #ifdef WINDOWS char *tmpDir = getenv("tmp"); From 3cf5b7697b50a9986df2d8c4be27a3a809c65342 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 20 Jul 2020 00:02:51 +0800 Subject: [PATCH 24/34] [td-225] source files rename. --- src/client/inc/tscLocalMerge.h | 4 ++-- src/client/inc/tscUtil.h | 6 +++--- src/client/inc/tsclient.h | 4 ++-- src/client/src/tscFunctionImpl.c | 14 +++++++------- src/client/src/tscLocal.c | 12 ++++++------ src/client/src/tscSQLParser.c | 2 +- src/client/src/tscSql.c | 2 +- src/client/src/tscSubquery.c | 4 ++-- src/client/src/tscUtil.c | 6 +++--- src/query/inc/{qast.h => qAst.h} | 0 src/query/inc/{qexecutor.h => qExecutor.h} | 6 +++--- src/query/inc/{qextbuffer.h => qExtbuffer.h} | 2 +- src/query/inc/{qfill.h => qFill.h} | 2 +- src/query/inc/{qhistogram.h => qHistogram.h} | 0 src/query/inc/{qpercentile.h => qPercentile.h} | 2 +- src/query/inc/{qresultbuf.h => qResultbuf.h} | 4 ++-- ...qsyntaxtreefunction.h => qSyntaxtreefunction.h} | 0 src/query/inc/{qtsbuf.h => qTsbuf.h} | 0 src/query/inc/{qutil.h => qUtil.h} | 0 src/query/src/{qast.c => qAst.c} | 8 ++++---- src/query/src/{qexecutor.c => qExecutor.c} | 12 ++++++------ src/query/src/{qextbuffer.c => qExtbuffer.c} | 7 +++---- src/query/src/{qfill.c => qFill.c} | 4 ++-- src/query/src/{qfilterfunc.c => qFilterfunc.c} | 2 +- src/query/src/{qhistogram.c => qHistogram.c} | 2 +- src/query/src/{qparserImpl.c => qParserImpl.c} | 0 src/query/src/{qpercentile.c => qPercentile.c} | 6 +++--- src/query/src/{qresultbuf.c => qResultbuf.c} | 4 ++-- ...qsyntaxtreefunction.c => qSyntaxtreefunction.c} | 2 +- src/query/src/{qtokenizer.c => qTokenizer.c} | 0 src/query/src/{qtsbuf.c => qTsbuf.c} | 4 ++-- src/query/src/{qutil.c => qUtil.c} | 4 ++-- src/query/tests/astTest.cpp | 2 +- src/query/tests/histogramTest.cpp | 2 +- src/query/tests/resultBufferTest.cpp | 2 +- src/query/tests/tsBufTest.cpp | 2 +- 36 files changed, 66 insertions(+), 67 deletions(-) rename src/query/inc/{qast.h => qAst.h} (100%) rename src/query/inc/{qexecutor.h => qExecutor.h} (99%) rename src/query/inc/{qextbuffer.h => qExtbuffer.h} (98%) rename src/query/inc/{qfill.h => qFill.h} (99%) rename src/query/inc/{qhistogram.h => qHistogram.h} (100%) rename src/query/inc/{qpercentile.h => qPercentile.h} (98%) rename src/query/inc/{qresultbuf.h => qResultbuf.h} (99%) rename src/query/inc/{qsyntaxtreefunction.h => qSyntaxtreefunction.h} (100%) rename src/query/inc/{qtsbuf.h => qTsbuf.h} (100%) rename src/query/inc/{qutil.h => qUtil.h} (100%) rename src/query/src/{qast.c => qAst.c} (99%) rename src/query/src/{qexecutor.c => qExecutor.c} (99%) rename src/query/src/{qextbuffer.c => qExtbuffer.c} (99%) rename src/query/src/{qfill.c => qFill.c} (99%) rename src/query/src/{qfilterfunc.c => qFilterfunc.c} (99%) rename src/query/src/{qhistogram.c => qHistogram.c} (99%) rename src/query/src/{qparserImpl.c => qParserImpl.c} (100%) rename src/query/src/{qpercentile.c => qPercentile.c} (99%) rename src/query/src/{qresultbuf.c => qResultbuf.c} (99%) rename src/query/src/{qsyntaxtreefunction.c => qSyntaxtreefunction.c} (99%) rename src/query/src/{qtokenizer.c => qTokenizer.c} (100%) rename src/query/src/{qtsbuf.c => qTsbuf.c} (99%) rename src/query/src/{qutil.c => qUtil.c} (99%) diff --git a/src/client/inc/tscLocalMerge.h b/src/client/inc/tscLocalMerge.h index d38885ab2e..397a60d140 100644 --- a/src/client/inc/tscLocalMerge.h +++ b/src/client/inc/tscLocalMerge.h @@ -20,8 +20,8 @@ extern "C" { #endif -#include "qextbuffer.h" -#include "qfill.h" +#include "qExtbuffer.h" +#include "qFill.h" #include "taosmsg.h" #include "tlosertree.h" #include "tsclient.h" diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index 786133a8f3..590f205e1d 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -23,11 +23,11 @@ extern "C" { /* * @date 2018/09/30 */ -#include "os.h" -#include "tbuffer.h" #include "exception.h" -#include "qextbuffer.h" +#include "os.h" +#include "qExtbuffer.h" #include "taosdef.h" +#include "tbuffer.h" #include "tscLocalMerge.h" #include "tsclient.h" diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index 3f548c18ca..3b19a475dc 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -30,9 +30,9 @@ extern "C" { #include "tsqlfunction.h" #include "tutil.h" -#include "qexecutor.h" +#include "qExecutor.h" +#include "qTsbuf.h" #include "qsqlparser.h" -#include "qtsbuf.h" #include "tcmdtype.h" // forward declaration diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c index fb15e4efd5..72ccd5adc6 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/client/src/tscFunctionImpl.c @@ -14,15 +14,15 @@ */ #include "os.h" -#include "qextbuffer.h" -#include "qfill.h" -#include "qhistogram.h" -#include "qpercentile.h" -#include "qsyntaxtreefunction.h" -#include "qtsbuf.h" +#include "qAst.h" +#include "qExtbuffer.h" +#include "qFill.h" +#include "qHistogram.h" +#include "qPercentile.h" +#include "qSyntaxtreefunction.h" +#include "qTsbuf.h" #include "taosdef.h" #include "taosmsg.h" -#include "qast.h" #include "tscLog.h" #include "tscSubquery.h" #include "tscompression.h" diff --git a/src/client/src/tscLocal.c b/src/client/src/tscLocal.c index 83700ce0a5..7f336daa91 100644 --- a/src/client/src/tscLocal.c +++ b/src/client/src/tscLocal.c @@ -16,14 +16,14 @@ #include "os.h" #include "taosmsg.h" -#include "tcache.h" -#include "tscUtil.h" -#include "tsclient.h" +#include "qExtbuffer.h" #include "taosdef.h" -#include "tscLog.h" -#include "qextbuffer.h" -#include "tschemautil.h" +#include "tcache.h" #include "tname.h" +#include "tscLog.h" +#include "tscUtil.h" +#include "tschemautil.h" +#include "tsclient.h" static void tscSetLocalQueryResult(SSqlObj *pSql, const char *val, const char *columnName, int16_t type, size_t valueLength); diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index b97e486449..09db65a105 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -18,9 +18,9 @@ #define _DEFAULT_SOURCE #include "os.h" +#include "qAst.h" #include "taos.h" #include "taosmsg.h" -#include "qast.h" #include "tcompare.h" #include "tname.h" #include "tscLog.h" diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index 6a14d3a65e..77584f168c 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -15,7 +15,7 @@ #include "hash.h" #include "os.h" -#include "qast.h" +#include "qAst.h" #include "tcache.h" #include "tnote.h" #include "trpc.h" diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 1dbc52efb0..5d26d09fae 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -14,8 +14,8 @@ */ #include "os.h" -#include "qtsbuf.h" -#include "qast.h" +#include "qAst.h" +#include "qTsbuf.h" #include "tcompare.h" #include "tscLog.h" #include "tscSubquery.h" diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 4e6133663e..27f6727339 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -13,11 +13,11 @@ * along with this program. If not, see . */ -#include "os.h" -#include "hash.h" #include "tscUtil.h" +#include "hash.h" +#include "os.h" +#include "qAst.h" #include "taosmsg.h" -#include "qast.h" #include "tcache.h" #include "tkey.h" #include "tmd5.h" diff --git a/src/query/inc/qast.h b/src/query/inc/qAst.h similarity index 100% rename from src/query/inc/qast.h rename to src/query/inc/qAst.h diff --git a/src/query/inc/qexecutor.h b/src/query/inc/qExecutor.h similarity index 99% rename from src/query/inc/qexecutor.h rename to src/query/inc/qExecutor.h index 97a7565de8..127c38a6f8 100644 --- a/src/query/inc/qexecutor.h +++ b/src/query/inc/qExecutor.h @@ -18,10 +18,10 @@ #include "os.h" #include "hash.h" -#include "qfill.h" -#include "qresultbuf.h" +#include "qFill.h" +#include "qResultbuf.h" +#include "qTsbuf.h" #include "qsqlparser.h" -#include "qtsbuf.h" #include "query.h" #include "taosdef.h" #include "tarray.h" diff --git a/src/query/inc/qextbuffer.h b/src/query/inc/qExtbuffer.h similarity index 98% rename from src/query/inc/qextbuffer.h rename to src/query/inc/qExtbuffer.h index 2cbef2b1be..b57c48933f 100644 --- a/src/query/inc/qextbuffer.h +++ b/src/query/inc/qExtbuffer.h @@ -28,7 +28,7 @@ extern "C" { #include "tdataformat.h" #include "talgo.h" -#define DEFAULT_PAGE_SIZE (1024L*64) // 16k larger than the SHistoInfo +#define DEFAULT_PAGE_SIZE (1024L*4) // 16k larger than the SHistoInfo #define MAX_TMPFILE_PATH_LENGTH PATH_MAX #define INITIAL_ALLOCATION_BUFFER_SIZE 64 diff --git a/src/query/inc/qfill.h b/src/query/inc/qFill.h similarity index 99% rename from src/query/inc/qfill.h rename to src/query/inc/qFill.h index ee5974708a..db6a69c2c5 100644 --- a/src/query/inc/qfill.h +++ b/src/query/inc/qFill.h @@ -21,8 +21,8 @@ extern "C" { #endif #include "os.h" +#include "qExtbuffer.h" #include "taosdef.h" -#include "qextbuffer.h" typedef struct { STColumn col; // column info diff --git a/src/query/inc/qhistogram.h b/src/query/inc/qHistogram.h similarity index 100% rename from src/query/inc/qhistogram.h rename to src/query/inc/qHistogram.h diff --git a/src/query/inc/qpercentile.h b/src/query/inc/qPercentile.h similarity index 98% rename from src/query/inc/qpercentile.h rename to src/query/inc/qPercentile.h index c1227dad77..52f666c338 100644 --- a/src/query/inc/qpercentile.h +++ b/src/query/inc/qPercentile.h @@ -16,7 +16,7 @@ #ifndef TDENGINE_QPERCENTILE_H #define TDENGINE_QPERCENTILE_H -#include "qextbuffer.h" +#include "qExtbuffer.h" typedef struct MinMaxEntry { union { diff --git a/src/query/inc/qresultbuf.h b/src/query/inc/qResultbuf.h similarity index 99% rename from src/query/inc/qresultbuf.h rename to src/query/inc/qResultbuf.h index f901474cc1..41fccd177d 100644 --- a/src/query/inc/qresultbuf.h +++ b/src/query/inc/qResultbuf.h @@ -20,9 +20,9 @@ extern "C" { #endif -#include "os.h" -#include "qextbuffer.h" #include "hash.h" +#include "os.h" +#include "qExtbuffer.h" typedef struct SArray* SIDList; diff --git a/src/query/inc/qsyntaxtreefunction.h b/src/query/inc/qSyntaxtreefunction.h similarity index 100% rename from src/query/inc/qsyntaxtreefunction.h rename to src/query/inc/qSyntaxtreefunction.h diff --git a/src/query/inc/qtsbuf.h b/src/query/inc/qTsbuf.h similarity index 100% rename from src/query/inc/qtsbuf.h rename to src/query/inc/qTsbuf.h diff --git a/src/query/inc/qutil.h b/src/query/inc/qUtil.h similarity index 100% rename from src/query/inc/qutil.h rename to src/query/inc/qUtil.h diff --git a/src/query/src/qast.c b/src/query/src/qAst.c similarity index 99% rename from src/query/src/qast.c rename to src/query/src/qAst.c index 42f9f214f5..c2578c15c0 100644 --- a/src/query/src/qast.c +++ b/src/query/src/qAst.c @@ -16,17 +16,17 @@ #include "os.h" -#include "tname.h" -#include "qast.h" -#include "tsdb.h" #include "exception.h" +#include "qAst.h" +#include "qSyntaxtreefunction.h" #include "qsqlparser.h" -#include "qsyntaxtreefunction.h" #include "taosdef.h" #include "taosmsg.h" #include "tarray.h" #include "tbuffer.h" #include "tcompare.h" +#include "tname.h" +#include "tsdb.h" #include "tskiplist.h" #include "tsqlfunction.h" #include "tstoken.h" diff --git a/src/query/src/qexecutor.c b/src/query/src/qExecutor.c similarity index 99% rename from src/query/src/qexecutor.c rename to src/query/src/qExecutor.c index 9348f702d0..b4f825a7c2 100644 --- a/src/query/src/qexecutor.c +++ b/src/query/src/qExecutor.c @@ -13,19 +13,19 @@ * along with this program. If not, see . */ #include "os.h" +#include "qFill.h" +#include "taosmsg.h" #include "tcache.h" #include "tglobal.h" -#include "qfill.h" -#include "taosmsg.h" #include "exception.h" #include "hash.h" -#include "qast.h" -#include "qexecutor.h" -#include "qresultbuf.h" +#include "qAst.h" +#include "qExecutor.h" +#include "qResultbuf.h" +#include "qUtil.h" #include "query.h" #include "queryLog.h" -#include "qutil.h" #include "tlosertree.h" #include "tscompression.h" #include "ttime.h" diff --git a/src/query/src/qextbuffer.c b/src/query/src/qExtbuffer.c similarity index 99% rename from src/query/src/qextbuffer.c rename to src/query/src/qExtbuffer.c index afcf902123..69c5f0e24f 100644 --- a/src/query/src/qextbuffer.c +++ b/src/query/src/qExtbuffer.c @@ -12,16 +12,15 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ +#include "qExtbuffer.h" #include "os.h" -#include "tulog.h" -#include "qextbuffer.h" +#include "queryLog.h" #include "taos.h" #include "taosdef.h" #include "taosmsg.h" #include "tsqlfunction.h" -#include "ttime.h" +#include "tulog.h" #include "tutil.h" -#include "queryLog.h" #define COLMODEL_GET_VAL(data, schema, allrow, rowId, colId) \ (data + (schema)->pFields[colId].offset * (allrow) + (rowId) * (schema)->pFields[colId].field.bytes) diff --git a/src/query/src/qfill.c b/src/query/src/qFill.c similarity index 99% rename from src/query/src/qfill.c rename to src/query/src/qFill.c index 65951a5b9e..9dec2598bc 100644 --- a/src/query/src/qfill.c +++ b/src/query/src/qFill.c @@ -13,9 +13,9 @@ * along with this program. If not, see . */ +#include "qFill.h" #include "os.h" -#include "qfill.h" -#include "qextbuffer.h" +#include "qExtbuffer.h" #include "taosdef.h" #include "taosmsg.h" #include "tsqlfunction.h" diff --git a/src/query/src/qfilterfunc.c b/src/query/src/qFilterfunc.c similarity index 99% rename from src/query/src/qfilterfunc.c rename to src/query/src/qFilterfunc.c index 6f336434e1..7e9f5c7da5 100644 --- a/src/query/src/qfilterfunc.c +++ b/src/query/src/qFilterfunc.c @@ -16,7 +16,7 @@ #define _DEFAULT_SOURCE #include "os.h" -#include "qexecutor.h" +#include "qExecutor.h" #include "taosmsg.h" #include "tcompare.h" #include "tsqlfunction.h" diff --git a/src/query/src/qhistogram.c b/src/query/src/qHistogram.c similarity index 99% rename from src/query/src/qhistogram.c rename to src/query/src/qHistogram.c index 26482e9f14..7835d82469 100644 --- a/src/query/src/qhistogram.c +++ b/src/query/src/qHistogram.c @@ -14,7 +14,7 @@ */ #include "os.h" -#include "qhistogram.h" +#include "qHistogram.h" #include "taosdef.h" #include "taosmsg.h" #include "tlosertree.h" diff --git a/src/query/src/qparserImpl.c b/src/query/src/qParserImpl.c similarity index 100% rename from src/query/src/qparserImpl.c rename to src/query/src/qParserImpl.c diff --git a/src/query/src/qpercentile.c b/src/query/src/qPercentile.c similarity index 99% rename from src/query/src/qpercentile.c rename to src/query/src/qPercentile.c index dc5ecb796a..c4490a01e7 100644 --- a/src/query/src/qpercentile.c +++ b/src/query/src/qPercentile.c @@ -13,12 +13,12 @@ * along with this program. If not, see . */ +#include "qPercentile.h" #include "os.h" -#include "tulog.h" -#include "qpercentile.h" +#include "queryLog.h" #include "taosdef.h" #include "taosmsg.h" -#include "queryLog.h" +#include "tulog.h" tExtMemBuffer *releaseBucketsExceptFor(tMemBucket *pMemBucket, int16_t segIdx, int16_t slotIdx) { tExtMemBuffer *pBuffer = NULL; diff --git a/src/query/src/qresultbuf.c b/src/query/src/qResultbuf.c similarity index 99% rename from src/query/src/qresultbuf.c rename to src/query/src/qResultbuf.c index 646d8b7986..f2194bbfb5 100644 --- a/src/query/src/qresultbuf.c +++ b/src/query/src/qResultbuf.c @@ -1,6 +1,6 @@ -#include "qresultbuf.h" +#include "qResultbuf.h" #include "hash.h" -#include "qextbuffer.h" +#include "qExtbuffer.h" #include "queryLog.h" #include "taoserror.h" diff --git a/src/query/src/qsyntaxtreefunction.c b/src/query/src/qSyntaxtreefunction.c similarity index 99% rename from src/query/src/qsyntaxtreefunction.c rename to src/query/src/qSyntaxtreefunction.c index 5719bb0188..2104edfd91 100644 --- a/src/query/src/qsyntaxtreefunction.c +++ b/src/query/src/qSyntaxtreefunction.c @@ -15,7 +15,7 @@ #include "os.h" -#include "qsyntaxtreefunction.h" +#include "qSyntaxtreefunction.h" #include "taosdef.h" #include "tutil.h" diff --git a/src/query/src/qtokenizer.c b/src/query/src/qTokenizer.c similarity index 100% rename from src/query/src/qtokenizer.c rename to src/query/src/qTokenizer.c diff --git a/src/query/src/qtsbuf.c b/src/query/src/qTsbuf.c similarity index 99% rename from src/query/src/qtsbuf.c rename to src/query/src/qTsbuf.c index b84fbded38..20b29107f5 100644 --- a/src/query/src/qtsbuf.c +++ b/src/query/src/qTsbuf.c @@ -1,7 +1,7 @@ -#include "qtsbuf.h" +#include "qTsbuf.h" +#include "taoserror.h" #include "tscompression.h" #include "tutil.h" -#include "taoserror.h" static int32_t getDataStartOffset(); static void TSBufUpdateVnodeInfo(STSBuf* pTSBuf, int32_t index, STSVnodeBlockInfo* pBlockInfo); diff --git a/src/query/src/qutil.c b/src/query/src/qUtil.c similarity index 99% rename from src/query/src/qutil.c rename to src/query/src/qUtil.c index 4c3595b0be..be84471493 100644 --- a/src/query/src/qutil.c +++ b/src/query/src/qUtil.c @@ -19,8 +19,8 @@ #include "taosmsg.h" #include "ttime.h" -#include "qexecutor.h" -#include "qutil.h" +#include "qExecutor.h" +#include "qUtil.h" int32_t getOutputInterResultBufSize(SQuery* pQuery) { int32_t size = 0; diff --git a/src/query/tests/astTest.cpp b/src/query/tests/astTest.cpp index df27087216..99f03a7ff8 100644 --- a/src/query/tests/astTest.cpp +++ b/src/query/tests/astTest.cpp @@ -3,8 +3,8 @@ #include #include +#include "qAst.h" #include "taosmsg.h" -#include "qast.h" #include "tsdb.h" #include "tskiplist.h" diff --git a/src/query/tests/histogramTest.cpp b/src/query/tests/histogramTest.cpp index c23f0f5924..4a5f7fbbbe 100644 --- a/src/query/tests/histogramTest.cpp +++ b/src/query/tests/histogramTest.cpp @@ -9,7 +9,7 @@ #include "tstoken.h" #include "tutil.h" -#include "qhistogram.h" +#include "qHistogram.h" /* test validate the names for table/database */ TEST(testCase, histogram_binary_search) { diff --git a/src/query/tests/resultBufferTest.cpp b/src/query/tests/resultBufferTest.cpp index bf6f3bb50f..f822deeb73 100644 --- a/src/query/tests/resultBufferTest.cpp +++ b/src/query/tests/resultBufferTest.cpp @@ -2,7 +2,7 @@ #include #include -#include "qresultbuf.h" +#include "qResultbuf.h" #include "taos.h" #include "tsdb.h" diff --git a/src/query/tests/tsBufTest.cpp b/src/query/tests/tsBufTest.cpp index f813261957..e9827518e1 100644 --- a/src/query/tests/tsBufTest.cpp +++ b/src/query/tests/tsBufTest.cpp @@ -5,10 +5,10 @@ #include "taos.h" #include "tsdb.h" +#include "qTsbuf.h" #include "tstoken.h" #include "ttime.h" #include "tutil.h" -#include "qtsbuf.h" namespace { /** From 731707608c9391392db8b95ab910f7f3fc6301d4 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 20 Jul 2020 00:04:21 +0800 Subject: [PATCH 25/34] [td-225] fix bugs in descending traverse over in-men data --- src/tsdb/src/tsdbRead.c | 79 ++++++++++++++++++++++++----------------- 1 file changed, 47 insertions(+), 32 deletions(-) diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 5cb0dffd46..5b1afe3da8 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -21,7 +21,7 @@ #include "tcompare.h" #include "exception.h" -#include "../../../query/inc/qast.h" // todo move to common module +#include "../../query/inc/qAst.h" // todo move to common module #include "tlosertree.h" #include "tsdb.h" #include "tsdbMain.h" @@ -331,7 +331,8 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh tsdbDebug("%p uid:%" PRId64", tid:%d check data in mem from skey:%" PRId64 ", order:%d, %p", pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, key, order, pHandle->qinfo); } else { - tsdbDebug("%p uid:%" PRId64 ", tid:%d no data in mem", pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid); + tsdbDebug("%p uid:%"PRId64", tid:%d no data in mem, %p", pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, + pHandle->qinfo); } if (!imemEmpty) { @@ -343,7 +344,8 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh tsdbDebug("%p uid:%" PRId64", tid:%d check data in imem from skey:%" PRId64 ", order:%d, %p", pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, key, order, pHandle->qinfo); } else { - tsdbDebug("%p uid:%"PRId64", tid:%d no data in imem", pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid); + tsdbDebug("%p uid:%"PRId64", tid:%d no data in imem, %p", pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, + pHandle->qinfo); } return true; @@ -354,7 +356,7 @@ static void destroyTableMemIterator(STableCheckInfo* pCheckInfo) { tSkipListDestroyIter(pCheckInfo->iiter); } -SDataRow getSDataRowInTableMem(STableCheckInfo* pCheckInfo) { +SDataRow getSDataRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order) { SDataRow rmem = NULL, rimem = NULL; if (pCheckInfo->iter) { SSkipListNode* node = tSkipListIterGet(pCheckInfo->iter); @@ -371,20 +373,35 @@ SDataRow getSDataRowInTableMem(STableCheckInfo* pCheckInfo) { } if (rmem != NULL && rimem != NULL) { - if (dataRowKey(rmem) < dataRowKey(rimem)) { - pCheckInfo->chosen = 0; - return rmem; - } else if (dataRowKey(rmem) == dataRowKey(rimem)) { - // data ts are duplicated, ignore the data in mem + TSKEY r1 = dataRowKey(rmem); + TSKEY r2 = dataRowKey(rimem); + + if (r1 == r2) { // data ts are duplicated, ignore the data in mem tSkipListIterNext(pCheckInfo->iter); pCheckInfo->chosen = 1; return rimem; } else { - pCheckInfo->chosen = 1; - return rimem; + if (ASCENDING_TRAVERSE(order)) { + if (r1 < r2) { + pCheckInfo->chosen = 0; + return rmem; + } else { + pCheckInfo->chosen = 1; + return rimem; + } + } else { + if (r1 < r2) { + pCheckInfo->chosen = 1; + return rimem; + } else { + pCheckInfo->chosen = 0; + return rmem; + } + } } } + // at least one (rmem or rimem) is absent here if (rmem != NULL) { pCheckInfo->chosen = 0; return rmem; @@ -398,7 +415,7 @@ SDataRow getSDataRowInTableMem(STableCheckInfo* pCheckInfo) { return NULL; } -static bool moveToNextRow(STableCheckInfo* pCheckInfo) { +static bool moveToNextRowInMem(STableCheckInfo* pCheckInfo) { bool hasNext = false; if (pCheckInfo->chosen == 0) { if (pCheckInfo->iter != NULL) { @@ -412,19 +429,17 @@ static bool moveToNextRow(STableCheckInfo* pCheckInfo) { if (pCheckInfo->iiter != NULL) { return tSkipListIterGet(pCheckInfo->iiter) != NULL; } - } else { - if (pCheckInfo->chosen == 1) { - if (pCheckInfo->iiter != NULL) { - hasNext = tSkipListIterNext(pCheckInfo->iiter); - } + } else { //pCheckInfo->chosen == 1 + if (pCheckInfo->iiter != NULL) { + hasNext = tSkipListIterNext(pCheckInfo->iiter); + } - if (hasNext) { - return hasNext; - } + if (hasNext) { + return hasNext; + } - if (pCheckInfo->iter != NULL) { - return tSkipListIterGet(pCheckInfo->iter) != NULL; - } + if (pCheckInfo->iter != NULL) { + return tSkipListIterGet(pCheckInfo->iter) != NULL; } } @@ -445,7 +460,7 @@ static bool hasMoreDataInCache(STsdbQueryHandle* pHandle) { initTableMemIterator(pHandle, pCheckInfo); } - SDataRow row = getSDataRowInTableMem(pCheckInfo); + SDataRow row = getSDataRowInTableMem(pCheckInfo, pHandle->order); if (row == NULL) { return false; } @@ -650,7 +665,7 @@ static void handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SCompBlock* SDataBlockInfo binfo = GET_FILE_DATA_BLOCK_INFO(pCheckInfo, pBlock); /*bool hasData = */ initTableMemIterator(pQueryHandle, pCheckInfo); - SDataRow row = getSDataRowInTableMem(pCheckInfo); + SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order); TSKEY key = (row != NULL)? dataRowKey(row):TSKEY_INITIAL_VAL; cur->pos = ASCENDING_TRAVERSE(pQueryHandle->order)? 0:(binfo.rows-1); @@ -1033,7 +1048,7 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* } else if (pCheckInfo->iter != NULL || pCheckInfo->iiter != NULL) { SSkipListNode* node = NULL; do { - SDataRow row = getSDataRowInTableMem(pCheckInfo); + SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order); if (row == NULL) { break; } @@ -1061,9 +1076,9 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* cur->lastKey = key + step; cur->mixBlock = true; - moveToNextRow(pCheckInfo); + moveToNextRowInMem(pCheckInfo); } else if (key == tsArray[pos]) { // data in buffer has the same timestamp of data in file block, ignore it - moveToNextRow(pCheckInfo); + moveToNextRowInMem(pCheckInfo); } else if ((key > tsArray[pos] && ASCENDING_TRAVERSE(pQueryHandle->order)) || (key < tsArray[pos] && !ASCENDING_TRAVERSE(pQueryHandle->order))) { if (cur->win.skey == TSKEY_INITIAL_VAL) { @@ -1072,7 +1087,7 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* int32_t end = doBinarySearchKey(pCols->cols[0].pData, pCols->numOfRows, key, order); if (tsArray[end] == key) { // the value of key in cache equals to the end timestamp value, ignore it - moveToNextRow(pCheckInfo); + moveToNextRowInMem(pCheckInfo); } int32_t start = -1; @@ -1754,7 +1769,7 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int STable* pTable = pCheckInfo->pTableObj; do { - SDataRow row = getSDataRowInTableMem(pCheckInfo); + SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order); if (row == NULL) { break; } @@ -1775,11 +1790,11 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int copyOneRowFromMem(pQueryHandle, maxRowsToRead, numOfRows, row, pMeta, numOfCols, pTable); if (++numOfRows >= maxRowsToRead) { - moveToNextRow(pCheckInfo); + moveToNextRowInMem(pCheckInfo); break; } - } while(moveToNextRow(pCheckInfo)); + } while(moveToNextRowInMem(pCheckInfo)); assert(numOfRows <= maxRowsToRead); From 31349ebbae85249454f293feff7568fdda361a0d Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 20 Jul 2020 01:12:37 +0800 Subject: [PATCH 26/34] [td-225] fix memory leaks; --- src/query/src/qResultbuf.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/query/src/qResultbuf.c b/src/query/src/qResultbuf.c index f2194bbfb5..0eefb106b9 100644 --- a/src/query/src/qResultbuf.c +++ b/src/query/src/qResultbuf.c @@ -207,7 +207,8 @@ void destroyResultBuf(SDiskbasedResultBuf* pResultBuf, void* handle) { taosArrayDestroy(pResultBuf->list); taosHashCleanup(pResultBuf->idsTable); - + + tfree(pResultBuf->iBuf); tfree(pResultBuf); } From 5a448cd9a3b8c3b53fe627f730b36c4f4dbe796f Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 20 Jul 2020 11:10:22 +0800 Subject: [PATCH 27/34] [td-225] fix bugs in test cases. --- src/query/inc/qResultbuf.h | 4 ++-- src/query/tests/resultBufferTest.cpp | 5 ++--- src/util/inc/tutil.h | 4 ++-- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/query/inc/qResultbuf.h b/src/query/inc/qResultbuf.h index 41fccd177d..7df39c00fd 100644 --- a/src/query/inc/qResultbuf.h +++ b/src/query/inc/qResultbuf.h @@ -90,9 +90,9 @@ SIDList getDataBufPagesIdList(SDiskbasedResultBuf* pResultBuf, int32_t groupId); //#define getResBufPage(buf, id) ((tFilePage*)((buf)->pBuf + (buf)->pageSize * (id))) static FORCE_INLINE tFilePage* getResBufPage(SDiskbasedResultBuf* pResultBuf, int32_t id) { if (id < pResultBuf->inMemPages) { - return pResultBuf->iBuf + id * pResultBuf->pageSize; + return (tFilePage*) ((char*) pResultBuf->iBuf + id * pResultBuf->pageSize); } else { - return pResultBuf->pBuf + (id - pResultBuf->inMemPages) * pResultBuf->pageSize; + return (tFilePage*) ((char*) pResultBuf->pBuf + (id - pResultBuf->inMemPages) * pResultBuf->pageSize); } } /** diff --git a/src/query/tests/resultBufferTest.cpp b/src/query/tests/resultBufferTest.cpp index f822deeb73..63ed89ab9f 100644 --- a/src/query/tests/resultBufferTest.cpp +++ b/src/query/tests/resultBufferTest.cpp @@ -10,7 +10,7 @@ namespace { // simple test void simpleTest() { SDiskbasedResultBuf* pResultBuf = NULL; - int32_t ret = createDiskbasedResultBuffer(&pResultBuf, 1000, 64, NULL); + int32_t ret = createDiskbasedResultBuffer(&pResultBuf, 1000, 64, 1024, 4, NULL); int32_t pageId = 0; int32_t groupId = 0; @@ -22,8 +22,7 @@ void simpleTest() { ASSERT_EQ(getResBufSize(pResultBuf), 1000*16384L); SIDList list = getDataBufPagesIdList(pResultBuf, groupId); - ASSERT_EQ(list.size, 1); - + ASSERT_EQ(taosArrayGetSize(list), 1); ASSERT_EQ(getNumOfResultBufGroupId(pResultBuf), 1); destroyResultBuf(pResultBuf, NULL); diff --git a/src/util/inc/tutil.h b/src/util/inc/tutil.h index f7c69e3973..34f35c3807 100644 --- a/src/util/inc/tutil.h +++ b/src/util/inc/tutil.h @@ -35,12 +35,12 @@ extern "C" { #define WCHAR wchar_t #define tfree(x) \ - { \ + do { \ if (x) { \ free((void *)(x)); \ x = 0; \ } \ - } + } while(0); #define tstrncpy(dst, src, size) \ do { \ From ac1d0a266b39c5d8dfc2e17f78d0da51b0da67d7 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 20 Jul 2020 11:36:12 +0800 Subject: [PATCH 28/34] fix query unittest compile error. [TD-964] --- src/query/tests/resultBufferTest.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/query/tests/resultBufferTest.cpp b/src/query/tests/resultBufferTest.cpp index 9270c6e458..aaea0836d1 100644 --- a/src/query/tests/resultBufferTest.cpp +++ b/src/query/tests/resultBufferTest.cpp @@ -22,7 +22,7 @@ void simpleTest() { ASSERT_EQ(getResBufSize(pResultBuf), 1000*16384L); SIDList list = getDataBufPagesIdList(pResultBuf, groupId); - ASSERT_EQ(list.size, 1); + ASSERT_EQ(taosArrayGetSize(list), 1); ASSERT_EQ(getNumOfResultBufGroupId(pResultBuf), 1); From 9213011d184c77ecdb6dc2035693726352000200 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 20 Jul 2020 12:32:58 +0800 Subject: [PATCH 29/34] [td-225] add the stop check during scan files --- src/query/inc/qResultbuf.h | 1 - src/tsdb/inc/tsdbMain.h | 1 + src/tsdb/src/tsdbMemTable.c | 3 +-- src/tsdb/src/tsdbRead.c | 38 +++++++++++++++++++++++-------------- 4 files changed, 26 insertions(+), 17 deletions(-) diff --git a/src/query/inc/qResultbuf.h b/src/query/inc/qResultbuf.h index 7df39c00fd..d76659cc91 100644 --- a/src/query/inc/qResultbuf.h +++ b/src/query/inc/qResultbuf.h @@ -87,7 +87,6 @@ SIDList getDataBufPagesIdList(SDiskbasedResultBuf* pResultBuf, int32_t groupId); * @param id * @return */ -//#define getResBufPage(buf, id) ((tFilePage*)((buf)->pBuf + (buf)->pageSize * (id))) static FORCE_INLINE tFilePage* getResBufPage(SDiskbasedResultBuf* pResultBuf, int32_t id) { if (id < pResultBuf->inMemPages) { return (tFilePage*) ((char*) pResultBuf->iBuf + id * pResultBuf->pageSize); diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h index 40f2dac660..210d95853c 100644 --- a/src/tsdb/inc/tsdbMain.h +++ b/src/tsdb/inc/tsdbMain.h @@ -426,6 +426,7 @@ int tsdbUpdateFileHeader(SFile* pFile, uint32_t version); int tsdbEncodeSFileInfo(void** buf, const STsdbFileInfo* pInfo); void* tsdbDecodeSFileInfo(void* buf, STsdbFileInfo* pInfo); void tsdbRemoveFileGroup(STsdbRepo* pRepo, SFileGroup* pFGroup); +void tsdbGetFidKeyRange(int daysPerFile, int8_t precision, int fileId, TSKEY *minKey, TSKEY *maxKey); // ------------------ tsdbRWHelper.c #define TSDB_HELPER_CLEAR_STATE 0x0 // Clear state diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index b29cec3cf9..1b7db635b3 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -31,7 +31,6 @@ static int tsdbCommitMeta(STsdbRepo *pRepo); static void tsdbEndCommit(STsdbRepo *pRepo); static int tsdbHasDataToCommit(SCommitIter *iters, int nIters, TSKEY minKey, TSKEY maxKey); static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHelper *pHelper, SDataCols *pDataCols); -static void tsdbGetFidKeyRange(int daysPerFile, int8_t precision, int fileId, TSKEY *minKey, TSKEY *maxKey); static SCommitIter *tsdbCreateCommitIters(STsdbRepo *pRepo); static void tsdbDestroyCommitIters(SCommitIter *iters, int maxTables); @@ -544,7 +543,7 @@ static int tsdbHasDataToCommit(SCommitIter *iters, int nIters, TSKEY minKey, TSK return 0; } -static void tsdbGetFidKeyRange(int daysPerFile, int8_t precision, int fileId, TSKEY *minKey, TSKEY *maxKey) { +void tsdbGetFidKeyRange(int daysPerFile, int8_t precision, int fileId, TSKEY *minKey, TSKEY *maxKey) { *minKey = fileId * daysPerFile * tsMsPerDay[precision]; *maxKey = *minKey + daysPerFile * tsMsPerDay[precision] - 1; } diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 5b1afe3da8..37784577c4 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -128,8 +128,7 @@ typedef struct STsdbQueryHandle { static void changeQueryHandleForLastrowQuery(TsdbQueryHandleT pqHandle); static void changeQueryHandleForInterpQuery(TsdbQueryHandleT pHandle); -static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SCompBlock* pBlock, - SArray* sa); +static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SCompBlock* pBlock); static int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order); static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int maxRowsToRead, STimeWindow* win, STsdbQueryHandle* pQueryHandle); @@ -695,7 +694,7 @@ static void handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SCompBlock* } doLoadFileDataBlock(pQueryHandle, pBlock, pCheckInfo); - doMergeTwoLevelData(pQueryHandle, pCheckInfo, pBlock, pQueryHandle->defaultLoadColumn); + doMergeTwoLevelData(pQueryHandle, pCheckInfo, pBlock); } else { /* * no data in cache, only load data from file @@ -711,6 +710,7 @@ static void handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SCompBlock* cur->mixBlock = false; cur->blockCompleted = true; cur->lastKey = binfo.window.ekey + (ASCENDING_TRAVERSE(pQueryHandle->order)? 1:-1); + pCheckInfo->lastKey = cur->lastKey; } } @@ -734,7 +734,7 @@ static bool loadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock cur->pos = 0; } - doMergeTwoLevelData(pQueryHandle, pCheckInfo, pBlock, pQueryHandle->defaultLoadColumn); + doMergeTwoLevelData(pQueryHandle, pCheckInfo, pBlock); } else { // the whole block is loaded in to buffer handleDataMergeIfNeeded(pQueryHandle, pBlock, pCheckInfo); } @@ -751,7 +751,7 @@ static bool loadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock cur->pos = pBlock->numOfRows - 1; } - doMergeTwoLevelData(pQueryHandle, pCheckInfo, pBlock, pQueryHandle->defaultLoadColumn); + doMergeTwoLevelData(pQueryHandle, pCheckInfo, pBlock); } else { handleDataMergeIfNeeded(pQueryHandle, pBlock, pCheckInfo); } @@ -907,12 +907,12 @@ static int32_t copyDataFromFileBlock(STsdbQueryHandle* pQueryHandle, int32_t cap pQueryHandle->cur.win.ekey = tsArray[end]; pQueryHandle->cur.lastKey = tsArray[end] + step; - + return numOfRows + num; } static void copyOneRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity, int32_t numOfRows, SDataRow row, - STsdbMeta *pMeta, int32_t numOfCols, STable* pTable) { + int32_t numOfCols, STable* pTable) { char* pData = NULL; // the schema version info is embeded in SDataRow @@ -973,8 +973,7 @@ static void copyOneRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity, // only return the qualified data to client in terms of query time window, data rows in the same block but do not // be included in the query time window will be discarded -static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SCompBlock* pBlock, - SArray* sa) { +static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SCompBlock* pBlock) { SQueryFilePos* cur = &pQueryHandle->cur; SDataBlockInfo blockInfo = GET_FILE_DATA_BLOCK_INFO(pCheckInfo, pBlock); @@ -987,7 +986,6 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1:-1; int32_t numOfCols = taosArrayGetSize(pQueryHandle->pColumns); - STsdbMeta* pMeta = tsdbGetMeta(pQueryHandle->pTsdb); STable* pTable = pCheckInfo->pTableObj; int32_t endPos = cur->pos; @@ -1066,7 +1064,7 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* if ((key < tsArray[pos] && ASCENDING_TRAVERSE(pQueryHandle->order)) || (key > tsArray[pos] && !ASCENDING_TRAVERSE(pQueryHandle->order))) { - copyOneRowFromMem(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, row, pMeta, numOfCols, pTable); + copyOneRowFromMem(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, row, numOfCols, pTable); numOfRows += 1; if (cur->win.skey == TSKEY_INITIAL_VAL) { cur->win.skey = key; @@ -1406,8 +1404,21 @@ static int32_t getDataBlocksInFilesImpl(STsdbQueryHandle* pQueryHandle, bool* ex int32_t numOfBlocks = 0; int32_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); - + + STsdbCfg* pCfg = &pQueryHandle->pTsdb->config; + STimeWindow win = TSWINDOW_INITIALIZER; + while ((pQueryHandle->pFileGroup = tsdbGetFileGroupNext(&pQueryHandle->fileIter)) != NULL) { + tsdbGetFidKeyRange(pCfg->daysPerFile, pCfg->precision, pQueryHandle->pFileGroup->fileId, &win.skey, &win.ekey); + + // current file are not overlapped with query time window, ignore remain files + if ((ASCENDING_TRAVERSE(pQueryHandle->order) && win.skey > pQueryHandle->window.ekey) || + (!ASCENDING_TRAVERSE(pQueryHandle->order) && win.ekey < pQueryHandle->window.ekey)) { + tsdbDebug("%p remain files are not qualified for qrange:%"PRId64"-%"PRId64", ignore, %p", pQueryHandle, pQueryHandle->window.skey, pQueryHandle->window.ekey, pQueryHandle->qinfo) + pQueryHandle->pFileGroup = NULL; + break; + } + if ((code = getFileCompInfo(pQueryHandle, &numOfBlocks)) != TSDB_CODE_SUCCESS) { break; } @@ -1765,7 +1776,6 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int win->skey = TSKEY_INITIAL_VAL; int64_t st = taosGetTimestampUs(); - STsdbMeta* pMeta = tsdbGetMeta(pQueryHandle->pTsdb); STable* pTable = pCheckInfo->pTableObj; do { @@ -1787,7 +1797,7 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int } win->ekey = key; - copyOneRowFromMem(pQueryHandle, maxRowsToRead, numOfRows, row, pMeta, numOfCols, pTable); + copyOneRowFromMem(pQueryHandle, maxRowsToRead, numOfRows, row, numOfCols, pTable); if (++numOfRows >= maxRowsToRead) { moveToNextRowInMem(pCheckInfo); From e57e4a6d654d968b5588b620d3a867eaa4782f2a Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 20 Jul 2020 16:06:00 +0800 Subject: [PATCH 30/34] add gcov to support coverage test. --- src/query/tests/CMakeLists.txt | 4 ++-- src/util/tests/CMakeLists.txt | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/query/tests/CMakeLists.txt b/src/query/tests/CMakeLists.txt index 0ae8600756..1856223391 100644 --- a/src/query/tests/CMakeLists.txt +++ b/src/query/tests/CMakeLists.txt @@ -11,5 +11,5 @@ IF (HEADER_GTEST_INCLUDE_DIR AND LIB_GTEST_STATIC_DIR) AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) ADD_EXECUTABLE(queryTest ${SOURCE_LIST}) - TARGET_LINK_LIBRARIES(queryTest taos query gtest pthread) -ENDIF() \ No newline at end of file + TARGET_LINK_LIBRARIES(queryTest taos query gtest pthread gcov) +ENDIF() diff --git a/src/util/tests/CMakeLists.txt b/src/util/tests/CMakeLists.txt index 9f66eba37a..b0b5d3013b 100644 --- a/src/util/tests/CMakeLists.txt +++ b/src/util/tests/CMakeLists.txt @@ -11,5 +11,5 @@ IF (HEADER_GTEST_INCLUDE_DIR AND LIB_GTEST_STATIC_DIR) AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) ADD_EXECUTABLE(utilTest ${SOURCE_LIST}) - TARGET_LINK_LIBRARIES(utilTest tutil common gtest pthread) -ENDIF() \ No newline at end of file + TARGET_LINK_LIBRARIES(utilTest tutil common gtest pthread gcov) +ENDIF() From dfe2c4fc67756167c13cfebe0de4ca515e1514b8 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 20 Jul 2020 17:16:59 +0800 Subject: [PATCH 31/34] [td-225] fix memory leaks. --- src/query/inc/qResultbuf.h | 1 + src/query/src/qResultbuf.c | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/query/inc/qResultbuf.h b/src/query/inc/qResultbuf.h index d76659cc91..8c8afb0957 100644 --- a/src/query/inc/qResultbuf.h +++ b/src/query/inc/qResultbuf.h @@ -42,6 +42,7 @@ typedef struct SDiskbasedResultBuf { void* iBuf; // inmemory buf void* handle; // for debug purpose + void* emptyDummyIdList; // dummy id list } SDiskbasedResultBuf; #define DEFAULT_INTERN_BUF_PAGE_SIZE (1024L) diff --git a/src/query/src/qResultbuf.c b/src/query/src/qResultbuf.c index 0eefb106b9..de59676e59 100644 --- a/src/query/src/qResultbuf.c +++ b/src/query/src/qResultbuf.c @@ -36,6 +36,7 @@ int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t nu pResBuf->fd = FD_INITIALIZER; pResBuf->pBuf = NULL; + pResBuf->emptyDummyIdList = taosArrayInit(1, sizeof(int32_t)); qDebug("QInfo:%p create resBuf for output, page size:%d, initial pages:%d, %" PRId64 "bytes", handle, pResBuf->pageSize, pResBuf->numOfPages, pResBuf->totalBufSize); @@ -173,7 +174,7 @@ int32_t getNumOfRowsPerPage(SDiskbasedResultBuf* pResultBuf) { return pResultBuf SIDList getDataBufPagesIdList(SDiskbasedResultBuf* pResultBuf, int32_t groupId) { int32_t slot = getGroupIndex(pResultBuf, groupId); if (slot < 0) { - return taosArrayInit(1, sizeof(int32_t)); + return pResultBuf->emptyDummyIdList; } else { return taosArrayGetP(pResultBuf->list, slot); } @@ -206,6 +207,7 @@ void destroyResultBuf(SDiskbasedResultBuf* pResultBuf, void* handle) { } taosArrayDestroy(pResultBuf->list); + taosArrayDestroy(pResultBuf->emptyDummyIdList); taosHashCleanup(pResultBuf->idsTable); tfree(pResultBuf->iBuf); From ce0ffb408090d3ae64398e8d9fb77e1f9e12e33f Mon Sep 17 00:00:00 2001 From: liu0x54 Date: Tue, 21 Jul 2020 03:36:34 +0000 Subject: [PATCH 32/34] [TD-970] fix taosdemo bugs --- src/kit/taosdemo/taosdemo.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 305302b71a..79cfbe2a37 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -361,7 +361,7 @@ int main(int argc, char *argv[]) { arguments.num_of_DPT = 100000; arguments.num_of_RPR = 1000; arguments.use_metric = true; - arguments.insert_only = true; + arguments.insert_only = false; // end change argp_parse(&argp, argc, argv, 0, 0, &arguments); @@ -954,13 +954,13 @@ void *readMetric(void *sarg) { for (int i = 1; i <= m; i++) { if (i == 1) { - sprintf(tempS, "index = %d", i); + sprintf(tempS, "areaid = %d", i); } else { - sprintf(tempS, " or index = %d ", i); + sprintf(tempS, " or areaid = %d ", i); } strcat(condition, tempS); - sprintf(command, "select %s from m1 where %s", aggreFunc[j], condition); + sprintf(command, "select %s from meters where %s", aggreFunc[j], condition); printf("Where condition: %s\n", condition); fprintf(fp, "%s\n", command); From a0ca29dcfa3635079e9f646f9b7cfd7d22c3a7f0 Mon Sep 17 00:00:00 2001 From: liu0x54 Date: Tue, 21 Jul 2020 04:02:53 +0000 Subject: [PATCH 33/34] [TD-971] change create db sql --- tests/examples/python/read_example.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/examples/python/read_example.py b/tests/examples/python/read_example.py index a96115dfb2..73c4b95214 100644 --- a/tests/examples/python/read_example.py +++ b/tests/examples/python/read_example.py @@ -29,7 +29,7 @@ if __name__ == '__main__': # Create a database named db try: - c1.execute('create database db') + c1.execute('create database if not exists db ') except Exception as err: conn.close() raise(err) From 36f4948b2598b8d5b049d853feca577b73e21e5b Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Tue, 21 Jul 2020 05:45:44 +0000 Subject: [PATCH 34/34] set link secured except code is NOT_READY --- src/rpc/src/rpcMain.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index 2325d12d92..4790c22668 100644 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -446,7 +446,10 @@ void rpcSendResponse(const SRpcMsg *pRsp) { // set the idle timer to monitor the activity taosTmrReset(rpcProcessIdleTimer, pRpc->idleTime, pConn, pRpc->tmrCtrl, &pConn->pIdleTimer); rpcSendMsgToPeer(pConn, msg, msgLen); - pConn->secured = 1; // connection shall be secured + + // if not set to secured, set it expcet NOT_READY case, since client wont treat it as secured + if (pConn->secured == 0 && pMsg->code != TSDB_CODE_RPC_NOT_READY) + pConn->secured = 1; // connection shall be secured if (pConn->pReqMsg) rpcFreeCont(pConn->pReqMsg); pConn->pReqMsg = NULL;