From f349c8af87d86cd4878573c6c1a33b844b7992aa Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 26 Nov 2020 17:34:09 +0000 Subject: [PATCH 01/58] TD-2257 --- src/client/inc/tscUtil.h | 2 +- src/client/inc/tsclient.h | 2 +- src/client/src/tscServer.c | 40 ++++++++++++++------------------------ src/client/src/tscSql.c | 24 +++++++++++++++++++---- src/client/src/tscSystem.c | 5 ----- src/client/src/tscUtil.c | 8 ++++---- 6 files changed, 41 insertions(+), 40 deletions(-) diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index bde27d2932..cfcecc6582 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -278,7 +278,7 @@ bool hasMoreClauseToTry(SSqlObj* pSql); void tscTryQueryNextVnode(SSqlObj *pSql, __async_cb_func_t fp); void tscAsyncQuerySingleRowForNextVnode(void *param, TAOS_RES *tres, int numOfRows); void tscTryQueryNextClause(SSqlObj* pSql, __async_cb_func_t fp); -int tscSetMgmtEpSetFromCfg(const char *first, const char *second); +int tscSetMgmtEpSetFromCfg(const char *first, const char *second, SRpcCorEpSet *corEpSet); bool tscSetSqlOwner(SSqlObj* pSql); void tscClearSqlOwner(SSqlObj* pSql); diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index c482375bff..35f9b3af62 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -336,6 +336,7 @@ typedef struct STscObj { struct SSqlObj * pHb; struct SSqlObj * sqlList; struct SSqlStream *streamList; + SRpcCorEpSet *tscCorMgmtEpSet; void* pDnodeConn; pthread_mutex_t mutex; T_REF_DECLARE() @@ -515,7 +516,6 @@ extern int tsInsertHeadSize; extern int tscNumOfThreads; extern int tscRefId; -extern SRpcCorEpSet tscMgmtEpSet; extern int (*tscBuildMsg[TSDB_SQL_MAX])(SSqlObj *pSql, SSqlInfo *pInfo); diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 9735e26ece..3a2c673f34 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -26,7 +26,7 @@ #include "ttimer.h" #include "tlockfree.h" -SRpcCorEpSet tscMgmtEpSet; +///SRpcCorEpSet tscMgmtEpSet; int (*tscBuildMsg[TSDB_SQL_MAX])(SSqlObj *pSql, SSqlInfo *pInfo) = {0}; @@ -73,10 +73,11 @@ static void tscSetDnodeEpSet(SSqlObj* pSql, SVgroupInfo* pVgroupInfo) { assert(hasFqdn); } -static void tscDumpMgmtEpSet(SRpcEpSet *epSet) { - taosCorBeginRead(&tscMgmtEpSet.version); - *epSet = tscMgmtEpSet.epSet; - taosCorEndRead(&tscMgmtEpSet.version); +static void tscDumpMgmtEpSet(SSqlObj *pSql) { + SRpcCorEpSet *pCorEpSet = pSql->pTscObj->tscCorMgmtEpSet; + taosCorBeginRead(&pCorEpSet->version); + pSql->epSet = pCorEpSet->epSet; + taosCorEndRead(&pCorEpSet->version); } static void tscEpSetHtons(SRpcEpSet *s) { for (int32_t i = 0; i < s->numOfEps; i++) { @@ -94,11 +95,12 @@ bool tscEpSetIsEqual(SRpcEpSet *s1, SRpcEpSet *s2) { } return true; } -void tscUpdateMgmtEpSet(SRpcEpSet *pEpSet) { +void tscUpdateMgmtEpSet(SSqlObj *pSql, SRpcEpSet *pEpSet) { // no need to update if equal - taosCorBeginWrite(&tscMgmtEpSet.version); - tscMgmtEpSet.epSet = *pEpSet; - taosCorEndWrite(&tscMgmtEpSet.version); + SRpcCorEpSet *pCorEpSet = pSql->pTscObj->tscCorMgmtEpSet; + taosCorBeginWrite(&pCorEpSet->version); + pCorEpSet->epSet = *pEpSet; + taosCorEndWrite(&pCorEpSet->version); } static void tscDumpEpSetFromVgroupInfo(SCorVgroupInfo *pVgroupInfo, SRpcEpSet *pEpSet) { if (pVgroupInfo == NULL) { return;} @@ -133,18 +135,6 @@ static void tscUpdateVgroupInfo(SSqlObj *pObj, SRpcEpSet *pEpSet) { taosCorEndWrite(&pVgroupInfo->version); } -void tscPrintMgmtEp() { - SRpcEpSet dump; - tscDumpMgmtEpSet(&dump); - if (dump.numOfEps <= 0) { - tscError("invalid mnode EP list:%d", dump.numOfEps); - } else { - for (int i = 0; i < dump.numOfEps; ++i) { - tscDebug("mnode index:%d %s:%d", i, dump.fqdn[i], dump.port[i]); - } - } -} - void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) { STscObj *pObj = (STscObj *)param; if (pObj == NULL) return; @@ -162,7 +152,7 @@ void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) { SRpcEpSet * epSet = &pRsp->epSet; if (epSet->numOfEps > 0) { tscEpSetHtons(epSet); - tscUpdateMgmtEpSet(epSet); + tscUpdateMgmtEpSet(pSql, epSet); } pSql->pTscObj->connId = htonl(pRsp->connId); @@ -227,7 +217,7 @@ int tscSendMsgToServer(SSqlObj *pSql) { // set the mgmt ip list if (pSql->cmd.command >= TSDB_SQL_MGMT) { - tscDumpMgmtEpSet(&pSql->epSet); + tscDumpMgmtEpSet(pSql); } memcpy(pMsg, pSql->cmd.payload, pSql->cmd.payloadLen); @@ -289,7 +279,7 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) { if (pCmd->command < TSDB_SQL_MGMT) { tscUpdateVgroupInfo(pSql, pEpSet); } else { - tscUpdateMgmtEpSet(pEpSet); + tscUpdateMgmtEpSet(pSql, pEpSet); } } } @@ -2070,7 +2060,7 @@ int tscProcessConnectRsp(SSqlObj *pSql) { if (pConnect->epSet.numOfEps > 0) { tscEpSetHtons(&pConnect->epSet); - tscUpdateMgmtEpSet(&pConnect->epSet); + tscUpdateMgmtEpSet(pSql, &pConnect->epSet); } strcpy(pObj->sversion, pConnect->serverVersion); diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index de01776472..761a222b26 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -58,6 +58,7 @@ static SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pa terrno = TSDB_CODE_TSC_INVALID_USER_LENGTH; return NULL; } + SRpcCorEpSet corMgmtEpSet; char secretEncrypt[32] = {0}; int secretEncryptLen = 0; @@ -82,11 +83,13 @@ static SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pa } secretEncryptLen = outlen; } - + if (ip) { - if (tscSetMgmtEpSetFromCfg(ip, NULL) < 0) return NULL; - if (port) tscMgmtEpSet.epSet.port[0] = port; - } + if (tscSetMgmtEpSetFromCfg(ip, NULL, &corMgmtEpSet) < 0) return NULL; + if (port) corMgmtEpSet.epSet.port[0] = port; + } else { + if (tscSetMgmtEpSetFromCfg(tsFirst, tsSecond, &corMgmtEpSet) < 0) return NULL; + } void *pDnodeConn = NULL; if (tscInitRpc(user, secretEncrypt, &pDnodeConn) != 0) { @@ -100,10 +103,20 @@ static SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pa rpcClose(pDnodeConn); return NULL; } + // set up tscObj's mgmtEpSet + pObj->tscCorMgmtEpSet = (SRpcCorEpSet *)malloc(sizeof(SRpcCorEpSet)); + if (NULL == pObj->tscCorMgmtEpSet) { + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + rpcClose(pDnodeConn); + free(pObj->tscCorMgmtEpSet); + free(pObj); + } + memcpy(pObj->tscCorMgmtEpSet, &corMgmtEpSet, sizeof(SRpcCorEpSet)); pObj->signature = pObj; pObj->pDnodeConn = pDnodeConn; T_REF_INIT_VAL(pObj, 1); + tstrncpy(pObj->user, user, sizeof(pObj->user)); secretEncryptLen = MIN(secretEncryptLen, sizeof(pObj->pass)); @@ -115,6 +128,7 @@ static SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pa if (len >= TSDB_DB_NAME_LEN) { terrno = TSDB_CODE_TSC_INVALID_DB_LENGTH; rpcClose(pDnodeConn); + free(pObj->tscCorMgmtEpSet); free(pObj); return NULL; } @@ -132,6 +146,7 @@ static SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pa if (NULL == pSql) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; rpcClose(pDnodeConn); + free(pObj->tscCorMgmtEpSet); free(pObj); return NULL; } @@ -149,6 +164,7 @@ static SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pa terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; rpcClose(pDnodeConn); free(pSql); + free(pObj->tscCorMgmtEpSet); free(pObj); return NULL; } diff --git a/src/client/src/tscSystem.c b/src/client/src/tscSystem.c index 9e9a00550a..a5bc1ae7ad 100644 --- a/src/client/src/tscSystem.c +++ b/src/client/src/tscSystem.c @@ -116,11 +116,6 @@ void taos_init_imp(void) { taosInitNote(tsNumOfLogLines / 10, 1, (char*)"tsc_note"); } - if (tscSetMgmtEpSetFromCfg(tsFirst, tsSecond) < 0) { - tscError("failed to init mnode EP list"); - return; - } - tscInitMsgsFp(); int queueSize = tsMaxConnections*2; diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index d22c2588cc..077fd8aa00 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -867,7 +867,7 @@ void tscCloseTscObj(void *param) { rpcClose(pObj->pDnodeConn); pObj->pDnodeConn = NULL; } - + tfree(pObj->tscCorMgmtEpSet); pthread_mutex_destroy(&pObj->mutex); tscDebug("%p DB connection is closed, dnodeConn:%p", pObj, p); @@ -2459,10 +2459,10 @@ char* strdup_throw(const char* str) { return p; } -int tscSetMgmtEpSetFromCfg(const char *first, const char *second) { +int tscSetMgmtEpSetFromCfg(const char *first, const char *second, SRpcCorEpSet *corMgmtEpSet) { + corMgmtEpSet->version = 0; // init mgmt ip set - tscMgmtEpSet.version = 0; - SRpcEpSet *mgmtEpSet = &(tscMgmtEpSet.epSet); + SRpcEpSet *mgmtEpSet = &(corMgmtEpSet->epSet); mgmtEpSet->numOfEps = 0; mgmtEpSet->inUse = 0; From dcb0bc49e846890780044873cf7913df8539e1c7 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 1 Dec 2020 13:20:53 +0800 Subject: [PATCH 02/58] [TD-2129] --- src/client/src/tscFunctionImpl.c | 218 +++++++++++++++----- src/query/src/qExecutor.c | 339 ++++++++++++++++++++----------- 2 files changed, 394 insertions(+), 163 deletions(-) diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c index 3867032d46..56b7f052f7 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/client/src/tscFunctionImpl.c @@ -426,8 +426,7 @@ static void count_function_f(SQLFunctionCtx *pCtx, int32_t index) { } SET_VAL(pCtx, 1, 1); - - *((int64_t *)pCtx->aOutputBuf) += 1; + *((int64_t *)pCtx->aOutputBuf) += pCtx->size; // do not need it actually SResultRowCellInfo *pInfo = GET_RES_INFO(pCtx); @@ -3632,114 +3631,119 @@ static bool twa_function_setup(SQLFunctionCtx *pCtx) { return true; } -static int32_t twa_function_impl(SQLFunctionCtx* pCtx, int32_t index, int32_t size) { +static int32_t twa_function_impl(SQLFunctionCtx* pCtx, int32_t tsIndex, int32_t index, int32_t size) { int32_t notNullElems = 0; TSKEY *primaryKey = pCtx->ptsList; SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); STwaInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); + int32_t i = index; + int32_t step = GET_FORWARD_DIRECTION_FACTOR(pCtx->order); if (pCtx->start.key != INT64_MIN) { - assert(pCtx->start.key < primaryKey[index] && pInfo->lastKey == INT64_MIN); + assert((pCtx->start.key < primaryKey[tsIndex + i] && pCtx->order == TSDB_ORDER_ASC) || + (pCtx->start.key > primaryKey[tsIndex + i] && pCtx->order == TSDB_ORDER_DESC)); - pInfo->lastKey = primaryKey[index]; - GET_TYPED_DATA(pInfo->lastValue, double, pCtx->inputType, GET_INPUT_CHAR_INDEX(pCtx, 0)); + assert(pInfo->lastKey == INT64_MIN); + + pInfo->lastKey = primaryKey[tsIndex + i]; + GET_TYPED_DATA(pInfo->lastValue, double, pCtx->inputType, GET_INPUT_CHAR_INDEX(pCtx, index)); pInfo->dOutput += ((pInfo->lastValue + pCtx->start.val) / 2) * (pInfo->lastKey - pCtx->start.key); pInfo->hasResult = DATA_SET_FLAG; - pInfo->win.skey = pCtx->start.key; + pInfo->win.skey = pCtx->start.key; notNullElems++; - i += 1; + i += step; } else if (pInfo->lastKey == INT64_MIN) { - pInfo->lastKey = primaryKey[index]; - GET_TYPED_DATA(pInfo->lastValue, double, pCtx->inputType, GET_INPUT_CHAR_INDEX(pCtx, 0)); + pInfo->lastKey = primaryKey[tsIndex + i]; + GET_TYPED_DATA(pInfo->lastValue, double, pCtx->inputType, GET_INPUT_CHAR_INDEX(pCtx, index)); pInfo->hasResult = DATA_SET_FLAG; - pInfo->win.skey = pInfo->lastKey; + pInfo->win.skey = pInfo->lastKey; notNullElems++; - i += 1; + i += step; } // calculate the value of switch(pCtx->inputType) { case TSDB_DATA_TYPE_TINYINT: { - int8_t *val = (int8_t*) GET_INPUT_CHAR_INDEX(pCtx, index); - for (; i < size; i++) { + int8_t *val = (int8_t*) GET_INPUT_CHAR_INDEX(pCtx, 0); + for (; i < size && i >= 0; i += step) { if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { continue; } - pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i] - pInfo->lastKey); + pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + tsIndex] - pInfo->lastKey); pInfo->lastValue = val[i]; - pInfo->lastKey = primaryKey[i]; + pInfo->lastKey = primaryKey[i + tsIndex]; } break; } case TSDB_DATA_TYPE_SMALLINT: { - int16_t *val = (int16_t*) GET_INPUT_CHAR_INDEX(pCtx, index); - for (; i < size; i++) { + int16_t *val = (int16_t*) GET_INPUT_CHAR_INDEX(pCtx, 0); + for (; i < size && i >= 0; i += step) { if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { continue; } - pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i] - pInfo->lastKey); + pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + tsIndex] - pInfo->lastKey); pInfo->lastValue = val[i]; - pInfo->lastKey = primaryKey[i]; + pInfo->lastKey = primaryKey[i + tsIndex]; } break; } case TSDB_DATA_TYPE_INT: { - int32_t *val = (int32_t*) GET_INPUT_CHAR_INDEX(pCtx, index); - for (; i < size; i++) { + int32_t *val = (int32_t*) GET_INPUT_CHAR_INDEX(pCtx, 0); + for (; i < size && i >= 0; i += step) { if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { continue; } - pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i] - pInfo->lastKey); + pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + tsIndex] - pInfo->lastKey); pInfo->lastValue = val[i]; - pInfo->lastKey = primaryKey[i]; + pInfo->lastKey = primaryKey[i + tsIndex]; } break; } case TSDB_DATA_TYPE_BIGINT: { - int64_t *val = (int64_t*) GET_INPUT_CHAR_INDEX(pCtx, index); - for (; i < size; i++) { + int64_t *val = (int64_t*) GET_INPUT_CHAR_INDEX(pCtx, 0); + for (; i < size && i >= 0; i += step) { if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { continue; } - pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i] - pInfo->lastKey); + pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + tsIndex] - pInfo->lastKey); pInfo->lastValue = (double) val[i]; - pInfo->lastKey = primaryKey[i]; + pInfo->lastKey = primaryKey[i + tsIndex]; } break; } case TSDB_DATA_TYPE_FLOAT: { - float *val = (float*) GET_INPUT_CHAR_INDEX(pCtx, index); - for (; i < size; i++) { + float *val = (float*) GET_INPUT_CHAR_INDEX(pCtx, 0); + for (; i < size && i >= 0; i += step) { if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { continue; } - pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i] - pInfo->lastKey); + pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + tsIndex] - pInfo->lastKey); pInfo->lastValue = val[i]; - pInfo->lastKey = primaryKey[i]; + pInfo->lastKey = primaryKey[i + tsIndex]; } break; } case TSDB_DATA_TYPE_DOUBLE: { - double *val = (double*) GET_INPUT_CHAR_INDEX(pCtx, index); - for (; i < size; i++) { + double *val = (double*) GET_INPUT_CHAR_INDEX(pCtx, 0); + for (; i < size && i >= 0; i += step) { if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { continue; } - pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i] - pInfo->lastKey); + pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + tsIndex] - pInfo->lastKey); pInfo->lastValue = val[i]; - pInfo->lastKey = primaryKey[i]; + pInfo->lastKey = primaryKey[i + tsIndex]; } break; } @@ -3764,16 +3768,13 @@ static void twa_function(SQLFunctionCtx *pCtx) { STwaInfo * pInfo = GET_ROWCELL_INTERBUF(pResInfo); // skip null value - int32_t i = 0; + int32_t step = GET_FORWARD_DIRECTION_FACTOR(pCtx->order); + int32_t i = (pCtx->order == TSDB_ORDER_ASC)? 0:(pCtx->size - 1); while (pCtx->hasNull && i < pCtx->size && isNull((char *)data + pCtx->inputBytes * i, pCtx->inputType)) { - i++; + i += step; } - - if (i >= pCtx->size) { - return; - } - - int32_t notNullElems = twa_function_impl(pCtx, pCtx->startOffset, pCtx->size); + + int32_t notNullElems = twa_function_impl(pCtx, pCtx->startOffset, i, pCtx->size); SET_VAL(pCtx, notNullElems, 1); if (notNullElems > 0) { @@ -3791,11 +3792,136 @@ static void twa_function_f(SQLFunctionCtx *pCtx, int32_t index) { return; } - int32_t notNullElems = twa_function_impl(pCtx, index, 1); + int32_t notNullElems = 0; + TSKEY *primaryKey = pCtx->ptsList; + + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + + STwaInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); + int32_t i = pCtx->startOffset; + int32_t size = pCtx->size; + + if (pCtx->start.key != INT64_MIN) { + assert(pInfo->lastKey == INT64_MIN); + + pInfo->lastKey = primaryKey[index]; + GET_TYPED_DATA(pInfo->lastValue, double, pCtx->inputType, GET_INPUT_CHAR_INDEX(pCtx, index)); + + pInfo->dOutput += ((pInfo->lastValue + pCtx->start.val) / 2) * (pInfo->lastKey - pCtx->start.key); + + pInfo->hasResult = DATA_SET_FLAG; + pInfo->win.skey = pCtx->start.key; + notNullElems++; + i += 1; + } else if (pInfo->lastKey == INT64_MIN) { + pInfo->lastKey = primaryKey[index]; + GET_TYPED_DATA(pInfo->lastValue, double, pCtx->inputType, GET_INPUT_CHAR_INDEX(pCtx, index)); + + pInfo->hasResult = DATA_SET_FLAG; + pInfo->win.skey = pInfo->lastKey; + notNullElems++; + i += 1; + } + + // calculate the value of + switch(pCtx->inputType) { + case TSDB_DATA_TYPE_TINYINT: { + int8_t *val = (int8_t*) GET_INPUT_CHAR_INDEX(pCtx, index); + for (; i < size; i++) { + if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { + continue; + } + + pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + index] - pInfo->lastKey); + pInfo->lastValue = val[i]; + pInfo->lastKey = primaryKey[i + index]; + } + break; + } + case TSDB_DATA_TYPE_SMALLINT: { + int16_t *val = (int16_t*) GET_INPUT_CHAR_INDEX(pCtx, index); + for (; i < size; i++) { + if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { + continue; + } + + pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + index] - pInfo->lastKey); + pInfo->lastValue = val[i]; + pInfo->lastKey = primaryKey[i + index]; + } + break; + } + case TSDB_DATA_TYPE_INT: { + int32_t *val = (int32_t*) GET_INPUT_CHAR_INDEX(pCtx, index); + for (; i < size; i++) { + if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { + continue; + } + + pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + index] - pInfo->lastKey); + pInfo->lastValue = val[i]; + pInfo->lastKey = primaryKey[i + index]; + } + break; + } + case TSDB_DATA_TYPE_BIGINT: { + int64_t *val = (int64_t*) GET_INPUT_CHAR_INDEX(pCtx, index); + for (; i < size; i++) { + if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { + continue; + } + + pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + index] - pInfo->lastKey); + pInfo->lastValue = (double) val[i]; + pInfo->lastKey = primaryKey[i + index]; + } + break; + } + case TSDB_DATA_TYPE_FLOAT: { + float *val = (float*) GET_INPUT_CHAR_INDEX(pCtx, index); + for (; i < size; i++) { + if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { + continue; + } + + pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + index] - pInfo->lastKey); + pInfo->lastValue = val[i]; + pInfo->lastKey = primaryKey[i + index]; + } + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + double *val = (double*) GET_INPUT_CHAR_INDEX(pCtx, index); + for (; i < size; i++) { + if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { + continue; + } + + pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + index] - pInfo->lastKey); + pInfo->lastValue = val[i]; + pInfo->lastKey = primaryKey[i + index]; + } + break; + } + default: assert(0); + } + + // the last interpolated time window value + if (pCtx->end.key != INT64_MIN) { + pInfo->dOutput += ((pInfo->lastValue + pCtx->end.val) / 2) * (pCtx->end.key - pInfo->lastKey); + pInfo->lastValue = pCtx->end.val; + pInfo->lastKey = pCtx->end.key; + } + + pInfo->win.ekey = pInfo->lastKey; + SET_VAL(pCtx, notNullElems, 1); + if (notNullElems > 0) { + pResInfo->hasResult = DATA_SET_FLAG; + } + if (pCtx->stableQuery) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(STwaInfo)); } } diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index ec70280c7f..0133035a7b 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -196,6 +196,7 @@ static int32_t flushFromResultBuf(SQueryRuntimeEnv* pRuntimeEnv, SGroupResInfo* static int32_t checkForQueryBuf(size_t numOfTables); static void releaseQueryBuf(size_t numOfTables); static int32_t binarySearchForKey(char *pValue, int num, TSKEY key, int order); +static void doRowwiseTimeWindowInterpolation(SQueryRuntimeEnv* pRuntimeEnv, SArray* pDataBlock, TSKEY prevTs, int32_t prevRowIndex, TSKEY curTs, int32_t curRowIndex, TSKEY windowKey, int32_t type); bool doFilterData(SQuery *pQuery, int32_t elemPos) { for (int32_t k = 0; k < pQuery->numOfFilterCols; ++k) { @@ -660,7 +661,7 @@ static void setResultRowInterpo(SResultRow* pResult, SResultTsInterpType type) { } } -static bool isResultRowInterpo(SResultRow* pResult, SResultTsInterpType type) { +static bool resultRowInterpolated(SResultRow* pResult, SResultTsInterpType type) { assert(pResult != NULL && (type == RESULT_ROW_START_INTERP || type == RESULT_ROW_END_INTERP)); if (type == RESULT_ROW_START_INTERP) { return pResult->startInterp == true; @@ -989,7 +990,7 @@ static char *getDataBlock(SQueryRuntimeEnv *pRuntimeEnv, SArithmeticSupport *sas if (functionId == TSDB_FUNC_ARITHM) { sas->pArithExpr = &pQuery->pExpr1[col]; - sas->offset = (QUERY_IS_ASC_QUERY(pQuery)) ? pQuery->pos : pQuery->pos - (size - 1); + sas->offset = (QUERY_IS_ASC_QUERY(pQuery))? pQuery->pos : pQuery->pos - (size - 1); sas->colList = pQuery->colList; sas->numOfCols = pQuery->numOfCols; sas->data = calloc(pQuery->numOfCols, POINTER_BYTES); @@ -1032,85 +1033,89 @@ static char *getDataBlock(SQueryRuntimeEnv *pRuntimeEnv, SArithmeticSupport *sas return dataBlock; } -// window start key interpolation -static bool setTimeWindowInterpolationStartTs(SQueryRuntimeEnv* pRuntimeEnv, int32_t pos, SArray* pDataBlock, TSKEY* tsCols, STimeWindow* win) { - SQuery* pQuery = pRuntimeEnv->pQuery; - - TSKEY start = tsCols[pos]; - TSKEY lastTs = *(TSKEY *) pRuntimeEnv->prevRow[0]; - TSKEY prevTs = (pos == 0)? lastTs : tsCols[pos - 1]; - - // if lastTs == INT64_MIN, it is the first block, no need to do the start time interpolation - if (((lastTs != INT64_MIN && pos >= 0) || (lastTs == INT64_MIN && pos > 0)) && win->skey > lastTs && - win->skey < start) { - - for (int32_t k = 0; k < pQuery->numOfCols; ++k) { - SColumnInfoData *pColInfo = taosArrayGet(pDataBlock, k); - if (k == 0 && pColInfo->info.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { - assert(pColInfo->info.type == TSDB_DATA_TYPE_TIMESTAMP); - continue; - } - - double v1 = 0, v2 = 0, v = 0; - - char *prevVal = pos == 0 ? pRuntimeEnv->prevRow[k] : ((char*)pColInfo->pData) + (pos - 1) * pColInfo->info.bytes; - - GET_TYPED_DATA(v1, double, pColInfo->info.type, (char *)prevVal); - GET_TYPED_DATA(v2, double, pColInfo->info.type, (char *)pColInfo->pData + pos * pColInfo->info.bytes); - - SPoint point1 = (SPoint){.key = prevTs, .val = &v1}; - SPoint point2 = (SPoint){.key = start, .val = &v2}; - SPoint point = (SPoint){.key = win->skey, .val = &v}; - taosGetLinearInterpolationVal(TSDB_DATA_TYPE_DOUBLE, &point1, &point2, &point); - pRuntimeEnv->pCtx[k].start.key = point.key; - pRuntimeEnv->pCtx[k].start.val = v; +static void setNotInterpoWindowKey(SQLFunctionCtx* pCtx, int32_t numOfOutput, int32_t type) { + if (type == RESULT_ROW_START_INTERP) { + for (int32_t k = 0; k < numOfOutput; ++k) { + pCtx[k].start.key = INT64_MIN; } - - return true; } else { - for (int32_t k = 0; k < pQuery->numOfCols; ++k) { - pRuntimeEnv->pCtx[k].start.key = INT64_MIN; + for (int32_t k = 0; k < numOfOutput; ++k) { + pCtx[k].end.key = INT64_MIN; } - - return false; } } -static bool setTimeWindowInterpolationEndTs(SQueryRuntimeEnv* pRuntimeEnv, int32_t pos, SArray* pDataBlock, TSKEY* tsCols, TSKEY ekey, STimeWindow* win) { +//static double getTSWindowInterpoVal(SColumnInfoData* pColInfo, int16_t srcColIndex, int16_t rowIndex, TSKEY key, char** prevRow, TSKEY* tsCols, int32_t step) { +// TSKEY start = tsCols[rowIndex]; +// TSKEY prevTs = (rowIndex == 0)? *(TSKEY *) prevRow[0] : tsCols[rowIndex - step]; +// +// double v1 = 0, v2 = 0, v = 0; +// char *prevVal = (rowIndex == 0)? prevRow[srcColIndex] : ((char*)pColInfo->pData) + (rowIndex - step) * pColInfo->info.bytes; +// +// GET_TYPED_DATA(v1, double, pColInfo->info.type, (char *)prevVal); +// GET_TYPED_DATA(v2, double, pColInfo->info.type, (char *)pColInfo->pData + rowIndex * pColInfo->info.bytes); +// +// SPoint point1 = (SPoint){.key = prevTs, .val = &v1}; +// SPoint point2 = (SPoint){.key = start, .val = &v2}; +// SPoint point = (SPoint){.key = key, .val = &v}; +// taosGetLinearInterpolationVal(TSDB_DATA_TYPE_DOUBLE, &point1, &point2, &point); +// +// return v; +//} + +// window start key interpolation +static bool setTimeWindowInterpolationStartTs(SQueryRuntimeEnv* pRuntimeEnv, int32_t pos, int32_t numOfRows, SArray* pDataBlock, TSKEY* tsCols, STimeWindow* win) { SQuery* pQuery = pRuntimeEnv->pQuery; - TSKEY trueEndKey = tsCols[pos]; - if (win->ekey < ekey && win->ekey != trueEndKey) { - int32_t nextIndex = pos + 1; - TSKEY next = tsCols[nextIndex]; - - for (int32_t k = 0; k < pQuery->numOfCols; ++k) { - SColumnInfoData *pColInfo = taosArrayGet(pDataBlock, k); - if (k == 0 && pColInfo->info.type == TSDB_DATA_TYPE_TIMESTAMP && - pColInfo->info.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { - continue; - } - - double v1 = 0, v2 = 0, v = 0; - GET_TYPED_DATA(v1, double, pColInfo->info.type, (char *)pColInfo->pData + pos * pColInfo->info.bytes); - GET_TYPED_DATA(v2, double, pColInfo->info.type, (char *)pColInfo->pData + nextIndex * pColInfo->info.bytes); - - SPoint point1 = (SPoint){.key = trueEndKey, .val = &v1}; - SPoint point2 = (SPoint){.key = next, .val = &v2}; - SPoint point = (SPoint){.key = win->ekey, .val = &v}; - taosGetLinearInterpolationVal(TSDB_DATA_TYPE_DOUBLE, &point1, &point2, &point); - pRuntimeEnv->pCtx[k].end.key = point.key; - pRuntimeEnv->pCtx[k].end.val = v; - } + TSKEY curTs = tsCols[pos]; + TSKEY lastTs = *(TSKEY *) pRuntimeEnv->prevRow[0]; + // lastTs == INT64_MIN and pos == 0 means this is the first time window, interpolation is not needed. + // start exactly from this point, no need to do interpolation + TSKEY key = QUERY_IS_ASC_QUERY(pQuery)? win->skey:win->ekey; + if (key == curTs) { + setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfCols, RESULT_ROW_START_INTERP); return true; - } else { // current time window does not ended in current data block, do nothing - for (int32_t k = 0; k < pQuery->numOfCols; ++k) { - pRuntimeEnv->pCtx[k].end.key = INT64_MIN; - } + } + if (lastTs == INT64_MIN && ((pos == 0 && QUERY_IS_ASC_QUERY(pQuery)) || (pos == (numOfRows - 1) && !QUERY_IS_ASC_QUERY(pQuery)))) { + setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfCols, RESULT_ROW_START_INTERP); + return true; + } + + int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); + TSKEY prevTs = ((pos == 0 && QUERY_IS_ASC_QUERY(pQuery)) || (pos == (numOfRows - 1) && !QUERY_IS_ASC_QUERY(pQuery)))? + lastTs:tsCols[pos - step]; + + doRowwiseTimeWindowInterpolation(pRuntimeEnv, pDataBlock, prevTs, pos - step, curTs, pos, key, RESULT_ROW_START_INTERP); + return true; +} + +static bool setTimeWindowInterpolationEndTs(SQueryRuntimeEnv* pRuntimeEnv, int32_t endRowIndex, SArray* pDataBlock, TSKEY* tsCols, TSKEY blockEkey, STimeWindow* win) { + SQuery* pQuery = pRuntimeEnv->pQuery; + TSKEY actualEndKey = tsCols[endRowIndex]; + + TSKEY key = QUERY_IS_ASC_QUERY(pQuery)? win->ekey:win->skey; + + // not ended in current data block, do not invoke interpolation + if ((key > blockEkey && QUERY_IS_ASC_QUERY(pQuery)) || (key < blockEkey && !QUERY_IS_ASC_QUERY(pQuery))) { + setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfCols, RESULT_ROW_END_INTERP); return false; } + + // there is actual end point of current time window, no interpolation need + if (key == actualEndKey) { + setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfCols, RESULT_ROW_END_INTERP); + return true; + } + + int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); + int32_t nextRowIndex = endRowIndex + step; + assert(nextRowIndex >= 0); + + TSKEY nextKey = tsCols[nextRowIndex]; + doRowwiseTimeWindowInterpolation(pRuntimeEnv, pDataBlock, actualEndKey, endRowIndex, nextKey, nextRowIndex, key, RESULT_ROW_END_INTERP); + return true; } static void saveDataBlockLastRow(SQueryRuntimeEnv* pRuntimeEnv, SDataBlockInfo* pDataBlockInfo, SArray* pDataBlock) { @@ -1119,10 +1124,10 @@ static void saveDataBlockLastRow(SQueryRuntimeEnv* pRuntimeEnv, SDataBlockInfo* } SQuery* pQuery = pRuntimeEnv->pQuery; + int32_t rowIndex = QUERY_IS_ASC_QUERY(pQuery)? pDataBlockInfo->rows-1:0; for (int32_t k = 0; k < pQuery->numOfCols; ++k) { SColumnInfoData *pColInfo = taosArrayGet(pDataBlock, k); - memcpy(pRuntimeEnv->prevRow[k], ((char*)pColInfo->pData) + (pColInfo->info.bytes * (pDataBlockInfo->rows - 1)), - pColInfo->info.bytes); + memcpy(pRuntimeEnv->prevRow[k], ((char*)pColInfo->pData) + (pColInfo->info.bytes * rowIndex), pColInfo->info.bytes); } } @@ -1174,11 +1179,13 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis * int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); if (QUERY_IS_INTERVAL_QUERY(pQuery)) { + int32_t prevIndex = curTimeWindowIndex(pWindowResInfo); + TSKEY ts = getStartTsKey(pQuery, pDataBlockInfo, tsCols, step); + STimeWindow win = getActiveTimeWindow(pWindowResInfo, ts, pQuery); bool hasTimeWindow = false; SResultRow* pResult = NULL; - STimeWindow win = getActiveTimeWindow(pWindowResInfo, ts, pQuery); int32_t ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo, &win, masterScan, &hasTimeWindow, &pResult); if (ret != TSDB_CODE_SUCCESS) { tfree(sasArray); @@ -1193,20 +1200,47 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis * TSKEY ekey = reviseWindowEkey(pQuery, &win); forwardStep = getNumOfRowsInTimeWindow(pQuery, pDataBlockInfo, tsCols, pQuery->pos, ekey, searchFn, true); + // prev time window not interpolation yet. + int32_t curIndex = curTimeWindowIndex(pWindowResInfo); + if (prevIndex != -1 && prevIndex < curIndex) { + for(int32_t j = prevIndex; j < curIndex; ++j) { + SResultRow *pRes = pWindowResInfo->pResult[j]; + + STimeWindow w = pRes->win; + ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo, &w, masterScan, &hasTimeWindow, &pResult); + assert(ret == TSDB_CODE_SUCCESS && !resultRowInterpolated(pResult, RESULT_ROW_END_INTERP)); + + int32_t p = QUERY_IS_ASC_QUERY(pQuery)? 0:pDataBlockInfo->rows-1; + doRowwiseTimeWindowInterpolation(pRuntimeEnv, pDataBlock, *(TSKEY*) pRuntimeEnv->prevRow[0], -1, tsCols[0], p, w.ekey, RESULT_ROW_END_INTERP); + setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); + setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfOutput, RESULT_ROW_START_INTERP); + + bool closed = getResultRowStatus(pWindowResInfo, curTimeWindowIndex(pWindowResInfo)); + doBlockwiseApplyFunctions(pRuntimeEnv, closed, &w, startPos, 0, tsCols, pDataBlockInfo->rows); + } + + // restore current time window + ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo, &win, masterScan, &hasTimeWindow, &pResult); + assert (ret == TSDB_CODE_SUCCESS); // null data, too many state code + } + // window start key interpolation if (pRuntimeEnv->timeWindowInterpo) { - bool alreadyInterp = isResultRowInterpo(pResult, RESULT_ROW_START_INTERP); - if (!alreadyInterp) { - bool interp = setTimeWindowInterpolationStartTs(pRuntimeEnv, pQuery->pos, pDataBlock, tsCols, &win); + bool done = resultRowInterpolated(pResult, RESULT_ROW_START_INTERP); + if (!done) { + int32_t startRowIndex = pQuery->pos; + bool interp = setTimeWindowInterpolationStartTs(pRuntimeEnv, startRowIndex, pDataBlockInfo->rows, pDataBlock, tsCols, &win); if (interp) { setResultRowInterpo(pResult, RESULT_ROW_START_INTERP); } } - alreadyInterp = isResultRowInterpo(pResult, RESULT_ROW_END_INTERP); - if (!alreadyInterp) { - bool interp = setTimeWindowInterpolationEndTs(pRuntimeEnv, pQuery->pos + forwardStep - 1, pDataBlock, tsCols, - pDataBlockInfo->window.ekey, &win); + done = resultRowInterpolated(pResult, RESULT_ROW_END_INTERP); + if (!done) { + int32_t endRowIndex = pQuery->pos + (forwardStep - 1) * step; + + TSKEY endKey = QUERY_IS_ASC_QUERY(pQuery)? pDataBlockInfo->window.ekey:pDataBlockInfo->window.skey; + bool interp = setTimeWindowInterpolationEndTs(pRuntimeEnv, endRowIndex, pDataBlock, tsCols, endKey, &win); if (interp) { setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); } @@ -1243,17 +1277,20 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis * // window start(end) key interpolation if (pRuntimeEnv->timeWindowInterpo) { - bool alreadyInterp = isResultRowInterpo(pResult, RESULT_ROW_START_INTERP); - if (!alreadyInterp) { - bool interp = setTimeWindowInterpolationStartTs(pRuntimeEnv, startPos, pDataBlock, tsCols, &nextWin); + bool done = resultRowInterpolated(pResult, RESULT_ROW_START_INTERP); + if (!done) { + int32_t startRowIndex = startPos; + bool interp = setTimeWindowInterpolationStartTs(pRuntimeEnv, startRowIndex, pDataBlockInfo->rows, pDataBlock, tsCols, &nextWin); if (interp) { setResultRowInterpo(pResult, RESULT_ROW_START_INTERP); } } - alreadyInterp = isResultRowInterpo(pResult, RESULT_ROW_END_INTERP); - if (!alreadyInterp) { - bool interp = setTimeWindowInterpolationEndTs(pRuntimeEnv, startPos + forwardStep - 1, pDataBlock, tsCols, pDataBlockInfo->window.ekey, &nextWin); + done = resultRowInterpolated(pResult, RESULT_ROW_END_INTERP); + if (!done) { + int32_t endRowIndex = startPos + (forwardStep - 1)*step; + TSKEY endKey = QUERY_IS_ASC_QUERY(pQuery)? pDataBlockInfo->window.ekey:pDataBlockInfo->window.skey; + bool interp = setTimeWindowInterpolationEndTs(pRuntimeEnv, endRowIndex, pDataBlock, tsCols, endKey, &nextWin); if (interp) { setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); } @@ -1459,6 +1496,45 @@ static bool functionNeedToExecute(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx return true; } +void doRowwiseTimeWindowInterpolation(SQueryRuntimeEnv* pRuntimeEnv, SArray* pDataBlock, TSKEY prevTs, int32_t prevRowIndex, TSKEY curTs, int32_t curRowIndex, TSKEY windowKey, int32_t type) { + SQuery* pQuery = pRuntimeEnv->pQuery; + for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { + int32_t functionId = pQuery->pExpr1[k].base.functionId; + if (functionId != TSDB_FUNC_TWA) { + pRuntimeEnv->pCtx[k].start.key = INT64_MIN; + continue; + } + + SColIndex* pColIndex = &pQuery->pExpr1[k].base.colInfo; + int16_t index = pColIndex->colIndex; + SColumnInfoData* pColInfo = taosArrayGet(pDataBlock, index); + + assert(pColInfo->info.colId == pColIndex->colId && curTs != windowKey); + double v1 = 0, v2 = 0, v = 0; + + if (prevRowIndex == -1) { + GET_TYPED_DATA(v1, double, pColInfo->info.type, (char *)pRuntimeEnv->prevRow[k]); + } else { + GET_TYPED_DATA(v1, double, pColInfo->info.type, (char *)pColInfo->pData + prevRowIndex * pColInfo->info.bytes); + } + + GET_TYPED_DATA(v2, double, pColInfo->info.type, (char *)pColInfo->pData + curRowIndex * pColInfo->info.bytes); + + SPoint point1 = (SPoint){.key = prevTs, .val = &v1}; + SPoint point2 = (SPoint){.key = curTs, .val = &v2}; + SPoint point = (SPoint){.key = windowKey, .val = &v}; + taosGetLinearInterpolationVal(TSDB_DATA_TYPE_DOUBLE, &point1, &point2, &point); + + if (type == RESULT_ROW_START_INTERP) { + pRuntimeEnv->pCtx[k].start.key = point.key; + pRuntimeEnv->pCtx[k].start.val = v; + } else { + pRuntimeEnv->pCtx[k].end.key = point.key; + pRuntimeEnv->pCtx[k].end.val = v; + } + } +} + static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pStatis, SDataBlockInfo *pDataBlockInfo, SWindowResInfo *pWindowResInfo, SArray *pDataBlock) { SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; @@ -1489,6 +1565,7 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { char *dataBlock = getDataBlock(pRuntimeEnv, &sasArray[k], k, pDataBlockInfo->rows, pDataBlock); setExecParams(pQuery, &pCtx[k], dataBlock, tsCols, pDataBlockInfo, pStatis, &sasArray[k], k, pQInfo->vgId); + pCtx[k].size = 1; } // set the input column data @@ -1508,7 +1585,8 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS } int32_t offset = -1; -// TSKEY prev = -1; + TSKEY prevTs = *(TSKEY*) pRuntimeEnv->prevRow[0]; + int32_t prevRowIndex = -1; for (int32_t j = 0; j < pDataBlockInfo->rows; ++j) { offset = GET_COL_DATA_POS(pQuery, j, step); @@ -1530,7 +1608,9 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS // interval window query, decide the time window according to the primary timestamp if (QUERY_IS_INTERVAL_QUERY(pQuery)) { - int64_t ts = tsCols[offset]; + int32_t prevWindowIndex = curTimeWindowIndex(pWindowResInfo); + + int64_t ts = tsCols[offset]; STimeWindow win = getActiveTimeWindow(pWindowResInfo, ts, pQuery); bool hasTimeWindow = false; @@ -1543,27 +1623,58 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS if (!hasTimeWindow) { continue; } -/* + // window start key interpolation if (pRuntimeEnv->timeWindowInterpo) { - bool alreadyInterp = isResultRowInterpo(pResult, RESULT_ROW_START_INTERP); - if (!alreadyInterp) { - bool interp = setTimeWindowInterpolationStartTs(pRuntimeEnv, pos, pDataBlock, tsCols, &win); - if (interp) { - setResultRowInterpo(pResult, RESULT_ROW_START_INTERP); + // check for the time window end time interpolation + int32_t curIndex = curTimeWindowIndex(pWindowResInfo); + if (prevWindowIndex != -1 && prevWindowIndex < curIndex) { + for (int32_t k = prevWindowIndex; k < curIndex; ++k) { + SResultRow *pRes = pWindowResInfo->pResult[k]; + STimeWindow w = pRes->win; + ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo, &w, masterScan, &hasTimeWindow, &pResult); + assert(ret == TSDB_CODE_SUCCESS && !resultRowInterpolated(pResult, RESULT_ROW_END_INTERP)); + + TSKEY key = QUERY_IS_ASC_QUERY(pQuery)? w.ekey:w.skey; + doRowwiseTimeWindowInterpolation(pRuntimeEnv, pDataBlock, prevTs, prevRowIndex, ts, offset, key, RESULT_ROW_END_INTERP); + setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); + + setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfCols, RESULT_ROW_START_INTERP); + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { + pRuntimeEnv->pCtx[i].size = 0; + } + + bool closed = getResultRowStatus(pWindowResInfo, curTimeWindowIndex(pWindowResInfo)); + doRowwiseApplyFunctions(pRuntimeEnv, closed, &w, offset); + } + + // restore current time window + ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo, &win, masterScan, &hasTimeWindow, + &pResult); + if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code + continue; } } - alreadyInterp = isResultRowInterpo(pResult, RESULT_ROW_END_INTERP); - if (!alreadyInterp) { - bool interp = setTimeWindowInterpolationEndTs(pRuntimeEnv, pQuery->pos + forwardStep - 1, pDataBlock, tsCols, - pDataBlockInfo->window.ekey, &win); - if (interp) { - setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); + bool done = resultRowInterpolated(pResult, RESULT_ROW_START_INTERP); + if (!done) { + TSKEY key = QUERY_IS_ASC_QUERY(pQuery)? win.skey:win.ekey; + if (prevTs != INT64_MIN && ts != key) { + doRowwiseTimeWindowInterpolation(pRuntimeEnv, pDataBlock, prevTs, prevRowIndex, ts, offset, key, RESULT_ROW_START_INTERP); + setResultRowInterpo(pResult, RESULT_ROW_START_INTERP); + } else { + setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfCols, RESULT_ROW_START_INTERP); } + + setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfCols, RESULT_ROW_END_INTERP); + for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { + pRuntimeEnv->pCtx[k].size = 1; + } + } else { + setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfCols, RESULT_ROW_START_INTERP); } } -*/ + bool closed = getResultRowStatus(pWindowResInfo, curTimeWindowIndex(pWindowResInfo)); doRowwiseApplyFunctions(pRuntimeEnv, closed, &win, offset); @@ -1588,26 +1699,19 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS } if (hasTimeWindow) { -/* - // window start(end) key interpolation - if (pRuntimeEnv->timeWindowInterpo) { - bool alreadyInterp = isResultRowInterpo(pResult, RESULT_ROW_START_INTERP); - if (!alreadyInterp) { - bool interp = setTimeWindowInterpolationStartTs(pRuntimeEnv, startPos, pDataBlock, tsCols, &nextWin); - if (interp) { - setResultRowInterpo(pResult, RESULT_ROW_START_INTERP); - } + bool done = resultRowInterpolated(pResult, RESULT_ROW_START_INTERP); + if (!done) { + if (prevTs != INT64_MIN && ((QUERY_IS_ASC_QUERY(pQuery) && (prevTs < nextWin.skey)) || (!QUERY_IS_ASC_QUERY(pQuery) && prevTs > nextWin.ekey))) { + TSKEY key = QUERY_IS_ASC_QUERY(pQuery)? nextWin.skey:nextWin.ekey; + doRowwiseTimeWindowInterpolation(pRuntimeEnv, pDataBlock, prevTs, prevRowIndex, ts, offset, key, RESULT_ROW_START_INTERP); + setResultRowInterpo(pResult, RESULT_ROW_START_INTERP); + } else { + setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfCols, RESULT_ROW_START_INTERP); } - alreadyInterp = isResultRowInterpo(pResult, RESULT_ROW_END_INTERP); - if (!alreadyInterp) { - bool interp = setTimeWindowInterpolationEndTs(pRuntimeEnv, startPos + forwardStep - 1, pDataBlock, tsCols, pDataBlockInfo->window.ekey, &nextWin); - if (interp) { - setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); - } - } + setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfCols, RESULT_ROW_END_INTERP); } -*/ + closed = getResultRowStatus(pWindowResInfo, curTimeWindowIndex(pWindowResInfo)); doRowwiseApplyFunctions(pRuntimeEnv, closed, &nextWin, offset); } @@ -1633,7 +1737,8 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS } } -// prev = tsCols[offset]; + prevTs = tsCols[offset]; + prevRowIndex = offset; if (pRuntimeEnv->pTSBuf != NULL) { // if timestamp filter list is empty, quit current query From c3c7643fac71a9b844234df2585f768eefe078e8 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 1 Dec 2020 16:59:25 +0800 Subject: [PATCH 03/58] [TD-2277]: fix invalid column type in stable query. --- src/client/src/tscLocalMerge.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/client/src/tscLocalMerge.c b/src/client/src/tscLocalMerge.c index 2cd37013c5..9fdadfa957 100644 --- a/src/client/src/tscLocalMerge.c +++ b/src/client/src/tscLocalMerge.c @@ -721,10 +721,16 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr // final result depends on the fields number memset(pSchema, 0, sizeof(SSchema) * size); + for (int32_t i = 0; i < size; ++i) { SSqlExpr *pExpr = tscSqlExprGet(pQueryInfo, i); - SSchema *p1 = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, pExpr->colInfo.colIndex); + SSchema p1 = {0}; + if (pExpr->colInfo.colIndex != TSDB_TBNAME_COLUMN_INDEX) { + p1 = *tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, pExpr->colInfo.colIndex); + } else { + p1 = tGetTableNameColumnSchema(); + } int32_t inter = 0; int16_t type = -1; @@ -743,7 +749,8 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr functionId = TSDB_FUNC_LAST; } - getResultDataInfo(p1->type, p1->bytes, functionId, 0, &type, &bytes, &inter, 0, false); + int32_t ret = getResultDataInfo(p1.type, p1.bytes, functionId, 0, &type, &bytes, &inter, 0, false); + assert(ret == TSDB_CODE_SUCCESS); } pSchema[i].type = (uint8_t)type; From 43a3f513c4bd14f3ce702941451f641704bbfd51 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 1 Dec 2020 17:01:44 +0800 Subject: [PATCH 04/58] [TD-2129] --- src/query/src/qExecutor.c | 96 +++++----- tests/script/general/parser/function.sim | 228 +++++++++++++++++++++++ 2 files changed, 277 insertions(+), 47 deletions(-) create mode 100644 tests/script/general/parser/function.sim diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 0133035a7b..6de57b32fd 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -1074,12 +1074,12 @@ static bool setTimeWindowInterpolationStartTs(SQueryRuntimeEnv* pRuntimeEnv, int // start exactly from this point, no need to do interpolation TSKEY key = QUERY_IS_ASC_QUERY(pQuery)? win->skey:win->ekey; if (key == curTs) { - setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfCols, RESULT_ROW_START_INTERP); + setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfOutput, RESULT_ROW_START_INTERP); return true; } if (lastTs == INT64_MIN && ((pos == 0 && QUERY_IS_ASC_QUERY(pQuery)) || (pos == (numOfRows - 1) && !QUERY_IS_ASC_QUERY(pQuery)))) { - setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfCols, RESULT_ROW_START_INTERP); + setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfOutput, RESULT_ROW_START_INTERP); return true; } @@ -1099,13 +1099,13 @@ static bool setTimeWindowInterpolationEndTs(SQueryRuntimeEnv* pRuntimeEnv, int32 // not ended in current data block, do not invoke interpolation if ((key > blockEkey && QUERY_IS_ASC_QUERY(pQuery)) || (key < blockEkey && !QUERY_IS_ASC_QUERY(pQuery))) { - setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfCols, RESULT_ROW_END_INTERP); + setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfOutput, RESULT_ROW_END_INTERP); return false; } // there is actual end point of current time window, no interpolation need if (key == actualEndKey) { - setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfCols, RESULT_ROW_END_INTERP); + setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfOutput, RESULT_ROW_END_INTERP); return true; } @@ -1535,6 +1535,43 @@ void doRowwiseTimeWindowInterpolation(SQueryRuntimeEnv* pRuntimeEnv, SArray* pDa } } +static void setTimeWindowSKeyInterp(SQueryRuntimeEnv* pRuntimeEnv, SArray* pDataBlock, TSKEY prevTs, int32_t prevRowIndex, TSKEY ts, int32_t offset, SResultRow* pResult, STimeWindow* win) { + SQuery* pQuery = pRuntimeEnv->pQuery; + + bool done = resultRowInterpolated(pResult, RESULT_ROW_START_INTERP); + if (!done) { + TSKEY key = QUERY_IS_ASC_QUERY(pQuery)? win->skey:win->ekey; + if (key == ts) { + setResultRowInterpo(pResult, RESULT_ROW_START_INTERP); + } else if (prevTs != INT64_MIN && ((QUERY_IS_ASC_QUERY(pQuery) && prevTs < key) || (!QUERY_IS_ASC_QUERY(pQuery) && prevTs > key))) { + doRowwiseTimeWindowInterpolation(pRuntimeEnv, pDataBlock, prevTs, prevRowIndex, ts, offset, key, RESULT_ROW_START_INTERP); + setResultRowInterpo(pResult, RESULT_ROW_START_INTERP); + } else { + setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfOutput, RESULT_ROW_START_INTERP); + } + + setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfOutput, RESULT_ROW_END_INTERP); + for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { + pRuntimeEnv->pCtx[k].size = 1; + } + } else { + setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfOutput, RESULT_ROW_START_INTERP); + } +} + +static void setTimeWindowEKeyInterp(SQueryRuntimeEnv* pRuntimeEnv, SArray* pDataBlock, TSKEY prevTs, int32_t prevRowIndex, TSKEY ts, int32_t offset, SResultRow* pResult, STimeWindow* win) { + SQuery* pQuery = pRuntimeEnv->pQuery; + + TSKEY key = QUERY_IS_ASC_QUERY(pQuery)? win->ekey:win->skey; + doRowwiseTimeWindowInterpolation(pRuntimeEnv, pDataBlock, prevTs, prevRowIndex, ts, offset, key, RESULT_ROW_END_INTERP); + setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); + + setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfOutput, RESULT_ROW_START_INTERP); + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { + pRuntimeEnv->pCtx[i].size = 0; + } +} + static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pStatis, SDataBlockInfo *pDataBlockInfo, SWindowResInfo *pWindowResInfo, SArray *pDataBlock) { SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; @@ -1585,7 +1622,7 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS } int32_t offset = -1; - TSKEY prevTs = *(TSKEY*) pRuntimeEnv->prevRow[0]; + TSKEY prevTs = *(TSKEY*) pRuntimeEnv->prevRow[0]; int32_t prevRowIndex = -1; for (int32_t j = 0; j < pDataBlockInfo->rows; ++j) { @@ -1609,8 +1646,8 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS // interval window query, decide the time window according to the primary timestamp if (QUERY_IS_INTERVAL_QUERY(pQuery)) { int32_t prevWindowIndex = curTimeWindowIndex(pWindowResInfo); + int64_t ts = tsCols[offset]; - int64_t ts = tsCols[offset]; STimeWindow win = getActiveTimeWindow(pWindowResInfo, ts, pQuery); bool hasTimeWindow = false; @@ -1631,21 +1668,14 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS if (prevWindowIndex != -1 && prevWindowIndex < curIndex) { for (int32_t k = prevWindowIndex; k < curIndex; ++k) { SResultRow *pRes = pWindowResInfo->pResult[k]; - STimeWindow w = pRes->win; - ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo, &w, masterScan, &hasTimeWindow, &pResult); + + ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo, &pRes->win, masterScan, &hasTimeWindow, &pResult); assert(ret == TSDB_CODE_SUCCESS && !resultRowInterpolated(pResult, RESULT_ROW_END_INTERP)); - TSKEY key = QUERY_IS_ASC_QUERY(pQuery)? w.ekey:w.skey; - doRowwiseTimeWindowInterpolation(pRuntimeEnv, pDataBlock, prevTs, prevRowIndex, ts, offset, key, RESULT_ROW_END_INTERP); - setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); - - setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfCols, RESULT_ROW_START_INTERP); - for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - pRuntimeEnv->pCtx[i].size = 0; - } + setTimeWindowEKeyInterp(pRuntimeEnv, pDataBlock, prevTs, prevRowIndex, ts, offset, pResult, &pRes->win); bool closed = getResultRowStatus(pWindowResInfo, curTimeWindowIndex(pWindowResInfo)); - doRowwiseApplyFunctions(pRuntimeEnv, closed, &w, offset); + doRowwiseApplyFunctions(pRuntimeEnv, closed, &pRes->win, offset); } // restore current time window @@ -1656,23 +1686,7 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS } } - bool done = resultRowInterpolated(pResult, RESULT_ROW_START_INTERP); - if (!done) { - TSKEY key = QUERY_IS_ASC_QUERY(pQuery)? win.skey:win.ekey; - if (prevTs != INT64_MIN && ts != key) { - doRowwiseTimeWindowInterpolation(pRuntimeEnv, pDataBlock, prevTs, prevRowIndex, ts, offset, key, RESULT_ROW_START_INTERP); - setResultRowInterpo(pResult, RESULT_ROW_START_INTERP); - } else { - setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfCols, RESULT_ROW_START_INTERP); - } - - setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfCols, RESULT_ROW_END_INTERP); - for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { - pRuntimeEnv->pCtx[k].size = 1; - } - } else { - setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfCols, RESULT_ROW_START_INTERP); - } + setTimeWindowSKeyInterp(pRuntimeEnv, pDataBlock, prevTs, prevRowIndex, ts, offset, pResult, &win); } bool closed = getResultRowStatus(pWindowResInfo, curTimeWindowIndex(pWindowResInfo)); @@ -1699,19 +1713,7 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS } if (hasTimeWindow) { - bool done = resultRowInterpolated(pResult, RESULT_ROW_START_INTERP); - if (!done) { - if (prevTs != INT64_MIN && ((QUERY_IS_ASC_QUERY(pQuery) && (prevTs < nextWin.skey)) || (!QUERY_IS_ASC_QUERY(pQuery) && prevTs > nextWin.ekey))) { - TSKEY key = QUERY_IS_ASC_QUERY(pQuery)? nextWin.skey:nextWin.ekey; - doRowwiseTimeWindowInterpolation(pRuntimeEnv, pDataBlock, prevTs, prevRowIndex, ts, offset, key, RESULT_ROW_START_INTERP); - setResultRowInterpo(pResult, RESULT_ROW_START_INTERP); - } else { - setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfCols, RESULT_ROW_START_INTERP); - } - - setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfCols, RESULT_ROW_END_INTERP); - } - + setTimeWindowSKeyInterp(pRuntimeEnv, pDataBlock, prevTs, prevRowIndex, ts, offset, pResult, &nextWin); closed = getResultRowStatus(pWindowResInfo, curTimeWindowIndex(pWindowResInfo)); doRowwiseApplyFunctions(pRuntimeEnv, closed, &nextWin, offset); } diff --git a/tests/script/general/parser/function.sim b/tests/script/general/parser/function.sim new file mode 100644 index 0000000000..34e9844f71 --- /dev/null +++ b/tests/script/general/parser/function.sim @@ -0,0 +1,228 @@ +system sh/stop_dnodes.sh + +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c tableMetaKeepTimer -v 3 +system sh/exec.sh -n dnode1 -s start +sleep 500 +sql connect + +$dbPrefix = m_func_db +$tbPrefix = m_func_tb +$mtPrefix = m_func_mt + +$tbNum = 10 +$rowNum = 5 +$totalNum = $tbNum * $rowNum +$ts0 = 1537146000000 +$delta = 600000 +print ========== alter.sim +$i = 0 +$db = $dbPrefix . $i +$mt = $mtPrefix . $i + +sql drop database if exists $db +sql create database $db +sql use $db + +print =====================================> test case for twa in single block + +sql create table t1 (ts timestamp, k float); +sql insert into t1 values('2015-08-18 00:00:00', 2.064); +sql insert into t1 values('2015-08-18 00:06:00', 2.116); +sql insert into t1 values('2015-08-18 00:12:00', 2.028); +sql insert into t1 values('2015-08-18 00:18:00', 2.126); +sql insert into t1 values('2015-08-18 00:24:00', 2.041); +sql insert into t1 values('2015-08-18 00:30:00', 2.051); + +sql select twa(k),avg(k),count(1) from t1 where ts>='2015-8-18 00:00:00' and ts<='2015-8-18 00:05:00' +if $rows != 1 then + return -1 +endi + +if $data00 != 2.063999891 then + return -1 +endi + +if $data01 != 2.063999891 then + return -1 +endi + +if $data02 != 1 then + return -1 +endi + +sql select twa(k),avg(k),count(1) from t1 where ts>='2015-8-18 00:00:00' and ts<='2015-8-18 00:07:00' +if $rows != 1 then + return -1 +endi + +if $data00 != 2.089999914 then + return -1 +endi + +if $data01 != 2.089999914 then + return -1 +endi + +if $data02 != 2 then + return -1 +endi + +sql select twa(k),avg(k),count(1) from t1 where ts>='2015-8-18 00:00:00' and ts<='2015-8-18 00:07:00' interval(1m) order by ts asc +if $rows != 2 then + return -1 +endi + +if $data00 != @15-08-18 00:00:00.000@ then + return -1 +endi + +if $data01 != 2.068333156 then + return -1 +endi + +if $data02 != 2.063999891 then + return -1 +endi + +if $data03 != 1 then + return -1 +endi + +if $data10 != @15-08-18 00:06:00.000@ then + return -1 +endi + +if $data11 != 2.115999937 then + return -1 +endi + +if $data12 != 2.115999937 then + return -1 +endi + +if $data13 != 1 then + return -1 +endi + +sql select twa(k),avg(k),count(1) from t1 where ts>='2015-8-18 00:00:00' and ts<='2015-8-18 00:07:00' interval(1m) order by ts desc; +if $rows != 2 then + return -1 +endi + +if $data00 != @15-08-18 00:06:00.00@ then + return -1 +endi + +if $data01 != 2.115999937 then + return -1 +endi + +if $data02 != 2.115999937 then + return -1 +endi + +if $data03 != 1 then + return -1 +endi + +if $data11 != 2.068333156 then + return -1 +endi + +sql select twa(k),avg(k),count(1) from t1 where ts>='2015-8-18 00:00:00' and ts<='2015-8-18 00:27:00' interval(10m) order by ts asc +if $rows != 3 then + return -1 +endi + +if $data01 != 2.088666666 then + return -1 +endi + +if $data02 != 2.089999914 then + return -1 +endi + +if $data03 != 2 then + return -1 +endi + +if $data11 != 2.077099980 then + return -1 +endi + +if $data12 != 2.077000022 then + return -1 +endi + +if $data13 != 2 then + return -1 +endi + +if $data21 != 2.069333235 then + return -1 +endi + +if $data22 != 2.040999889 then + return -1 +endi + +if $data23 != 1 then + return -1 +endi + +sql select twa(k),avg(k),count(1) from t1 where ts>='2015-8-18 00:00:00' and ts<='2015-8-18 00:27:00' interval(10m) order by ts desc +if $rows != 3 then + return -1 +endi + +if $data01 != 2.069333235 then + return -1 +endi + +if $data11 != 2.077099980 then + return -1 +endi + +if $data21 != 2.088666666 then + return -1 +endi + +sql select twa(k),avg(k),count(1) from t1 where ts>='2015-8-18 00:00:00' and ts<='2015-8-18 00:30:00' order by ts asc +if $data00 != 2.073699975 then + return -1 +endi + +if $data01 != 2.070999980 then + return -1 +endi + +if $data02 != 6 then + return -1 +endi + +sql select twa(k),avg(k),count(1) from t1 where ts>='2015-8-18 00:00:00' and ts<='2015-8-18 00:30:00' order by ts desc +if $rows != 1 then + return -1 +endi + +if $data00 != 2.073699975 then + return -1 +endi + +if $data01 != 2.070999980 then + return -1 +endi + +if $data02 != 6 then + return -1 +endi + +sql select twa(k),avg(k),count(1) from t1 where ts>='2015-8-18 00:00:00' and ts<='2015-8-18 00:30:00' interval(10m) order by ts asc +sql select twa(k),avg(k),count(1) from t1 where ts>='2015-8-18 00:00:00' and ts<='2015-8-18 00:30:00' interval(10m) order by ts desc + + +#todo add test case while column filte exists. + +select count(*),TWA(k) from tm0 where ts>='1970-1-1 13:43:00' and ts<='1970-1-1 13:44:10' interval(9s) From 6526ea44a9d6bd531346ef056750d5d644126ad4 Mon Sep 17 00:00:00 2001 From: Xiaowei Su <46439638+Shawshank-Smile@users.noreply.github.com> Date: Tue, 1 Dec 2020 17:24:51 +0800 Subject: [PATCH 05/58] Update cluster-ch.md --- documentation20/webdocs/markdowndocs/cluster-ch.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation20/webdocs/markdowndocs/cluster-ch.md b/documentation20/webdocs/markdowndocs/cluster-ch.md index 10c28c284c..db479417c5 100644 --- a/documentation20/webdocs/markdowndocs/cluster-ch.md +++ b/documentation20/webdocs/markdowndocs/cluster-ch.md @@ -37,7 +37,7 @@ fqdn h1.taosdata.com // 配置本数据节点的端口号，缺省是6030 serverPort 6030 -// 服务端节点数为偶数的时候，需要配置，请参考《Arbitrator的使用》的部分 +// 使用场景，请参考《Arbitrator的使用》的部分 arbitrator ha.taosdata.com:6042 ``` From b55548782bda709da4a5a7d99a40644eb82734d8 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 2 Dec 2020 15:23:34 +0800 Subject: [PATCH 06/58] [TD-2293]: improve projection query performance for super table. --- src/client/src/tscSQLParser.c | 7 +- src/query/inc/qExecutor.h | 4 +- src/query/src/qExecutor.c | 318 ++++++++++++++++++++++------------ src/query/src/qUtil.c | 2 - src/util/src/hash.c | 5 +- 5 files changed, 217 insertions(+), 119 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 309d354849..0c9ad78cd9 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -5311,9 +5311,10 @@ int32_t parseLimitClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t clauseIn if (tscOrderedProjectionQueryOnSTable(pQueryInfo, 0)) { /* - * the limitation/offset value should be removed during retrieve data from virtual node, - * since the global order are done in client side, so the limitation should also - * be done at the client side. + * the offset value should be removed during retrieve data from virtual node, since the + * global order are done in client side, so the offset is applied at the client side + * However, note that the maximum allowed number of result for each table should be less + * than or equal to the value of limit. */ if (pQueryInfo->limit.limit > 0) { pQueryInfo->limit.limit = -1; diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h index 71fc01923a..4ec6ed30ba 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qExecutor.h @@ -190,6 +190,7 @@ typedef struct SQueryRuntimeEnv { bool groupbyNormalCol; // denote if this is a groupby normal column query bool hasTagResults; // if there are tag values in final result or not bool timeWindowInterpo;// if the time window start/end required interpolation + bool queryWindowIdentical; // all query time windows are identical for all tables in one group int32_t interBufSize; // intermediate buffer sizse int32_t prevGroupId; // previous executed group id SDiskbasedResultBuf* pResultBuf; // query result buffer based on blocked-wised disk file @@ -217,7 +218,8 @@ typedef struct SQInfo { STableGroupInfo tableGroupInfo; // table list SArray STableGroupInfo tableqinfoGroupInfo; // this is a group array list, including SArray structure SQueryRuntimeEnv runtimeEnv; - SArray* arrTableIdInfo; +// SArray* arrTableIdInfo; + SHashObj* arrTableIdInfo; int32_t groupIndex; /* diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 6de57b32fd..7e6950a735 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -197,6 +197,8 @@ static int32_t checkForQueryBuf(size_t numOfTables); static void releaseQueryBuf(size_t numOfTables); static int32_t binarySearchForKey(char *pValue, int num, TSKEY key, int order); static void doRowwiseTimeWindowInterpolation(SQueryRuntimeEnv* pRuntimeEnv, SArray* pDataBlock, TSKEY prevTs, int32_t prevRowIndex, TSKEY curTs, int32_t curRowIndex, TSKEY windowKey, int32_t type); +static STsdbQueryCond createTsdbQueryCond(SQuery* pQuery); +static STableIdInfo createTableIdInfo(SQuery* pQuery); bool doFilterData(SQuery *pQuery, int32_t elemPos) { for (int32_t k = 0; k < pQuery->numOfFilterCols; ++k) { @@ -2781,6 +2783,7 @@ static void ensureOutputBufferSimple(SQueryRuntimeEnv* pRuntimeEnv, int32_t capa pQuery->rec.capacity = capacity; } +// TODO merge with enuserOutputBufferSimple static void ensureOutputBuffer(SQueryRuntimeEnv* pRuntimeEnv, SDataBlockInfo* pBlockInfo) { // in case of prj/diff query, ensure the output buffer is sufficient to accommodate the results of current block SQuery* pQuery = pRuntimeEnv->pQuery; @@ -3916,13 +3919,7 @@ static void setEnvBeforeReverseScan(SQueryRuntimeEnv *pRuntimeEnv, SQueryStatusI SET_REVERSE_SCAN_FLAG(pRuntimeEnv); - STsdbQueryCond cond = { - .order = pQuery->order.order, - .colList = pQuery->colList, - .numOfCols = pQuery->numOfCols, - }; - - TIME_WINDOW_COPY(cond.twindow, pQuery->window); + STsdbQueryCond cond = createTsdbQueryCond(pQuery); setQueryStatus(pQuery, QUERY_NOT_COMPLETED); switchCtxOrder(pRuntimeEnv); @@ -4005,18 +4002,11 @@ void scanOneTableDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start) { break; } - STsdbQueryCond cond = { - .order = pQuery->order.order, - .colList = pQuery->colList, - .numOfCols = pQuery->numOfCols, - }; - - TIME_WINDOW_COPY(cond.twindow, qstatus.curWindow); - if (pRuntimeEnv->pSecQueryHandle != NULL) { tsdbCleanupQueryHandle(pRuntimeEnv->pSecQueryHandle); } + STsdbQueryCond cond = createTsdbQueryCond(pQuery); restoreTimeWindow(&pQInfo->tableGroupInfo, &cond); pRuntimeEnv->pSecQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo, &pQInfo->memRef); if (pRuntimeEnv->pSecQueryHandle == NULL) { @@ -4541,16 +4531,19 @@ static void doCopyQueryResultToMsg(SQInfo *pQInfo, int32_t numOfRows, char *data } } - int32_t numOfTables = (int32_t)taosArrayGetSize(pQInfo->arrTableIdInfo); + int32_t numOfTables = taosHashGetSize(pQInfo->arrTableIdInfo); *(int32_t*)data = htonl(numOfTables); data += sizeof(int32_t); - for(int32_t i = 0; i < numOfTables; i++) { - STableIdInfo* pSrc = taosArrayGet(pQInfo->arrTableIdInfo, i); + + STableIdInfo* item = taosHashIterate(pQInfo->arrTableIdInfo, NULL); + while(item) { STableIdInfo* pDst = (STableIdInfo*)data; - pDst->uid = htobe64(pSrc->uid); - pDst->tid = htonl(pSrc->tid); - pDst->key = htobe64(pSrc->key); + pDst->uid = htobe64(item->uid); + pDst->tid = htonl(item->tid); + pDst->key = htobe64(item->key); + data += sizeof(STableIdInfo); + item = taosHashIterate(pQInfo->arrTableIdInfo, item); } // Check if query is completed or not for stable query or normal table query respectively. @@ -4877,13 +4870,7 @@ static int32_t setupQueryHandle(void* tsdb, SQInfo* pQInfo, bool isSTableQuery) return TSDB_CODE_SUCCESS; } - STsdbQueryCond cond = { - .order = pQuery->order.order, - .colList = pQuery->colList, - .numOfCols = pQuery->numOfCols, - }; - - TIME_WINDOW_COPY(cond.twindow, pQuery->window); + STsdbQueryCond cond = createTsdbQueryCond(pQuery); if (!isSTableQuery && (pQInfo->tableqinfoGroupInfo.numOfTables == 1) @@ -5276,6 +5263,41 @@ static bool multiTableMultioutputHelper(SQInfo *pQInfo, int32_t index) { return true; } +STsdbQueryCond createTsdbQueryCond(SQuery* pQuery) { + STsdbQueryCond cond = { + .colList = pQuery->colList, + .order = pQuery->order.order, + .numOfCols = pQuery->numOfCols, + }; + + TIME_WINDOW_COPY(cond.twindow, pQuery->window); + return cond; +} + +static STableIdInfo createTableIdInfo(SQuery* pQuery) { + assert(pQuery != NULL && pQuery->current != NULL); + + STableIdInfo tidInfo; + STableId* id = TSDB_TABLEID(pQuery->current->pTable); + + tidInfo.uid = id->uid; + tidInfo.tid = id->tid; + tidInfo.key = pQuery->current->lastKey; + + return tidInfo; +} + +static void updateTableIdInfo(SQuery* pQuery, SHashObj* pTableIdInfo) { + STableIdInfo tidInfo = createTableIdInfo(pQuery); + STableIdInfo* idinfo = taosHashGet(pTableIdInfo, &tidInfo.tid, sizeof(tidInfo.tid)); + if (idinfo != NULL) { + assert(idinfo->tid == tidInfo.tid && idinfo->uid == tidInfo.uid); + idinfo->key = tidInfo.key; + } else { + taosHashPut(pTableIdInfo, &tidInfo.tid, sizeof(tidInfo.tid), &tidInfo, sizeof(STableIdInfo)); + } +} + /** * super table query handler * 1. super table projection query, group-by on normal columns query, ts-comp query @@ -5295,18 +5317,11 @@ static void sequentialTableProcess(SQInfo *pQInfo) { assert(pQuery->limit.offset == 0 && pQuery->limit.limit != 0); while (pQInfo->groupIndex < numOfGroups) { - SArray* group = taosArrayGetP(pQInfo->tableGroupInfo.pGroupList, pQInfo->groupIndex); + SArray *group = taosArrayGetP(pQInfo->tableGroupInfo.pGroupList, pQInfo->groupIndex); - qDebug("QInfo:%p last_row query on group:%d, total group:%" PRIzu ", current group:%p", pQInfo, pQInfo->groupIndex, - numOfGroups, group); - - STsdbQueryCond cond = { - .colList = pQuery->colList, - .order = pQuery->order.order, - .numOfCols = pQuery->numOfCols, - }; - - TIME_WINDOW_COPY(cond.twindow, pQuery->window); + qDebug("QInfo:%p point interpolation query on group:%d, total group:%" PRIzu ", current group:%p", pQInfo, + pQInfo->groupIndex, numOfGroups, group); + STsdbQueryCond cond = createTsdbQueryCond(pQuery); SArray *g1 = taosArrayInit(1, POINTER_BYTES); SArray *tx = taosArrayClone(group); @@ -5330,14 +5345,14 @@ static void sequentialTableProcess(SQInfo *pQInfo) { initCtxOutputBuf(pRuntimeEnv); - SArray* s = tsdbGetQueriedTableList(pRuntimeEnv->pQueryHandle); + SArray *s = tsdbGetQueriedTableList(pRuntimeEnv->pQueryHandle); assert(taosArrayGetSize(s) >= 1); setTagVal(pRuntimeEnv, taosArrayGetP(s, 0), pQInfo->tsdb); taosArrayDestroy(s); // here we simply set the first table as current table - SArray* first = GET_TABLEGROUP(pQInfo, pQInfo->groupIndex); + SArray *first = GET_TABLEGROUP(pQInfo, pQInfo->groupIndex); pQuery->current = taosArrayGetP(first, 0); scanOneTableDataBlocks(pRuntimeEnv, pQuery->current->lastKey); @@ -5359,19 +5374,14 @@ static void sequentialTableProcess(SQInfo *pQInfo) { break; } } - } else if (pRuntimeEnv->groupbyNormalCol) { // group-by on normal columns query + } else if (pRuntimeEnv->groupbyNormalCol) { // group-by on normal columns query while (pQInfo->groupIndex < numOfGroups) { - SArray* group = taosArrayGetP(pQInfo->tableGroupInfo.pGroupList, pQInfo->groupIndex); + SArray *group = taosArrayGetP(pQInfo->tableGroupInfo.pGroupList, pQInfo->groupIndex); - qDebug("QInfo:%p group by normal columns group:%d, total group:%" PRIzu "", pQInfo, pQInfo->groupIndex, numOfGroups); + qDebug("QInfo:%p group by normal columns group:%d, total group:%" PRIzu "", pQInfo, pQInfo->groupIndex, + numOfGroups); - STsdbQueryCond cond = { - .colList = pQuery->colList, - .order = pQuery->order.order, - .numOfCols = pQuery->numOfCols, - }; - - TIME_WINDOW_COPY(cond.twindow, pQuery->window); + STsdbQueryCond cond = createTsdbQueryCond(pQuery); SArray *g1 = taosArrayInit(1, POINTER_BYTES); SArray *tx = taosArrayClone(group); @@ -5394,7 +5404,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) { longjmp(pRuntimeEnv->env, terrno); } - SArray* s = tsdbGetQueriedTableList(pRuntimeEnv->pQueryHandle); + SArray *s = tsdbGetQueriedTableList(pRuntimeEnv->pQueryHandle); assert(taosArrayGetSize(s) >= 1); setTagVal(pRuntimeEnv, taosArrayGetP(s, 0), pQInfo->tsdb); @@ -5405,24 +5415,24 @@ static void sequentialTableProcess(SQInfo *pQInfo) { SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; - // no results generated for current group, continue to try the next group + // no results generated for current group, continue to try the next group taosArrayDestroy(s); if (pWindowResInfo->size <= 0) { continue; } for (int32_t i = 0; i < pWindowResInfo->size; ++i) { - pWindowResInfo->pResult[i]->closed = true; // enable return all results for group by normal columns + pWindowResInfo->pResult[i]->closed = true; // enable return all results for group by normal columns SResultRow *pResult = pWindowResInfo->pResult[i]; for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { - SResultRowCellInfo* pCell = getResultCell(pRuntimeEnv, pResult, j); + SResultRowCellInfo *pCell = getResultCell(pRuntimeEnv, pResult, j); pResult->numOfRows = (uint16_t)(MAX(pResult->numOfRows, pCell->numOfRes)); } } qDebug("QInfo:%p generated groupby columns results %d rows for group %d completed", pQInfo, pWindowResInfo->size, - pQInfo->groupIndex); + pQInfo->groupIndex); int32_t currentGroupIndex = pQInfo->groupIndex; pQuery->rec.rows = 0; @@ -5431,16 +5441,119 @@ static void sequentialTableProcess(SQInfo *pQInfo) { ensureOutputBufferSimple(pRuntimeEnv, pWindowResInfo->size); copyFromWindowResToSData(pQInfo, pWindowResInfo); - pQInfo->groupIndex = currentGroupIndex; //restore the group index + pQInfo->groupIndex = currentGroupIndex; // restore the group index assert(pQuery->rec.rows == pWindowResInfo->size); clearClosedTimeWindow(pRuntimeEnv); break; } + } else if (pRuntimeEnv->queryWindowIdentical && pRuntimeEnv->pTSBuf == NULL) { + //super table projection query with identical query time range for all tables. + SDataBlockInfo blockInfo = SDATA_BLOCK_INITIALIZER; + + resetDefaultResInfoOutputBuf(pRuntimeEnv); + + SArray *group = GET_TABLEGROUP(pQInfo, 0); + assert(taosArrayGetSize(group) == pQInfo->tableqinfoGroupInfo.numOfTables && + 1 == taosArrayGetSize(pQInfo->tableqinfoGroupInfo.pGroupList)); + + void *pQueryHandle = pRuntimeEnv->pQueryHandle; + if (pQueryHandle == NULL) { + STsdbQueryCond con = createTsdbQueryCond(pQuery); + pRuntimeEnv->pQueryHandle = tsdbQueryTables(pQInfo->tsdb, &con, &pQInfo->tableGroupInfo, pQInfo, &pQInfo->memRef); + pQueryHandle = pRuntimeEnv->pQueryHandle; + } + + // skip blocks without load the actual data block from file if no filter condition present + // skipBlocks(&pQInfo->runtimeEnv); + // if (pQuery->limit.offset > 0 && pQuery->numOfFilterCols == 0) { + // setQueryStatus(pQuery, QUERY_COMPLETED); + // return; + // } + + bool hasMoreBlock = true; + SQueryCostInfo *summary = &pRuntimeEnv->summary; + while ((hasMoreBlock = tsdbNextDataBlock(pQueryHandle)) == true) { + summary->totalBlocks += 1; + + if (IS_QUERY_KILLED(pQInfo)) { + longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); + } + + tsdbRetrieveDataBlockInfo(pQueryHandle, &blockInfo); + STableQueryInfo **pTableQueryInfo = + (STableQueryInfo **)taosHashGet(pQInfo->tableqinfoGroupInfo.map, &blockInfo.tid, sizeof(blockInfo.tid)); + if (pTableQueryInfo == NULL) { + break; + } + + pQuery->current = *pTableQueryInfo; + if (QUERY_IS_ASC_QUERY(pQuery)) { + assert(((*pTableQueryInfo)->win.skey <= (*pTableQueryInfo)->win.ekey) && + ((*pTableQueryInfo)->lastKey >= (*pTableQueryInfo)->win.skey) && + ((*pTableQueryInfo)->win.skey >= pQuery->window.skey && + (*pTableQueryInfo)->win.ekey <= pQuery->window.ekey)); + } else { + assert(((*pTableQueryInfo)->win.skey >= (*pTableQueryInfo)->win.ekey) && + ((*pTableQueryInfo)->lastKey <= (*pTableQueryInfo)->win.skey) && + ((*pTableQueryInfo)->win.skey <= pQuery->window.skey && + (*pTableQueryInfo)->win.ekey >= pQuery->window.ekey)); + } + + if (pRuntimeEnv->hasTagResults) { + setTagVal(pRuntimeEnv, pQuery->current->pTable, pQInfo->tsdb); + } + + uint32_t status = 0; + SDataStatis *pStatis = NULL; + SArray * pDataBlock = NULL; + + int32_t ret = loadDataBlockOnDemand(pRuntimeEnv, &pQuery->current->windowResInfo, pQueryHandle, &blockInfo, + &pStatis, &pDataBlock, &status); + if (ret != TSDB_CODE_SUCCESS) { + break; + } + + assert(status != BLK_DATA_DISCARD); + ensureOutputBuffer(pRuntimeEnv, &blockInfo); + + pQuery->pos = QUERY_IS_ASC_QUERY(pQuery) ? 0 : blockInfo.rows - 1; + int32_t numOfRes = tableApplyFunctionsOnBlock(pRuntimeEnv, &blockInfo, pStatis, binarySearchForKey, pDataBlock); + + summary->totalRows += blockInfo.rows; + qDebug("QInfo:%p check data block, brange:%" PRId64 "-%" PRId64 ", numOfRows:%d, numOfRes:%d, lastKey:%" PRId64, + GET_QINFO_ADDR(pRuntimeEnv), blockInfo.window.skey, blockInfo.window.ekey, blockInfo.rows, numOfRes, + pQuery->current->lastKey); + + pQuery->rec.rows = getNumOfResult(pRuntimeEnv); + + // the flag may be set by tableApplyFunctionsOnBlock, clear it here + CLEAR_QUERY_STATUS(pQuery, QUERY_COMPLETED); + + updateTableIdInfo(pQuery, pQInfo->arrTableIdInfo); + skipResults(pRuntimeEnv); + + // the limitation of output result is reached, set the query completed + if (limitResults(pRuntimeEnv)) { + SET_STABLE_QUERY_OVER(pQInfo); + break; + } + + // while the output buffer is full or limit/offset is applied, query may be paused here + if (Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL|QUERY_COMPLETED)) { + break; + } + } + + if (!hasMoreBlock) { + setQueryStatus(pQuery, QUERY_COMPLETED); + SET_STABLE_QUERY_OVER(pQInfo); + } } else { /* - * 1. super table projection query, 2. ts-comp query - * if the subgroup index is larger than 0, results generated by group by tbname,k is existed. + * the following two cases handled here. + * 1. ts-comp query, and 2. the super table projection query with different query time range for each table. + * If the subgroup index is larger than 0, results generated by group by tbname,k is existed. * we need to return it to client in the first place. */ if (pQInfo->groupIndex > 0) { @@ -5503,14 +5616,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) { * to ensure that, we can reset the query range once query on a meter is completed. */ pQInfo->tableIndex++; - - STableIdInfo tidInfo = {0}; - - STableId* id = TSDB_TABLEID(pQuery->current->pTable); - tidInfo.uid = id->uid; - tidInfo.tid = id->tid; - tidInfo.key = pQuery->current->lastKey; - taosArrayPush(pQInfo->arrTableIdInfo, &tidInfo); + updateTableIdInfo(pQuery, pQInfo->arrTableIdInfo); // if the buffer is full or group by each table, we need to jump out of the loop if (Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL)) { @@ -5537,31 +5643,31 @@ static void sequentialTableProcess(SQInfo *pQInfo) { if (pQInfo->tableIndex >= pQInfo->tableqinfoGroupInfo.numOfTables) { setQueryStatus(pQuery, QUERY_COMPLETED); } - } - /* - * 1. super table projection query, group-by on normal columns query, ts-comp query - * 2. point interpolation query, last row query - * - * group-by on normal columns query and last_row query do NOT invoke the finalizer here, - * since the finalize stage will be done at the client side. - * - * projection query, point interpolation query do not need the finalizer. - * - * Only the ts-comp query requires the finalizer function to be executed here. - */ - if (isTSCompQuery(pQuery)) { - finalizeQueryResult(pRuntimeEnv); - } + /* + * 1. super table projection query, group-by on normal columns query, ts-comp query + * 2. point interpolation query, last row query + * + * group-by on normal columns query and last_row query do NOT invoke the finalizer here, + * since the finalize stage will be done at the client side. + * + * projection query, point interpolation query do not need the finalizer. + * + * Only the ts-comp query requires the finalizer function to be executed here. + */ + if (isTSCompQuery(pQuery)) { + finalizeQueryResult(pRuntimeEnv); + } - if (pRuntimeEnv->pTSBuf != NULL) { - pRuntimeEnv->cur = pRuntimeEnv->pTSBuf->cur; - } + if (pRuntimeEnv->pTSBuf != NULL) { + pRuntimeEnv->cur = pRuntimeEnv->pTSBuf->cur; + } - qDebug( - "QInfo %p numOfTables:%" PRIu64 ", index:%d, numOfGroups:%" PRIzu ", %" PRId64 " points returned, total:%" PRId64 ", offset:%" PRId64, - pQInfo, (uint64_t)pQInfo->tableqinfoGroupInfo.numOfTables, pQInfo->tableIndex, numOfGroups, pQuery->rec.rows, pQuery->rec.total, - pQuery->limit.offset); + qDebug("QInfo %p numOfTables:%" PRIu64 ", index:%d, numOfGroups:%" PRIzu ", %" PRId64 + " points returned, total:%" PRId64 ", offset:%" PRId64, + pQInfo, (uint64_t)pQInfo->tableqinfoGroupInfo.numOfTables, pQInfo->tableIndex, numOfGroups, pQuery->rec.rows, + pQuery->rec.total, pQuery->limit.offset); + } } static void doSaveContext(SQInfo *pQInfo) { @@ -5576,13 +5682,7 @@ static void doSaveContext(SQInfo *pQInfo) { SWITCH_ORDER(pRuntimeEnv->pTSBuf->cur.order); } - STsdbQueryCond cond = { - .order = pQuery->order.order, - .colList = pQuery->colList, - .numOfCols = pQuery->numOfCols, - }; - - TIME_WINDOW_COPY(cond.twindow, pQuery->window); + STsdbQueryCond cond = createTsdbQueryCond(pQuery); // clean unused handle if (pRuntimeEnv->pSecQueryHandle != NULL) { @@ -5855,13 +5955,8 @@ static void tableMultiOutputProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) qDebug("QInfo:%p query paused due to output limitation, next qrange:%" PRId64 "-%" PRId64, pQInfo, pQuery->current->lastKey, pQuery->window.ekey); } else if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { - STableIdInfo tidInfo; - STableId* id = TSDB_TABLEID(pQuery->current->pTable); - - tidInfo.uid = id->uid; - tidInfo.tid = id->tid; - tidInfo.key = pQuery->current->lastKey; - taosArrayPush(pQInfo->arrTableIdInfo, &tidInfo); + STableIdInfo tidInfo = createTableIdInfo(pQuery); + taosHashPut(pQInfo->arrTableIdInfo, &tidInfo.tid, sizeof(tidInfo.tid), &tidInfo, sizeof(STableIdInfo)); } if (!isTSCompQuery(pQuery)) { @@ -6859,8 +6954,6 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); } - int tableIndex = 0; - pQInfo->runtimeEnv.interBufSize = getOutputInterResultBufSize(pQuery); pQInfo->runtimeEnv.summary.tableInfoSize += (pTableGroupInfo->numOfTables * sizeof(STableQueryInfo)); @@ -6882,7 +6975,7 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou } // NOTE: pTableCheckInfo need to update the query time range and the lastKey info - pQInfo->arrTableIdInfo = taosArrayInit(tableIndex, sizeof(STableIdInfo)); + pQInfo->arrTableIdInfo = taosHashInit(pTableGroupInfo->numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); pQInfo->dataReady = QUERY_RESULT_NOT_READY; pQInfo->rspContext = NULL; pthread_mutex_init(&pQInfo->lock, NULL); @@ -6892,10 +6985,10 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou pQuery->window = pQueryMsg->window; changeExecuteScanOrder(pQInfo, pQueryMsg, stableQuery); + pQInfo->runtimeEnv.queryWindowIdentical = true; STimeWindow window = pQuery->window; int32_t index = 0; - for(int32_t i = 0; i < numOfGroups; ++i) { SArray* pa = taosArrayGetP(pQInfo->tableGroupInfo.pGroupList, i); @@ -6910,9 +7003,12 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou for(int32_t j = 0; j < s; ++j) { STableKeyInfo* info = taosArrayGet(pa, j); - void* buf = (char*) pQInfo->pBuf + index * sizeof(STableQueryInfo); - window.skey = info->lastKey; + if (info->lastKey != pQuery->window.skey) { + pQInfo->runtimeEnv.queryWindowIdentical = false; + } + + void* buf = (char*) pQInfo->pBuf + index * sizeof(STableQueryInfo); STableQueryInfo* item = createTableQueryInfo(&pQInfo->runtimeEnv, info->pTable, window, buf); if (item == NULL) { goto _cleanup; @@ -7126,7 +7222,7 @@ static void freeQInfo(SQInfo *pQInfo) { tfree(pQInfo->pBuf); tsdbDestroyTableGroup(&pQInfo->tableGroupInfo); - taosArrayDestroy(pQInfo->arrTableIdInfo); + taosHashCleanup(pQInfo->arrTableIdInfo); pQInfo->signature = 0; @@ -7506,7 +7602,7 @@ int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp **pRsp, int32_t *co size_t size = getResultSize(pQInfo, &pQuery->rec.rows); size += sizeof(int32_t); - size += sizeof(STableIdInfo) * taosArrayGetSize(pQInfo->arrTableIdInfo); + size += sizeof(STableIdInfo) * taosHashGetSize(pQInfo->arrTableIdInfo); *contLen = (int32_t)(size + sizeof(SRetrieveTableRsp)); diff --git a/src/query/src/qUtil.c b/src/query/src/qUtil.c index 2c41d9bfc5..f5f2da9f25 100644 --- a/src/query/src/qUtil.c +++ b/src/query/src/qUtil.c @@ -83,8 +83,6 @@ void resetTimeWindowInfo(SQueryRuntimeEnv *pRuntimeEnv, SWindowResInfo *pWindowR return; } -// assert(pWindowResInfo->size == 1); - for (int32_t i = 0; i < pWindowResInfo->size; ++i) { SResultRow *pWindowRes = pWindowResInfo->pResult[i]; clearResultRow(pRuntimeEnv, pWindowRes, pWindowResInfo->type); diff --git a/src/util/src/hash.c b/src/util/src/hash.c index 3ac54eedd8..0e3e0d3e24 100644 --- a/src/util/src/hash.c +++ b/src/util/src/hash.c @@ -151,8 +151,9 @@ static void pushfrontNodeInEntryList(SHashEntry *pEntry, SHashNode *pNode); */ SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, SHashLockTypeE type) { - if (capacity == 0 || fn == NULL) { - return NULL; + assert(fn != NULL); + if (capacity == 0) { + capacity = 4; } SHashObj *pHashObj = (SHashObj *)calloc(1, sizeof(SHashObj)); From 5746fdabe71d3c97381bdb8dab114d4d3d66e15c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 2 Dec 2020 15:28:34 +0800 Subject: [PATCH 07/58] [TD-2129] refactor codes. --- src/query/src/qExecutor.c | 38 ++++++++++++++++---------------------- 1 file changed, 16 insertions(+), 22 deletions(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 7e6950a735..d65b2e8f0a 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -5078,6 +5078,20 @@ static FORCE_INLINE void setEnvForEachBlock(SQInfo* pQInfo, STableQueryInfo* pTa } } +static void doTableQueryInfoTimeWindowCheck(SQuery* pQuery, STableQueryInfo* pTableQueryInfo) { + if (QUERY_IS_ASC_QUERY(pQuery)) { + assert( + (pTableQueryInfo->win.skey <= pTableQueryInfo->win.ekey) && + (pTableQueryInfo->lastKey >= pTableQueryInfo->win.skey) && + (pTableQueryInfo->win.skey >= pQuery->window.skey && pTableQueryInfo->win.ekey <= pQuery->window.ekey)); + } else { + assert( + (pTableQueryInfo->win.skey >= pTableQueryInfo->win.ekey) && + (pTableQueryInfo->lastKey <= pTableQueryInfo->win.skey) && + (pTableQueryInfo->win.skey <= pQuery->window.skey && pTableQueryInfo->win.ekey >= pQuery->window.ekey)); + } +} + static int64_t scanMultiTableDataBlocks(SQInfo *pQInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery* pQuery = pRuntimeEnv->pQuery; @@ -5104,17 +5118,7 @@ static int64_t scanMultiTableDataBlocks(SQInfo *pQInfo) { } pQuery->current = *pTableQueryInfo; - if (QUERY_IS_ASC_QUERY(pQuery)) { - assert( - ((*pTableQueryInfo)->win.skey <= (*pTableQueryInfo)->win.ekey) && - ((*pTableQueryInfo)->lastKey >= (*pTableQueryInfo)->win.skey) && - ((*pTableQueryInfo)->win.skey >= pQuery->window.skey && (*pTableQueryInfo)->win.ekey <= pQuery->window.ekey)); - } else { - assert( - ((*pTableQueryInfo)->win.skey >= (*pTableQueryInfo)->win.ekey) && - ((*pTableQueryInfo)->lastKey <= (*pTableQueryInfo)->win.skey) && - ((*pTableQueryInfo)->win.skey <= pQuery->window.skey && (*pTableQueryInfo)->win.ekey >= pQuery->window.ekey)); - } + doTableQueryInfoTimeWindowCheck(pQuery, *pTableQueryInfo); if (!pRuntimeEnv->groupbyNormalCol) { setEnvForEachBlock(pQInfo, *pTableQueryInfo, &blockInfo); @@ -5488,17 +5492,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) { } pQuery->current = *pTableQueryInfo; - if (QUERY_IS_ASC_QUERY(pQuery)) { - assert(((*pTableQueryInfo)->win.skey <= (*pTableQueryInfo)->win.ekey) && - ((*pTableQueryInfo)->lastKey >= (*pTableQueryInfo)->win.skey) && - ((*pTableQueryInfo)->win.skey >= pQuery->window.skey && - (*pTableQueryInfo)->win.ekey <= pQuery->window.ekey)); - } else { - assert(((*pTableQueryInfo)->win.skey >= (*pTableQueryInfo)->win.ekey) && - ((*pTableQueryInfo)->lastKey <= (*pTableQueryInfo)->win.skey) && - ((*pTableQueryInfo)->win.skey <= pQuery->window.skey && - (*pTableQueryInfo)->win.ekey >= pQuery->window.ekey)); - } + doTableQueryInfoTimeWindowCheck(pQuery, *pTableQueryInfo); if (pRuntimeEnv->hasTagResults) { setTagVal(pRuntimeEnv, pQuery->current->pTable, pQInfo->tsdb); From 6b2d676ac1628fc2357f3033040fae9a135ec738 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 2 Dec 2020 15:43:50 +0800 Subject: [PATCH 08/58] [TD-2293]add parameters in query message. --- src/inc/taosmsg.h | 16 +++++++++------- src/query/src/qExecutor.c | 2 +- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index b77db69c46..b4d3bec958 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -476,19 +476,21 @@ typedef struct { int16_t numOfGroupCols; // num of group by columns int16_t orderByIdx; int16_t orderType; // used in group by xx order by xxx + int64_t tableLimit; // limit the number of rows for each table, used in order by + limit in stable projection query. + int16_t prjOrder; // global order in super table projection query. int64_t limit; int64_t offset; uint32_t queryType; // denote another query process int16_t numOfOutput; // final output columns numbers int16_t tagNameRelType; // relation of tag criteria and tbname criteria - int16_t fillType; // interpolate type - uint64_t fillVal; // default value array list + int16_t fillType; // interpolate type + uint64_t fillVal; // default value array list int32_t secondStageOutput; - int32_t tsOffset; // offset value in current msg body, NOTE: ts list is compressed - int32_t tsLen; // total length of ts comp block - int32_t tsNumOfBlocks; // ts comp block numbers - int32_t tsOrder; // ts comp block order - int32_t numOfTags; // number of tags columns involved + int32_t tsOffset; // offset value in current msg body, NOTE: ts list is compressed + int32_t tsLen; // total length of ts comp block + int32_t tsNumOfBlocks; // ts comp block numbers + int32_t tsOrder; // ts comp block order + int32_t numOfTags; // number of tags columns involved SColumnInfo colList[]; } SQueryTableMsg; diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index d65b2e8f0a..8c705c893c 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -5500,7 +5500,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) { uint32_t status = 0; SDataStatis *pStatis = NULL; - SArray * pDataBlock = NULL; + SArray *pDataBlock = NULL; int32_t ret = loadDataBlockOnDemand(pRuntimeEnv, &pQuery->current->windowResInfo, pQueryHandle, &blockInfo, &pStatis, &pDataBlock, &status); From 0c38e1d9d3b339294ae2290e632bc9f03b5cc0f0 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 2 Dec 2020 18:37:28 +0800 Subject: [PATCH 09/58] [TD-2298]: fix taosd crash during table interval query. --- src/query/src/qExecutor.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 8c705c893c..cda9455d6f 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -3597,7 +3597,7 @@ static void updateTableQueryInfoForReverseScan(SQuery *pQuery, STableQueryInfo * if (pTableQueryInfo->lastKey == pTableQueryInfo->win.skey) { // do nothing, no results - } else { + } else {// even win.skey != lastKey, the results may not generated. pTableQueryInfo->win.ekey = pTableQueryInfo->lastKey + step; } @@ -3986,11 +3986,13 @@ void scanOneTableDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start) { qstatus.status = pQuery->status; // do nothing if no data blocks are found qualified during scan + qstatus.lastKey = pTableQueryInfo->lastKey; if (qstatus.lastKey != pTableQueryInfo->lastKey) { qstatus.curWindow.ekey = pTableQueryInfo->lastKey - step; + } else { // the lastkey does not increase, which means no data checked yet + qDebug("QInfo:%p no results generated in this scan, abort", pQInfo); + return; } - - qstatus.lastKey = pTableQueryInfo->lastKey; } if (!needScanDataBlocksAgain(pRuntimeEnv)) { From 81036205759f1558abc056e38a416fd1f0a6c4ab Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 2 Dec 2020 18:43:23 +0800 Subject: [PATCH 10/58] [TD-225] refactor codes. --- src/query/inc/qExecutor.h | 15 +++++++-------- src/query/inc/qUtil.h | 2 +- src/query/src/qExecutor.c | 28 ++++++---------------------- src/query/src/qUtil.c | 5 ++--- 4 files changed, 16 insertions(+), 34 deletions(-) diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h index 4ec6ed30ba..aa018e75d4 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qExecutor.h @@ -84,14 +84,13 @@ typedef struct SResultRec { } SResultRec; typedef struct SWindowResInfo { - SResultRow** pResult; // result list - int16_t type:8; // data type for hash key - int32_t size:24; // number of result set - int32_t threshold; // threshold to halt query and return the generated results. - int32_t capacity; // max capacity - int32_t curIndex; // current start active index - int64_t startTime; // start time of the first time window for sliding query - int64_t prevSKey; // previous (not completed) sliding window start key + SResultRow** pResult; // result list + int16_t type:8; // data type for hash key + int32_t size:24; // number of result set + int32_t capacity; // max capacity + int32_t curIndex; // current start active index + int64_t startTime; // start time of the first time window for sliding query + int64_t prevSKey; // previous (not completed) sliding window start key } SWindowResInfo; typedef struct SColumnFilterElem { diff --git a/src/query/inc/qUtil.h b/src/query/inc/qUtil.h index 8b84ac0182..f15f1f33eb 100644 --- a/src/query/inc/qUtil.h +++ b/src/query/inc/qUtil.h @@ -30,7 +30,7 @@ void clearResultRow(SQueryRuntimeEnv* pRuntimeEnv, SResultRow* pRow, int16_t typ void copyResultRow(SQueryRuntimeEnv* pRuntimeEnv, SResultRow* dst, const SResultRow* src, int16_t type); SResultRowCellInfo* getResultCell(SQueryRuntimeEnv* pRuntimeEnv, const SResultRow* pRow, int32_t index); -int32_t initWindowResInfo(SWindowResInfo* pWindowResInfo, int32_t size, int32_t threshold, int16_t type); +int32_t initWindowResInfo(SWindowResInfo* pWindowResInfo, int32_t size, int16_t type); void cleanupTimeWindowInfo(SWindowResInfo* pWindowResInfo); void resetTimeWindowInfo(SQueryRuntimeEnv* pRuntimeEnv, SWindowResInfo* pWindowResInfo); diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index cda9455d6f..bfdcd6d28a 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -761,7 +761,7 @@ static int32_t doCheckQueryCompleted(SQueryRuntimeEnv *pRuntimeEnv, TSKEY lastKe pWindowResInfo->prevSKey = pWindowResInfo->pResult[pWindowResInfo->curIndex]->win.skey; // the number of completed slots are larger than the threshold, return current generated results to client. - if (numOfClosed > pWindowResInfo->threshold) { + if (numOfClosed > pQuery->rec.threshold) { qDebug("QInfo:%p total result window:%d closed:%d, reached the output threshold %d, return", GET_QINFO_ADDR(pRuntimeEnv), pWindowResInfo->size, numOfClosed, pQuery->rec.threshold); @@ -3587,17 +3587,9 @@ static void updateTableQueryInfoForReverseScan(SQuery *pQuery, STableQueryInfo * // order has changed already int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); - - // TODO validate the assertion -// if (!QUERY_IS_ASC_QUERY(pQuery)) { -// assert(pTableQueryInfo->win.ekey >= pTableQueryInfo->lastKey + step); -// } else { -// assert(pTableQueryInfo->win.ekey <= pTableQueryInfo->lastKey + step); -// } - if (pTableQueryInfo->lastKey == pTableQueryInfo->win.skey) { // do nothing, no results - } else {// even win.skey != lastKey, the results may not generated. + } else {// NOTE: even win.skey != lastKey, the results may not generated. pTableQueryInfo->win.ekey = pTableQueryInfo->lastKey + step; } @@ -4102,9 +4094,8 @@ static STableQueryInfo *createTableQueryInfo(SQueryRuntimeEnv *pRuntimeEnv, void // set more initial size of interval/groupby query if (QUERY_IS_INTERVAL_QUERY(pQuery) || pRuntimeEnv->groupbyNormalCol) { - int32_t initialSize = 16; - int32_t initialThreshold = 100; - int32_t code = initWindowResInfo(&pTableQueryInfo->windowResInfo, initialSize, initialThreshold, TSDB_DATA_TYPE_INT); + int32_t initialSize = 128; + int32_t code = initWindowResInfo(&pTableQueryInfo->windowResInfo, initialSize, TSDB_DATA_TYPE_INT); if (code != TSDB_CODE_SUCCESS) { return NULL; } @@ -4989,20 +4980,13 @@ int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bo if (!QUERY_IS_INTERVAL_QUERY(pQuery)) { int16_t type = TSDB_DATA_TYPE_NULL; - int32_t threshold = 0; - if (pRuntimeEnv->groupbyNormalCol) { // group by columns not tags; type = getGroupbyColumnType(pQuery, pQuery->pGroupbyExpr); - threshold = 4000; } else { type = TSDB_DATA_TYPE_INT; // group id - threshold = (int32_t)(GET_NUM_OF_TABLEGROUP(pQInfo)); - if (threshold < 8) { - threshold = 8; - } } - code = initWindowResInfo(&pRuntimeEnv->windowResInfo, 8, threshold, type); + code = initWindowResInfo(&pRuntimeEnv->windowResInfo, 8, type); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -5022,7 +5006,7 @@ int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bo type = TSDB_DATA_TYPE_TIMESTAMP; } - code = initWindowResInfo(&pRuntimeEnv->windowResInfo, numOfResultRows, 1024, type); + code = initWindowResInfo(&pRuntimeEnv->windowResInfo, numOfResultRows, type); if (code != TSDB_CODE_SUCCESS) { return code; } diff --git a/src/query/src/qUtil.c b/src/query/src/qUtil.c index f5f2da9f25..3f010a579d 100644 --- a/src/query/src/qUtil.c +++ b/src/query/src/qUtil.c @@ -43,10 +43,9 @@ int32_t getOutputInterResultBufSize(SQuery* pQuery) { return size; } -int32_t initWindowResInfo(SWindowResInfo *pWindowResInfo, int32_t size, int32_t threshold, int16_t type) { +int32_t initWindowResInfo(SWindowResInfo *pWindowResInfo, int32_t size, int16_t type) { pWindowResInfo->capacity = size; - pWindowResInfo->threshold = threshold; - + pWindowResInfo->type = type; pWindowResInfo->curIndex = -1; pWindowResInfo->size = 0; From c121f5b7b91e5680a63a7c35b4bb5e3cd46f1fe5 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 2 Dec 2020 18:44:32 +0800 Subject: [PATCH 11/58] [TD-225] refactor codes. --- src/query/inc/qExecutor.h | 8 +++---- src/query/inc/qUtil.h | 18 +++++++------- src/query/src/qExecutor.c | 50 +++++++++++++++++++-------------------- src/query/src/qUtil.c | 20 ++++++++-------- 4 files changed, 48 insertions(+), 48 deletions(-) diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h index aa018e75d4..b73f7ce3f5 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qExecutor.h @@ -83,7 +83,7 @@ typedef struct SResultRec { int32_t threshold; // result size threshold in rows. } SResultRec; -typedef struct SWindowResInfo { +typedef struct SResultRowInfo { SResultRow** pResult; // result list int16_t type:8; // data type for hash key int32_t size:24; // number of result set @@ -91,7 +91,7 @@ typedef struct SWindowResInfo { int32_t curIndex; // current start active index int64_t startTime; // start time of the first time window for sliding query int64_t prevSKey; // previous (not completed) sliding window start key -} SWindowResInfo; +} SResultRowInfo; typedef struct SColumnFilterElem { int16_t bytes; // column length @@ -114,7 +114,7 @@ typedef struct STableQueryInfo { STimeWindow win; STSCursor cur; void* pTable; // for retrieve the page id list - SWindowResInfo windowResInfo; + SResultRowInfo windowResInfo; } STableQueryInfo; typedef struct SQueryCostInfo { @@ -178,7 +178,7 @@ typedef struct SQueryRuntimeEnv { uint16_t* offset; uint16_t scanFlag; // denotes reversed scan of data or not SFillInfo* pFillInfo; - SWindowResInfo windowResInfo; + SResultRowInfo windowResInfo; STSBuf* pTSBuf; STSCursor cur; SQueryCostInfo summary; diff --git a/src/query/inc/qUtil.h b/src/query/inc/qUtil.h index f15f1f33eb..fb71c8a5fe 100644 --- a/src/query/inc/qUtil.h +++ b/src/query/inc/qUtil.h @@ -30,19 +30,19 @@ void clearResultRow(SQueryRuntimeEnv* pRuntimeEnv, SResultRow* pRow, int16_t typ void copyResultRow(SQueryRuntimeEnv* pRuntimeEnv, SResultRow* dst, const SResultRow* src, int16_t type); SResultRowCellInfo* getResultCell(SQueryRuntimeEnv* pRuntimeEnv, const SResultRow* pRow, int32_t index); -int32_t initWindowResInfo(SWindowResInfo* pWindowResInfo, int32_t size, int16_t type); +int32_t initWindowResInfo(SResultRowInfo* pWindowResInfo, int32_t size, int16_t type); -void cleanupTimeWindowInfo(SWindowResInfo* pWindowResInfo); -void resetTimeWindowInfo(SQueryRuntimeEnv* pRuntimeEnv, SWindowResInfo* pWindowResInfo); +void cleanupTimeWindowInfo(SResultRowInfo* pWindowResInfo); +void resetTimeWindowInfo(SQueryRuntimeEnv* pRuntimeEnv, SResultRowInfo* pWindowResInfo); void clearFirstNWindowRes(SQueryRuntimeEnv *pRuntimeEnv, int32_t num); void clearClosedTimeWindow(SQueryRuntimeEnv* pRuntimeEnv); -int32_t numOfClosedTimeWindow(SWindowResInfo* pWindowResInfo); -void closeTimeWindow(SWindowResInfo* pWindowResInfo, int32_t slot); -void closeAllTimeWindow(SWindowResInfo* pWindowResInfo); -void removeRedundantWindow(SWindowResInfo *pWindowResInfo, TSKEY lastKey, int32_t order); +int32_t numOfClosedTimeWindow(SResultRowInfo* pWindowResInfo); +void closeTimeWindow(SResultRowInfo* pWindowResInfo, int32_t slot); +void closeAllTimeWindow(SResultRowInfo* pWindowResInfo); +void removeRedundantWindow(SResultRowInfo *pWindowResInfo, TSKEY lastKey, int32_t order); -static FORCE_INLINE SResultRow *getResultRow(SWindowResInfo *pWindowResInfo, int32_t slot) { +static FORCE_INLINE SResultRow *getResultRow(SResultRowInfo *pWindowResInfo, int32_t slot) { assert(pWindowResInfo != NULL && slot >= 0 && slot < pWindowResInfo->size); return pWindowResInfo->pResult[slot]; } @@ -50,7 +50,7 @@ static FORCE_INLINE SResultRow *getResultRow(SWindowResInfo *pWindowResInfo, int #define curTimeWindowIndex(_winres) ((_winres)->curIndex) #define GET_ROW_PARAM_FOR_MULTIOUTPUT(_q, tbq, sq) (((tbq) && (!sq))? (_q)->pExpr1[1].base.arg->argValue.i64:1) -bool isWindowResClosed(SWindowResInfo *pWindowResInfo, int32_t slot); +bool isWindowResClosed(SResultRowInfo *pWindowResInfo, int32_t slot); int32_t initResultRow(SResultRow *pResultRow); diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index bfdcd6d28a..3e683609d1 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -461,7 +461,7 @@ static bool hasNullValue(SColIndex* pColIndex, SDataStatis *pStatis, SDataStatis return true; } -static SResultRow *doPrepareResultRowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SWindowResInfo *pWindowResInfo, char *pData, +static SResultRow *doPrepareResultRowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pWindowResInfo, char *pData, int16_t bytes, bool masterscan, uint64_t uid) { SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, pData, bytes, uid); int32_t *p1 = @@ -518,7 +518,7 @@ static SResultRow *doPrepareResultRowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SWin } // get the correct time window according to the handled timestamp -static STimeWindow getActiveTimeWindow(SWindowResInfo *pWindowResInfo, int64_t ts, SQuery *pQuery) { +static STimeWindow getActiveTimeWindow(SResultRowInfo *pWindowResInfo, int64_t ts, SQuery *pQuery) { STimeWindow w = {0}; if (pWindowResInfo->curIndex == -1) { // the first window, from the previous stored value @@ -611,7 +611,7 @@ static int32_t addNewWindowResultBuf(SResultRow *pWindowRes, SDiskbasedResultBuf return 0; } -static int32_t setWindowOutputBufByKey(SQueryRuntimeEnv *pRuntimeEnv, SWindowResInfo *pWindowResInfo, SDataBlockInfo* pBockInfo, +static int32_t setWindowOutputBufByKey(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pWindowResInfo, SDataBlockInfo* pBockInfo, STimeWindow *win, bool masterscan, bool* newWind, SResultRow** pResult) { assert(win->skey <= win->ekey); SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf; @@ -644,7 +644,7 @@ static int32_t setWindowOutputBufByKey(SQueryRuntimeEnv *pRuntimeEnv, SWindowRes return TSDB_CODE_SUCCESS; } -static bool getResultRowStatus(SWindowResInfo *pWindowResInfo, int32_t slot) { +static bool getResultRowStatus(SResultRowInfo *pWindowResInfo, int32_t slot) { assert(slot >= 0 && slot < pWindowResInfo->size); return pWindowResInfo->pResult[slot]->closed; } @@ -703,7 +703,7 @@ static FORCE_INLINE int32_t getForwardStepsInBlock(int32_t numOfRows, __block_se /** * NOTE: the query status only set for the first scan of master scan. */ -static int32_t doCheckQueryCompleted(SQueryRuntimeEnv *pRuntimeEnv, TSKEY lastKey, SWindowResInfo *pWindowResInfo) { +static int32_t doCheckQueryCompleted(SQueryRuntimeEnv *pRuntimeEnv, TSKEY lastKey, SResultRowInfo *pWindowResInfo) { SQuery *pQuery = pRuntimeEnv->pQuery; if (pRuntimeEnv->scanFlag != MASTER_SCAN) { return pWindowResInfo->size; @@ -1157,7 +1157,7 @@ static TSKEY getStartTsKey(SQuery* pQuery, SDataBlockInfo* pDataBlockInfo, TSKEY * such as count/min/max etc. */ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pStatis, SDataBlockInfo *pDataBlockInfo, - SWindowResInfo *pWindowResInfo, __block_search_fn_t searchFn, SArray *pDataBlock) { + SResultRowInfo *pWindowResInfo, __block_search_fn_t searchFn, SArray *pDataBlock) { SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; bool masterScan = IS_MASTER_SCAN(pRuntimeEnv); @@ -1575,7 +1575,7 @@ static void setTimeWindowEKeyInterp(SQueryRuntimeEnv* pRuntimeEnv, SArray* pData } static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pStatis, SDataBlockInfo *pDataBlockInfo, - SWindowResInfo *pWindowResInfo, SArray *pDataBlock) { + SResultRowInfo *pWindowResInfo, SArray *pDataBlock) { SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; bool masterScan = IS_MASTER_SCAN(pRuntimeEnv); @@ -1781,7 +1781,7 @@ static int32_t tableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBl SQuery *pQuery = pRuntimeEnv->pQuery; STableQueryInfo* pTableQInfo = pQuery->current; - SWindowResInfo* pWindowResInfo = &pRuntimeEnv->windowResInfo; + SResultRowInfo* pWindowResInfo = &pRuntimeEnv->windowResInfo; if (pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTSBuf != NULL || pRuntimeEnv->groupbyNormalCol) { rowwiseApplyFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pWindowResInfo, pDataBlock); @@ -2605,7 +2605,7 @@ static bool overlapWithTimeWindow(SQuery* pQuery, SDataBlockInfo* pBlockInfo) { return false; } -int32_t loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, SWindowResInfo * pWindowResInfo, void* pQueryHandle, SDataBlockInfo* pBlockInfo, SDataStatis **pStatis, SArray** pDataBlock, uint32_t* status) { +int32_t loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo * pWindowResInfo, void* pQueryHandle, SDataBlockInfo* pBlockInfo, SDataStatis **pStatis, SArray** pDataBlock, uint32_t* status) { SQuery *pQuery = pRuntimeEnv->pQuery; *status = BLK_DATA_NO_NEEDED; @@ -2828,7 +2828,7 @@ static void doSetInitialTimewindow(SQueryRuntimeEnv* pRuntimeEnv, SDataBlockInfo if (QUERY_IS_INTERVAL_QUERY(pQuery) && pRuntimeEnv->windowResInfo.prevSKey == TSKEY_INITIAL_VAL) { STimeWindow w = TSWINDOW_INITIALIZER; - SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; + SResultRowInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; if (QUERY_IS_ASC_QUERY(pQuery)) { getAlignQueryTimeWindow(pQuery, pBlockInfo->window.skey, pBlockInfo->window.skey, pQuery->window.ekey, &w); @@ -3192,14 +3192,14 @@ int32_t tableResultComparFn(const void *pLeft, const void *pRight, void *param) return -1; } - SWindowResInfo *pWindowResInfo1 = &supporter->pTableQueryInfo[left]->windowResInfo; + SResultRowInfo *pWindowResInfo1 = &supporter->pTableQueryInfo[left]->windowResInfo; SResultRow * pWindowRes1 = getResultRow(pWindowResInfo1, leftPos); tFilePage *page1 = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes1->pageId); char *b1 = getPosInResultPage(pRuntimeEnv, PRIMARYKEY_TIMESTAMP_COL_INDEX, pWindowRes1, page1); TSKEY leftTimestamp = GET_INT64_VAL(b1); - SWindowResInfo *pWindowResInfo2 = &supporter->pTableQueryInfo[right]->windowResInfo; + SResultRowInfo *pWindowResInfo2 = &supporter->pTableQueryInfo[right]->windowResInfo; SResultRow * pWindowRes2 = getResultRow(pWindowResInfo2, rightPos); tFilePage *page2 = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes2->pageId); @@ -3439,7 +3439,7 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { int32_t pos = pTree->pNode[0].index; - SWindowResInfo *pWindowResInfo = &pTableList[pos]->windowResInfo; + SResultRowInfo *pWindowResInfo = &pTableList[pos]->windowResInfo; SResultRow *pWindowRes = getResultRow(pWindowResInfo, cs.position[pos]); tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes->pageId); @@ -3603,7 +3603,7 @@ static void updateTableQueryInfoForReverseScan(SQuery *pQuery, STableQueryInfo * pTableQueryInfo->windowResInfo.curIndex = pTableQueryInfo->windowResInfo.size - 1; } -static void disableFuncInReverseScanImpl(SQueryRuntimeEnv* pRuntimeEnv, SWindowResInfo *pWindowResInfo, int32_t order) { +static void disableFuncInReverseScanImpl(SQueryRuntimeEnv* pRuntimeEnv, SResultRowInfo *pWindowResInfo, int32_t order) { SQuery* pQuery = pRuntimeEnv->pQuery; for (int32_t i = 0; i < pWindowResInfo->size; ++i) { @@ -3635,7 +3635,7 @@ void disableFuncInReverseScan(SQInfo *pQInfo) { int32_t order = pQuery->order.order; // group by normal columns and interval query on normal table - SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; + SResultRowInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; if (pRuntimeEnv->groupbyNormalCol || QUERY_IS_INTERVAL_QUERY(pQuery)) { disableFuncInReverseScanImpl(pRuntimeEnv, pWindowResInfo, order); } else { // for simple result of table query, @@ -3826,7 +3826,7 @@ bool needScanDataBlocksAgain(SQueryRuntimeEnv *pRuntimeEnv) { bool toContinue = false; if (pRuntimeEnv->groupbyNormalCol || QUERY_IS_INTERVAL_QUERY(pQuery)) { // for each group result, call the finalize function for each column - SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; + SResultRowInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; for (int32_t i = 0; i < pWindowResInfo->size; ++i) { SResultRow *pResult = getResultRow(pWindowResInfo, i); @@ -4038,7 +4038,7 @@ void finalizeQueryResult(SQueryRuntimeEnv *pRuntimeEnv) { if (pRuntimeEnv->groupbyNormalCol || QUERY_IS_INTERVAL_QUERY(pQuery)) { // for each group result, call the finalize function for each column - SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; + SResultRowInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; if (pRuntimeEnv->groupbyNormalCol) { closeAllTimeWindow(pWindowResInfo); } @@ -4122,7 +4122,7 @@ void destroyTableQueryInfoImpl(STableQueryInfo *pTableQueryInfo) { void setExecutionContext(SQInfo *pQInfo, int32_t groupIndex, TSKEY nextKey) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; STableQueryInfo *pTableQueryInfo = pRuntimeEnv->pQuery->current; - SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; + SResultRowInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; // lastKey needs to be updated pTableQueryInfo->lastKey = nextKey; @@ -4290,7 +4290,7 @@ void setIntervalQueryRange(SQInfo *pQInfo, TSKEY key) { * operations involve. */ STimeWindow w = TSWINDOW_INITIALIZER; - SWindowResInfo *pWindowResInfo = &pTableQueryInfo->windowResInfo; + SResultRowInfo *pWindowResInfo = &pTableQueryInfo->windowResInfo; TSKEY sk = MIN(win.skey, win.ekey); TSKEY ek = MAX(win.skey, win.ekey); @@ -4334,7 +4334,7 @@ bool needPrimaryTimestampCol(SQuery *pQuery, SDataBlockInfo *pDataBlockInfo) { return loadPrimaryTS; } -static int32_t doCopyToSData(SQInfo *pQInfo, SWindowResInfo *pResultInfo, int32_t orderType) { +static int32_t doCopyToSData(SQInfo *pQInfo, SResultRowInfo *pResultInfo, int32_t orderType) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery * pQuery = pRuntimeEnv->pQuery; @@ -4411,7 +4411,7 @@ static int32_t doCopyToSData(SQInfo *pQInfo, SWindowResInfo *pResultInfo, int32_ * @param pQInfo * @param result */ -void copyFromWindowResToSData(SQInfo *pQInfo, SWindowResInfo *pResultInfo) { +void copyFromWindowResToSData(SQInfo *pQInfo, SResultRowInfo *pResultInfo) { SQuery *pQuery = pQInfo->runtimeEnv.pQuery; int32_t orderType = (pQuery->pGroupbyExpr != NULL) ? pQuery->pGroupbyExpr->orderType : TSDB_ORDER_ASC; @@ -4450,7 +4450,7 @@ static void stableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBloc SQuery * pQuery = pRuntimeEnv->pQuery; STableQueryInfo* pTableQueryInfo = pQuery->current; - SWindowResInfo * pWindowResInfo = &pTableQueryInfo->windowResInfo; + SResultRowInfo * pWindowResInfo = &pTableQueryInfo->windowResInfo; pQuery->pos = QUERY_IS_ASC_QUERY(pQuery)? 0 : pDataBlockInfo->rows - 1; if (pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTSBuf != NULL || pRuntimeEnv->groupbyNormalCol) { @@ -4698,7 +4698,7 @@ void skipBlocks(SQueryRuntimeEnv *pRuntimeEnv) { static TSKEY doSkipIntervalProcess(SQueryRuntimeEnv* pRuntimeEnv, STimeWindow* win, SDataBlockInfo* pBlockInfo, STableQueryInfo* pTableQueryInfo) { SQuery *pQuery = pRuntimeEnv->pQuery; - SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; + SResultRowInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; assert(pQuery->limit.offset == 0); STimeWindow tw = *win; @@ -4764,7 +4764,7 @@ static bool skipTimeInterval(SQueryRuntimeEnv *pRuntimeEnv, TSKEY* start) { STimeWindow w = TSWINDOW_INITIALIZER; - SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; + SResultRowInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; STableQueryInfo *pTableQueryInfo = pQuery->current; SDataBlockInfo blockInfo = SDATA_BLOCK_INITIALIZER; @@ -5403,7 +5403,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) { scanMultiTableDataBlocks(pQInfo); pQInfo->groupIndex += 1; - SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; + SResultRowInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; // no results generated for current group, continue to try the next group taosArrayDestroy(s); diff --git a/src/query/src/qUtil.c b/src/query/src/qUtil.c index 3f010a579d..8260d8bc15 100644 --- a/src/query/src/qUtil.c +++ b/src/query/src/qUtil.c @@ -43,7 +43,7 @@ int32_t getOutputInterResultBufSize(SQuery* pQuery) { return size; } -int32_t initWindowResInfo(SWindowResInfo *pWindowResInfo, int32_t size, int16_t type) { +int32_t initWindowResInfo(SResultRowInfo *pWindowResInfo, int32_t size, int16_t type) { pWindowResInfo->capacity = size; pWindowResInfo->type = type; @@ -59,7 +59,7 @@ int32_t initWindowResInfo(SWindowResInfo *pWindowResInfo, int32_t size, int16_t return TSDB_CODE_SUCCESS; } -void cleanupTimeWindowInfo(SWindowResInfo *pWindowResInfo) { +void cleanupTimeWindowInfo(SResultRowInfo *pWindowResInfo) { if (pWindowResInfo == NULL) { return; } @@ -77,7 +77,7 @@ void cleanupTimeWindowInfo(SWindowResInfo *pWindowResInfo) { tfree(pWindowResInfo->pResult); } -void resetTimeWindowInfo(SQueryRuntimeEnv *pRuntimeEnv, SWindowResInfo *pWindowResInfo) { +void resetTimeWindowInfo(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pWindowResInfo) { if (pWindowResInfo == NULL || pWindowResInfo->capacity == 0) { return; } @@ -101,7 +101,7 @@ void resetTimeWindowInfo(SQueryRuntimeEnv *pRuntimeEnv, SWindowResInfo *pWindowR } void clearFirstNWindowRes(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) { - SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; + SResultRowInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; if (pWindowResInfo == NULL || pWindowResInfo->capacity == 0 || pWindowResInfo->size == 0 || num == 0) { return; } @@ -160,7 +160,7 @@ void clearFirstNWindowRes(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) { } void clearClosedTimeWindow(SQueryRuntimeEnv *pRuntimeEnv) { - SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; + SResultRowInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; if (pWindowResInfo == NULL || pWindowResInfo->capacity == 0 || pWindowResInfo->size == 0) { return; } @@ -169,7 +169,7 @@ void clearClosedTimeWindow(SQueryRuntimeEnv *pRuntimeEnv) { clearFirstNWindowRes(pRuntimeEnv, numOfClosed); } -int32_t numOfClosedTimeWindow(SWindowResInfo *pWindowResInfo) { +int32_t numOfClosedTimeWindow(SResultRowInfo *pWindowResInfo) { int32_t i = 0; while (i < pWindowResInfo->size && pWindowResInfo->pResult[i]->closed) { ++i; @@ -178,7 +178,7 @@ int32_t numOfClosedTimeWindow(SWindowResInfo *pWindowResInfo) { return i; } -void closeAllTimeWindow(SWindowResInfo *pWindowResInfo) { +void closeAllTimeWindow(SResultRowInfo *pWindowResInfo) { assert(pWindowResInfo->size >= 0 && pWindowResInfo->capacity >= pWindowResInfo->size); for (int32_t i = 0; i < pWindowResInfo->size; ++i) { @@ -195,7 +195,7 @@ void closeAllTimeWindow(SWindowResInfo *pWindowResInfo) { * the last qualified time stamp in case of sliding query, which the sliding time is not equalled to the interval time. * NOTE: remove redundant, only when the result set order equals to traverse order */ -void removeRedundantWindow(SWindowResInfo *pWindowResInfo, TSKEY lastKey, int32_t order) { +void removeRedundantWindow(SResultRowInfo *pWindowResInfo, TSKEY lastKey, int32_t order) { assert(pWindowResInfo->size >= 0 && pWindowResInfo->capacity >= pWindowResInfo->size); if (pWindowResInfo->size <= 1) { return; @@ -224,11 +224,11 @@ void removeRedundantWindow(SWindowResInfo *pWindowResInfo, TSKEY lastKey, int32_ } } -bool isWindowResClosed(SWindowResInfo *pWindowResInfo, int32_t slot) { +bool isWindowResClosed(SResultRowInfo *pWindowResInfo, int32_t slot) { return (getResultRow(pWindowResInfo, slot)->closed == true); } -void closeTimeWindow(SWindowResInfo *pWindowResInfo, int32_t slot) { +void closeTimeWindow(SResultRowInfo *pWindowResInfo, int32_t slot) { getResultRow(pWindowResInfo, slot)->closed = true; } From 597dd6341cc0c5eba5bf3c6dfbd27fd6f9b9bd4e Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 2 Dec 2020 18:46:28 +0800 Subject: [PATCH 12/58] [TD-225] --- src/query/src/qExecutor.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 3e683609d1..f47cd34729 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -5515,6 +5515,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) { // the limitation of output result is reached, set the query completed if (limitResults(pRuntimeEnv)) { + setQueryStatus(pQuery, QUERY_COMPLETED); SET_STABLE_QUERY_OVER(pQInfo); break; } From 8cc7daa929e2e0d079c92ad80aed67abbe048400 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 2 Dec 2020 19:39:31 +0800 Subject: [PATCH 13/58] [TD-2308] doc: add more items to faq document. --- documentation20/webdocs/markdowndocs/faq-ch.md | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/documentation20/webdocs/markdowndocs/faq-ch.md b/documentation20/webdocs/markdowndocs/faq-ch.md index 757b6d9929..42f9761c95 100644 --- a/documentation20/webdocs/markdowndocs/faq-ch.md +++ b/documentation20/webdocs/markdowndocs/faq-ch.md @@ -101,7 +101,20 @@ Connection = DriverManager.getConnection(url, properties); 2.0.4 ``` -## 14. 怎么报告问题？ +## 14. taos connect failed, reason: invalid timestamp + +常见原因是服务器和客户端时间没有校准，可以通过和时间服务器同步的方式（Linux 下使用 ntpdate 命令，Windows 在系统时间设置中选择自动同步）校准。 + + + +## 15. 表名显示不全 + +由于 taos shell 在终端中显示宽度有限，有可能比较长的表名显示不全，如果按照显示的不全的表名进行相关操作会发生 Table does not exist 错误。解决方法可以是通过修改 taos.cfg 文件中的设置项 maxBinaryDisplayWidth， 或者直接输入命令 set max_binary_display_width 100。或者在命令结尾使用 \G 参数来调整结果的显示方式。 + + + +## 16. 怎么报告问题？ + 如果 FAQ 中的信息不能够帮到您，需要 TDengine 技术团队的技术支持与协助，请将以下两个目录中内容打包: 1. /var/log/taos 2. /etc/taos From d59b8f18da1c940c4df942b6082bb05c280fff5b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 3 Dec 2020 10:32:55 +0800 Subject: [PATCH 14/58] [TD-2298] --- src/query/src/qExecutor.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index f47cd34729..1b2d3f12f5 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -3978,13 +3978,14 @@ void scanOneTableDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start) { qstatus.status = pQuery->status; // do nothing if no data blocks are found qualified during scan - qstatus.lastKey = pTableQueryInfo->lastKey; if (qstatus.lastKey != pTableQueryInfo->lastKey) { qstatus.curWindow.ekey = pTableQueryInfo->lastKey - step; } else { // the lastkey does not increase, which means no data checked yet qDebug("QInfo:%p no results generated in this scan, abort", pQInfo); return; } + + qstatus.lastKey = pTableQueryInfo->lastKey; } if (!needScanDataBlocksAgain(pRuntimeEnv)) { From caeabf382148fc8453ccb2422c563468a47d3e60 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 3 Dec 2020 10:33:40 +0800 Subject: [PATCH 15/58] [TD-225] refactor codes. --- src/client/inc/tsclient.h | 5 +- src/client/src/tscSQLParser.c | 4 +- src/client/src/tscServer.c | 1 + src/query/src/qExecutor.c | 4 +- src/query/src/qUtil.c | 126 +++++++++++++++++----------------- 5 files changed, 72 insertions(+), 68 deletions(-) diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index c482375bff..29841035f7 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -246,11 +246,14 @@ typedef struct SQueryInfo { int16_t fillType; // final result fill type int16_t numOfTables; STableMetaInfo **pTableMetaInfo; - struct STSBuf * tsBuf; + struct STSBuf *tsBuf; int64_t * fillVal; // default value for fill char * msg; // pointer to the pCmd->payload to keep error message temporarily int64_t clauseLimit; // limit for current sub clause + int64_t prjOffset; // offset value in the original sql expression, only applied at client side + int64_t tableLimit; // table limit in case of super table projection query + global order + limit + int32_t udColumnId; // current user-defined constant output field column id, monotonically decreases from TSDB_UD_COLUMN_INDEX int16_t resColumnId; // result column id } SQueryInfo; diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 0c9ad78cd9..abbe26204d 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -5307,7 +5307,8 @@ int32_t parseLimitClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t clauseIn // keep original limitation value in globalLimit pQueryInfo->clauseLimit = pQueryInfo->limit.limit; - pQueryInfo->prjOffset = pQueryInfo->limit.offset; + pQueryInfo->prjOffset = pQueryInfo->limit.offset; + pQueryInfo->tableLimit = -1; if (tscOrderedProjectionQueryOnSTable(pQueryInfo, 0)) { /* @@ -5317,6 +5318,7 @@ int32_t parseLimitClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t clauseIn * than or equal to the value of limit. */ if (pQueryInfo->limit.limit > 0) { + pQueryInfo->tableLimit = pQueryInfo->limit.limit + pQueryInfo->limit.offset; pQueryInfo->limit.limit = -1; } diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 9735e26ece..5fc09c0586 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -697,6 +697,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pQueryMsg->tagNameRelType = htons(pQueryInfo->tagCond.relType); pQueryMsg->numOfTags = htonl(numOfTags); pQueryMsg->queryType = htonl(pQueryInfo->type); + pQueryMsg->tableLimit = htobe64(pQueryInfo->tableLimit); size_t numOfOutput = tscSqlExprNumOfExprs(pQueryInfo); pQueryMsg->numOfOutput = htons((int16_t)numOfOutput); // this is the stage one output column number diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 1b2d3f12f5..03de6aea1f 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -6260,11 +6260,9 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, pQueryMsg->interval.interval = htobe64(pQueryMsg->interval.interval); pQueryMsg->interval.sliding = htobe64(pQueryMsg->interval.sliding); pQueryMsg->interval.offset = htobe64(pQueryMsg->interval.offset); - // pQueryMsg->interval.intervalUnit = pQueryMsg->interval.intervalUnit; - // pQueryMsg->interval.slidingUnit = pQueryMsg->interval.slidingUnit; - // pQueryMsg->interval.offsetUnit = pQueryMsg->interval.offsetUnit; pQueryMsg->limit = htobe64(pQueryMsg->limit); pQueryMsg->offset = htobe64(pQueryMsg->offset); + pQueryMsg->tableLimit = htobe64(pQueryMsg->tableLimit); pQueryMsg->order = htons(pQueryMsg->order); pQueryMsg->orderColId = htons(pQueryMsg->orderColId); diff --git a/src/query/src/qUtil.c b/src/query/src/qUtil.c index 8260d8bc15..6c845b012f 100644 --- a/src/query/src/qUtil.c +++ b/src/query/src/qUtil.c @@ -43,48 +43,48 @@ int32_t getOutputInterResultBufSize(SQuery* pQuery) { return size; } -int32_t initWindowResInfo(SResultRowInfo *pWindowResInfo, int32_t size, int16_t type) { - pWindowResInfo->capacity = size; +int32_t initWindowResInfo(SResultRowInfo *pResultRowInfo, int32_t size, int16_t type) { + pResultRowInfo->capacity = size; - pWindowResInfo->type = type; - pWindowResInfo->curIndex = -1; - pWindowResInfo->size = 0; - pWindowResInfo->prevSKey = TSKEY_INITIAL_VAL; + pResultRowInfo->type = type; + pResultRowInfo->curIndex = -1; + pResultRowInfo->size = 0; + pResultRowInfo->prevSKey = TSKEY_INITIAL_VAL; - pWindowResInfo->pResult = calloc(pWindowResInfo->capacity, POINTER_BYTES); - if (pWindowResInfo->pResult == NULL) { + pResultRowInfo->pResult = calloc(pResultRowInfo->capacity, POINTER_BYTES); + if (pResultRowInfo->pResult == NULL) { return TSDB_CODE_QRY_OUT_OF_MEMORY; } return TSDB_CODE_SUCCESS; } -void cleanupTimeWindowInfo(SResultRowInfo *pWindowResInfo) { - if (pWindowResInfo == NULL) { +void cleanupTimeWindowInfo(SResultRowInfo *pResultRowInfo) { + if (pResultRowInfo == NULL) { return; } - if (pWindowResInfo->capacity == 0) { - assert(pWindowResInfo->pResult == NULL); + if (pResultRowInfo->capacity == 0) { + assert(pResultRowInfo->pResult == NULL); return; } - if (pWindowResInfo->type == TSDB_DATA_TYPE_BINARY || pWindowResInfo->type == TSDB_DATA_TYPE_NCHAR) { - for(int32_t i = 0; i < pWindowResInfo->size; ++i) { - tfree(pWindowResInfo->pResult[i]->key); + if (pResultRowInfo->type == TSDB_DATA_TYPE_BINARY || pResultRowInfo->type == TSDB_DATA_TYPE_NCHAR) { + for(int32_t i = 0; i < pResultRowInfo->size; ++i) { + tfree(pResultRowInfo->pResult[i]->key); } } - tfree(pWindowResInfo->pResult); + tfree(pResultRowInfo->pResult); } -void resetTimeWindowInfo(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pWindowResInfo) { - if (pWindowResInfo == NULL || pWindowResInfo->capacity == 0) { +void resetTimeWindowInfo(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRowInfo) { + if (pResultRowInfo == NULL || pResultRowInfo->capacity == 0) { return; } - for (int32_t i = 0; i < pWindowResInfo->size; ++i) { - SResultRow *pWindowRes = pWindowResInfo->pResult[i]; - clearResultRow(pRuntimeEnv, pWindowRes, pWindowResInfo->type); + for (int32_t i = 0; i < pResultRowInfo->size; ++i) { + SResultRow *pWindowRes = pResultRowInfo->pResult[i]; + clearResultRow(pRuntimeEnv, pWindowRes, pResultRowInfo->type); int32_t groupIndex = 0; int64_t uid = 0; @@ -93,30 +93,30 @@ void resetTimeWindowInfo(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pWindowR taosHashRemove(pRuntimeEnv->pResultRowHashTable, (const char *)pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(sizeof(groupIndex))); } - pWindowResInfo->curIndex = -1; - pWindowResInfo->size = 0; + pResultRowInfo->curIndex = -1; + pResultRowInfo->size = 0; - pWindowResInfo->startTime = TSKEY_INITIAL_VAL; - pWindowResInfo->prevSKey = TSKEY_INITIAL_VAL; + pResultRowInfo->startTime = TSKEY_INITIAL_VAL; + pResultRowInfo->prevSKey = TSKEY_INITIAL_VAL; } void clearFirstNWindowRes(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) { - SResultRowInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; - if (pWindowResInfo == NULL || pWindowResInfo->capacity == 0 || pWindowResInfo->size == 0 || num == 0) { + SResultRowInfo *pResultRowInfo = &pRuntimeEnv->windowResInfo; + if (pResultRowInfo == NULL || pResultRowInfo->capacity == 0 || pResultRowInfo->size == 0 || num == 0) { return; } - int32_t numOfClosed = numOfClosedTimeWindow(pWindowResInfo); + int32_t numOfClosed = numOfClosedTimeWindow(pResultRowInfo); assert(num >= 0 && num <= numOfClosed); - int16_t type = pWindowResInfo->type; + int16_t type = pResultRowInfo->type; int64_t uid = getResultInfoUId(pRuntimeEnv); char *key = NULL; int16_t bytes = -1; for (int32_t i = 0; i < num; ++i) { - SResultRow *pResult = pWindowResInfo->pResult[i]; + SResultRow *pResult = pResultRowInfo->pResult[i]; if (pResult->closed) { // remove the window slot from hash table getResultRowKeyInfo(pResult, type, &key, &bytes); SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, key, bytes, uid); @@ -126,23 +126,23 @@ void clearFirstNWindowRes(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) { } } - int32_t remain = pWindowResInfo->size - num; + int32_t remain = pResultRowInfo->size - num; // clear all the closed windows from the window list for (int32_t k = 0; k < remain; ++k) { - copyResultRow(pRuntimeEnv, pWindowResInfo->pResult[k], pWindowResInfo->pResult[num + k], type); + copyResultRow(pRuntimeEnv, pResultRowInfo->pResult[k], pResultRowInfo->pResult[num + k], type); } // move the unclosed window in the front of the window list - for (int32_t k = remain; k < pWindowResInfo->size; ++k) { - SResultRow *pWindowRes = pWindowResInfo->pResult[k]; - clearResultRow(pRuntimeEnv, pWindowRes, pWindowResInfo->type); + for (int32_t k = remain; k < pResultRowInfo->size; ++k) { + SResultRow *pWindowRes = pResultRowInfo->pResult[k]; + clearResultRow(pRuntimeEnv, pWindowRes, pResultRowInfo->type); } - pWindowResInfo->size = remain; + pResultRowInfo->size = remain; - for (int32_t k = 0; k < pWindowResInfo->size; ++k) { - SResultRow *pResult = pWindowResInfo->pResult[k]; + for (int32_t k = 0; k < pResultRowInfo->size; ++k) { + SResultRow *pResult = pResultRowInfo->pResult[k]; getResultRowKeyInfo(pResult, type, &key, &bytes); SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, key, bytes, uid); @@ -150,43 +150,43 @@ void clearFirstNWindowRes(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) { assert(p != NULL); int32_t v = (*p - num); - assert(v >= 0 && v <= pWindowResInfo->size); + assert(v >= 0 && v <= pResultRowInfo->size); SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, key, bytes, uid); taosHashPut(pRuntimeEnv->pResultRowHashTable, pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes), (char *)&v, sizeof(int32_t)); } - pWindowResInfo->curIndex = -1; + pResultRowInfo->curIndex = -1; } void clearClosedTimeWindow(SQueryRuntimeEnv *pRuntimeEnv) { - SResultRowInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; - if (pWindowResInfo == NULL || pWindowResInfo->capacity == 0 || pWindowResInfo->size == 0) { + SResultRowInfo *pResultRowInfo = &pRuntimeEnv->windowResInfo; + if (pResultRowInfo == NULL || pResultRowInfo->capacity == 0 || pResultRowInfo->size == 0) { return; } - int32_t numOfClosed = numOfClosedTimeWindow(pWindowResInfo); + int32_t numOfClosed = numOfClosedTimeWindow(pResultRowInfo); clearFirstNWindowRes(pRuntimeEnv, numOfClosed); } -int32_t numOfClosedTimeWindow(SResultRowInfo *pWindowResInfo) { +int32_t numOfClosedTimeWindow(SResultRowInfo *pResultRowInfo) { int32_t i = 0; - while (i < pWindowResInfo->size && pWindowResInfo->pResult[i]->closed) { + while (i < pResultRowInfo->size && pResultRowInfo->pResult[i]->closed) { ++i; } return i; } -void closeAllTimeWindow(SResultRowInfo *pWindowResInfo) { - assert(pWindowResInfo->size >= 0 && pWindowResInfo->capacity >= pWindowResInfo->size); +void closeAllTimeWindow(SResultRowInfo *pResultRowInfo) { + assert(pResultRowInfo->size >= 0 && pResultRowInfo->capacity >= pResultRowInfo->size); - for (int32_t i = 0; i < pWindowResInfo->size; ++i) { - if (pWindowResInfo->pResult[i]->closed) { + for (int32_t i = 0; i < pResultRowInfo->size; ++i) { + if (pResultRowInfo->pResult[i]->closed) { continue; } - pWindowResInfo->pResult[i]->closed = true; + pResultRowInfo->pResult[i]->closed = true; } } @@ -195,41 +195,41 @@ void closeAllTimeWindow(SResultRowInfo *pWindowResInfo) { * the last qualified time stamp in case of sliding query, which the sliding time is not equalled to the interval time. * NOTE: remove redundant, only when the result set order equals to traverse order */ -void removeRedundantWindow(SResultRowInfo *pWindowResInfo, TSKEY lastKey, int32_t order) { - assert(pWindowResInfo->size >= 0 && pWindowResInfo->capacity >= pWindowResInfo->size); - if (pWindowResInfo->size <= 1) { +void removeRedundantWindow(SResultRowInfo *pResultRowInfo, TSKEY lastKey, int32_t order) { + assert(pResultRowInfo->size >= 0 && pResultRowInfo->capacity >= pResultRowInfo->size); + if (pResultRowInfo->size <= 1) { return; } // get the result order - int32_t resultOrder = (pWindowResInfo->pResult[0]->win.skey < pWindowResInfo->pResult[1]->win.skey)? 1:-1; + int32_t resultOrder = (pResultRowInfo->pResult[0]->win.skey < pResultRowInfo->pResult[1]->win.skey)? 1:-1; if (order != resultOrder) { return; } int32_t i = 0; if (order == QUERY_ASC_FORWARD_STEP) { - TSKEY ekey = pWindowResInfo->pResult[i]->win.ekey; - while (i < pWindowResInfo->size && (ekey < lastKey)) { + TSKEY ekey = pResultRowInfo->pResult[i]->win.ekey; + while (i < pResultRowInfo->size && (ekey < lastKey)) { ++i; } } else if (order == QUERY_DESC_FORWARD_STEP) { - while (i < pWindowResInfo->size && (pWindowResInfo->pResult[i]->win.skey > lastKey)) { + while (i < pResultRowInfo->size && (pResultRowInfo->pResult[i]->win.skey > lastKey)) { ++i; } } - if (i < pWindowResInfo->size) { - pWindowResInfo->size = (i + 1); + if (i < pResultRowInfo->size) { + pResultRowInfo->size = (i + 1); } } -bool isWindowResClosed(SResultRowInfo *pWindowResInfo, int32_t slot) { - return (getResultRow(pWindowResInfo, slot)->closed == true); +bool isWindowResClosed(SResultRowInfo *pResultRowInfo, int32_t slot) { + return (getResultRow(pResultRowInfo, slot)->closed == true); } -void closeTimeWindow(SResultRowInfo *pWindowResInfo, int32_t slot) { - getResultRow(pWindowResInfo, slot)->closed = true; +void closeTimeWindow(SResultRowInfo *pResultRowInfo, int32_t slot) { + getResultRow(pResultRowInfo, slot)->closed = true; } void clearResultRow(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pWindowRes, int16_t type) { From ef901f1077da070c6f555eb5041c9a99a0316c02 Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Thu, 3 Dec 2020 03:29:10 +0000 Subject: [PATCH 16/58] [TD-2309]: add config option to enable/disable stream (continuous query) --- .../webdocs/markdowndocs/administrator-ch.md | 1 + packaging/cfg/taos.cfg | 5 ++++- src/common/inc/tglobal.h | 3 +++ src/common/src/tglobal.c | 13 +++++++++++ src/cq/src/cqMain.c | 18 +++++++++++++++ src/vnode/src/vnodeMain.c | 22 ++++++++++--------- 6 files changed, 51 insertions(+), 11 deletions(-) diff --git a/documentation20/webdocs/markdowndocs/administrator-ch.md b/documentation20/webdocs/markdowndocs/administrator-ch.md index 36466d2b7e..f54c6b91a1 100644 --- a/documentation20/webdocs/markdowndocs/administrator-ch.md +++ b/documentation20/webdocs/markdowndocs/administrator-ch.md @@ -95,6 +95,7 @@ TDengine系统后台服务由taosd提供，可以在配置文件taos.cfg里修 - logKeepDays：日志文件的最长保存时间。大于0时，日志文件会被重命名为taosdlog.xxx，其中xxx为日志文件最后修改的时间戳，单位为秒。默认值：0天。 - maxSQLLength：单条SQL语句允许最长限制。默认值：65380字节。 - telemetryReporting: 是否允许 TDengine 采集和上报基本使用信息，0表示不允许，1表示允许。 默认值：1。 +- stream: 是否启用连续查询（流计算功能），0表示不允许，1表示允许。 默认值：1。 **注意：**对于端口，TDengine会使用从serverPort起13个连续的TCP和UDP端口号，请务必在防火墙打开。因此如果是缺省配置，需要打开从6030都6042共13个端口，而且必须TCP和UDP都打开。 diff --git a/packaging/cfg/taos.cfg b/packaging/cfg/taos.cfg index ca88bca3c8..7662d49280 100644 --- a/packaging/cfg/taos.cfg +++ b/packaging/cfg/taos.cfg @@ -260,4 +260,7 @@ # maxBinaryDisplayWidth 30 # enable/disable telemetry reporting -# telemetryReporting 1 \ No newline at end of file +# telemetryReporting 1 + +# enable/disable stream (continuous query) +# stream 1 diff --git a/src/common/inc/tglobal.h b/src/common/inc/tglobal.h index efe3d7678a..6e4274b358 100644 --- a/src/common/inc/tglobal.h +++ b/src/common/inc/tglobal.h @@ -125,6 +125,9 @@ extern char tsMonitorDbName[]; extern char tsInternalPass[]; extern int32_t tsMonitorInterval; +// stream +extern int32_t tsEnableStream; + // internal extern int32_t tsPrintAuth; extern int32_t tscEmbedded; diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index d26746a5d1..18aa0ae6e3 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -161,6 +161,9 @@ char tsMonitorDbName[TSDB_DB_NAME_LEN] = "log"; char tsInternalPass[] = "secretkey"; int32_t tsMonitorInterval = 30; // seconds +// stream +int32_t tsEnableStream = 1; + // internal int32_t tsPrintAuth = 0; int32_t tscEmbedded = 0; @@ -1015,6 +1018,16 @@ static void doInitGlobalConfig(void) { cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); + cfg.option = "stream"; + cfg.ptr = &tsEnableStream; + cfg.valType = TAOS_CFG_VTYPE_INT32; + cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; + cfg.minValue = 0; + cfg.maxValue = 1; + cfg.ptrLength = 1; + cfg.unitType = TAOS_CFG_UTYPE_NONE; + taosInitConfigOption(cfg); + cfg.option = "httpEnableRecordSql"; cfg.ptr = &tsHttpEnableRecordSql; cfg.valType = TAOS_CFG_VTYPE_INT32; diff --git a/src/cq/src/cqMain.c b/src/cq/src/cqMain.c index b1977fd5d9..efb8795962 100644 --- a/src/cq/src/cqMain.c +++ b/src/cq/src/cqMain.c @@ -69,6 +69,9 @@ static void cqProcessStreamRes(void *param, TAOS_RES *tres, TAOS_ROW row); static void cqCreateStream(SCqContext *pContext, SCqObj *pObj); void *cqOpen(void *ahandle, const SCqCfg *pCfg) { + if (tsEnableStream == 0) { + return NULL; + } SCqContext *pContext = calloc(sizeof(SCqContext), 1); if (pContext == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); @@ -99,6 +102,9 @@ void *cqOpen(void *ahandle, const SCqCfg *pCfg) { } void cqClose(void *handle) { + if (tsEnableStream == 0) { + return; + } SCqContext *pContext = handle; if (handle == NULL) return; @@ -129,6 +135,9 @@ void cqClose(void *handle) { } void cqStart(void *handle) { + if (tsEnableStream == 0) { + return; + } SCqContext *pContext = handle; if (pContext->dbConn || pContext->master) return; @@ -147,6 +156,9 @@ void cqStart(void *handle) { } void cqStop(void *handle) { + if (tsEnableStream == 0) { + return; + } SCqContext *pContext = handle; cInfo("vgId:%d, stop all CQs", pContext->vgId); if (pContext->dbConn == NULL || pContext->master == 0) return; @@ -174,6 +186,9 @@ void cqStop(void *handle) { } void *cqCreate(void *handle, uint64_t uid, int32_t tid, char *sqlStr, STSchema *pSchema) { + if (tsEnableStream == 0) { + return NULL; + } SCqContext *pContext = handle; SCqObj *pObj = calloc(sizeof(SCqObj), 1); @@ -203,6 +218,9 @@ void *cqCreate(void *handle, uint64_t uid, int32_t tid, char *sqlStr, STSchema * } void cqDrop(void *handle) { + if (tsEnableStream == 0) { + return; + } SCqObj *pObj = handle; SCqContext *pContext = pObj->pContext; diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index 204dd2f3dd..b516c9d90e 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -267,16 +267,18 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) { return terrno; } - SCqCfg cqCfg = {0}; - sprintf(cqCfg.user, "_root"); - strcpy(cqCfg.pass, tsInternalPass); - strcpy(cqCfg.db, pVnode->db); - cqCfg.vgId = vnode; - cqCfg.cqWrite = vnodeWriteToCache; - pVnode->cq = cqOpen(pVnode, &cqCfg); - if (pVnode->cq == NULL) { - vnodeCleanUp(pVnode); - return terrno; + if (tsEnableStream) { + SCqCfg cqCfg = {0}; + sprintf(cqCfg.user, "_root"); + strcpy(cqCfg.pass, tsInternalPass); + strcpy(cqCfg.db, pVnode->db); + cqCfg.vgId = vnode; + cqCfg.cqWrite = vnodeWriteToCache; + pVnode->cq = cqOpen(pVnode, &cqCfg); + if (pVnode->cq == NULL) { + vnodeCleanUp(pVnode); + return terrno; + } } STsdbAppH appH = {0}; From 51a8b29a8f4efdfaed88e685a318d34e3108ca1f Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 3 Dec 2020 13:29:13 +0800 Subject: [PATCH 17/58] [TD-2298] --- src/query/src/qExecutor.c | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 03de6aea1f..6ce40b9204 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -3981,8 +3981,7 @@ void scanOneTableDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start) { if (qstatus.lastKey != pTableQueryInfo->lastKey) { qstatus.curWindow.ekey = pTableQueryInfo->lastKey - step; } else { // the lastkey does not increase, which means no data checked yet - qDebug("QInfo:%p no results generated in this scan, abort", pQInfo); - return; + qDebug("QInfo:%p no results generated in this scan", pQInfo); } qstatus.lastKey = pTableQueryInfo->lastKey; @@ -4749,7 +4748,23 @@ static TSKEY doSkipIntervalProcess(SQueryRuntimeEnv* pRuntimeEnv, STimeWindow* w static bool skipTimeInterval(SQueryRuntimeEnv *pRuntimeEnv, TSKEY* start) { SQuery *pQuery = pRuntimeEnv->pQuery; - *start = pQuery->current->lastKey; + + // get the first unclosed time window + bool assign = false; + for(int32_t i = 0; i < pRuntimeEnv->windowResInfo.size; ++i) { + if (pRuntimeEnv->windowResInfo.pResult[i]->closed) { + continue; + } + + assign = true; + *start = pRuntimeEnv->windowResInfo.pResult[i]->win.skey; + } + + if (!assign) { + *start = pQuery->current->lastKey; + } + + assert(*start <= pQuery->current->lastKey); // if queried with value filter, do NOT forward query start position if (pQuery->limit.offset <= 0 || pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTSBuf != NULL || pRuntimeEnv->pFillInfo != NULL) { @@ -5441,7 +5456,6 @@ static void sequentialTableProcess(SQInfo *pQInfo) { } else if (pRuntimeEnv->queryWindowIdentical && pRuntimeEnv->pTSBuf == NULL) { //super table projection query with identical query time range for all tables. SDataBlockInfo blockInfo = SDATA_BLOCK_INITIALIZER; - resetDefaultResInfoOutputBuf(pRuntimeEnv); SArray *group = GET_TABLEGROUP(pQInfo, 0); From 024397f265f5115caa83808890d101f0776debcb Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 3 Dec 2020 06:57:49 +0000 Subject: [PATCH 18/58] fix TD-2314 --- src/tsdb/src/tsdbFile.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/tsdb/src/tsdbFile.c b/src/tsdb/src/tsdbFile.c index 03c50d42f7..5d8933d141 100644 --- a/src/tsdb/src/tsdbFile.c +++ b/src/tsdb/src/tsdbFile.c @@ -248,8 +248,13 @@ SFileGroup *tsdbCreateFGroupIfNeed(STsdbRepo *pRepo, char *dataDir, int fid) { if (pGroup == NULL) { // if not exists, create one pFGroup->fileId = fid; for (int type = 0; type < TSDB_FILE_TYPE_MAX; type++) { - if (tsdbCreateFile(&pFGroup->files[type], pRepo, fid, type) < 0) - goto _err; + if (tsdbCreateFile(&pFGroup->files[type], pRepo, fid, type) < 0) { + for (int i = type; i >= 0; i--) { + remove(pFGroup->files[i].fname); + } + + return NULL; + } } pthread_rwlock_wrlock(&pFileH->fhlock); @@ -261,10 +266,6 @@ SFileGroup *tsdbCreateFGroupIfNeed(STsdbRepo *pRepo, char *dataDir, int fid) { } return pGroup; - -_err: - for (int type = 0; type < TSDB_FILE_TYPE_MAX; type++) tsdbDestroyFile(&pGroup->files[type]); - return NULL; } void tsdbInitFileGroupIter(STsdbFileH *pFileH, SFileGroupIter *pIter, int direction) { From b6d30543d82c5f7c900e839a11d79a9ede653c82 Mon Sep 17 00:00:00 2001 From: zyyang Date: Thu, 3 Dec 2020 15:06:11 +0800 Subject: [PATCH 19/58] change --- .../example/jdbcTaosdemo/domain/JdbcTaosdemoConfig.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/domain/JdbcTaosdemoConfig.java b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/domain/JdbcTaosdemoConfig.java index 82613037db..36745a9394 100644 --- a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/domain/JdbcTaosdemoConfig.java +++ b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/domain/JdbcTaosdemoConfig.java @@ -14,9 +14,9 @@ public final class JdbcTaosdemoConfig { //Destination database. Default is 'test' private String dbName = "test"; //keep - private int keep = 3650; + private int keep = 36500; //days - private int days = 10; + private int days = 120; //Super table Name. Default is 'meters' private String stbName = "meters"; From 9887f533c6e27f03e4e5c7687a97cbfddd2e99dd Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 3 Dec 2020 15:25:43 +0800 Subject: [PATCH 20/58] [TD-2298] add test cases. --- tests/script/general/parser/first_last.sim | 42 ++++++++++++++++++- .../general/parser/first_last_query.sim | 4 +- 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/tests/script/general/parser/first_last.sim b/tests/script/general/parser/first_last.sim index a934d3bcab..773f92afcf 100644 --- a/tests/script/general/parser/first_last.sim +++ b/tests/script/general/parser/first_last.sim @@ -46,7 +46,8 @@ while $i < $tbNum endw $i = $i + 1 -endw +endw + $ts = $ts + 60000 $tb = $tbPrefix . 0 sql insert into $tb (ts) values ( $ts ) @@ -84,4 +85,43 @@ sleep 500 run general/parser/first_last_query.sim +print =================> insert data regression test +sql create database test keep 36500 +sql use test +sql create table tm0 (ts timestamp, k int) + +print =========================> td-2298 +$ts0 = 1537146000000 +$xs = 6000 + +$x = 0 +while $x < 5000 + $ts = $ts0 + $xs + $ts1 = $ts + $xs + $x1 = $x + 1 + + sql insert into tm0 values ( $ts , $x ) ( $ts1 , $x1 ) + $x = $x1 + $ts0 = $ts1 +endw + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +sleep 3000 +system sh/exec.sh -n dnode1 -s start +print ================== server restart completed +sql connect +sleep 500 + +sql use test +sql select count(*), last(ts) from tm0 interval(1s) +if $rows != 10000 then + print expect 10000, actual: $rows + return -1 +endi + +sql select last(ts) from tm0 interval(1s) +if $rows != 10000 then + return -1 +endi + system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/parser/first_last_query.sim b/tests/script/general/parser/first_last_query.sim index a982f10362..52d888b04d 100644 --- a/tests/script/general/parser/first_last_query.sim +++ b/tests/script/general/parser/first_last_query.sim @@ -266,4 +266,6 @@ endi if $data14 != @test2@ then print expect test2 , actual: $data14 return -1 -endi \ No newline at end of file +endi + +sql drop table stest \ No newline at end of file From a837f33731213ee8f5fa9a860e3d762f15ee0cf7 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Thu, 3 Dec 2020 15:43:21 +0800 Subject: [PATCH 21/58] [TD-2298] add test case for interval --- tests/pytest/fulltest.sh | 3 ++- tests/pytest/query/queryInterval.py | 14 ++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index 042fd826e8..448a0413e0 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -159,7 +159,8 @@ python3 ./test.py -f query/bug1471.py python3 ./test.py -f query/bug1874.py python3 ./test.py -f query/bug1875.py python3 ./test.py -f query/bug1876.py -python3 ./test.py -f query/bug2218.py +python3 ./test.py -f query/bug2218.py +python3 ./test.py -f query/sliding.py #stream python3 ./test.py -f stream/metric_1.py diff --git a/tests/pytest/query/queryInterval.py b/tests/pytest/query/queryInterval.py index 98b8c9fbef..871c076c08 100644 --- a/tests/pytest/query/queryInterval.py +++ b/tests/pytest/query/queryInterval.py @@ -16,6 +16,7 @@ import taos from util.log import tdLog from util.cases import tdCases from util.sql import tdSql +from util.dnodes import tdDnodes class TDTestCase: @@ -72,6 +73,19 @@ class TDTestCase: tdSql.checkData(6, 0, "2020-09-16 00:00:00") tdSql.checkData(6, 1, 222.0) + # test case for https://jira.taosdata.com:18080/browse/TD-2298 + tdSql.execute("create database test keep 36500") + tdSql.execute("use test") + tdSql.execute("create table t (ts timestamp, voltage int)") + for i in range(10000): + tdSql.execute("insert into t values(%d, 0)" % (1000000 + i * 6000)) + + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select last(*) from t interval(1s)") + tdSql.checkRows(10000) + + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) From 92ca5ded0d222f4914876b745a6428d0dc276f75 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Thu, 3 Dec 2020 16:19:56 +0800 Subject: [PATCH 22/58] [TD-785]Improve the robustness of python connector --- src/connector/python/linux/python2/taos/__init__.py | 2 +- src/connector/python/linux/python2/taos/cinterface.py | 2 ++ src/connector/python/linux/python3/taos/__init__.py | 4 ++-- src/connector/python/linux/python3/taos/cinterface.py | 2 ++ src/connector/python/windows/python2/taos/__init__.py | 2 +- src/connector/python/windows/python2/taos/cinterface.py | 2 ++ src/connector/python/windows/python3/taos/__init__.py | 2 +- src/connector/python/windows/python3/taos/cinterface.py | 2 ++ 8 files changed, 13 insertions(+), 5 deletions(-) diff --git a/src/connector/python/linux/python2/taos/__init__.py b/src/connector/python/linux/python2/taos/__init__.py index 4894488bc8..5d21671eed 100644 --- a/src/connector/python/linux/python2/taos/__init__.py +++ b/src/connector/python/linux/python2/taos/__init__.py @@ -3,7 +3,7 @@ from .connection import TDengineConnection from .cursor import TDengineCursor # Globals -apilevel = '2.0' +apilevel = '2.0.2' threadsafety = 0 paramstyle = 'pyformat' diff --git a/src/connector/python/linux/python2/taos/cinterface.py b/src/connector/python/linux/python2/taos/cinterface.py index 6f0435722f..2b1b29ee31 100644 --- a/src/connector/python/linux/python2/taos/cinterface.py +++ b/src/connector/python/linux/python2/taos/cinterface.py @@ -81,6 +81,7 @@ def _crow_double_to_python(data, num_of_rows, nbytes=None, micro=False): def _crow_binary_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C binary row to python row """ + assert(nbytes is not None) if num_of_rows > 0: return [ None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode('utf-8') for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]] else: @@ -106,6 +107,7 @@ def _crow_nchar_to_python(data, num_of_rows, nbytes=None, micro=False): def _crow_binary_to_python_block(data, num_of_rows, nbytes=None, micro=False): """Function to convert C binary row to python row """ + assert(nbytes is not None) res=[] if num_of_rows > 0: for i in range(abs(num_of_rows)): diff --git a/src/connector/python/linux/python3/taos/__init__.py b/src/connector/python/linux/python3/taos/__init__.py index 4894488bc8..c46131eeaf 100644 --- a/src/connector/python/linux/python3/taos/__init__.py +++ b/src/connector/python/linux/python3/taos/__init__.py @@ -3,7 +3,7 @@ from .connection import TDengineConnection from .cursor import TDengineCursor # Globals -apilevel = '2.0' +apilevel = '2.0.2' threadsafety = 0 paramstyle = 'pyformat' @@ -21,4 +21,4 @@ def connect(*args, **kwargs): @rtype: TDengineConnector """ - return TDengineConnection(*args, **kwargs) \ No newline at end of file + return TDengineConnection(*args, **kwargs) diff --git a/src/connector/python/linux/python3/taos/cinterface.py b/src/connector/python/linux/python3/taos/cinterface.py index bbf5a0c714..fdebe349fe 100644 --- a/src/connector/python/linux/python3/taos/cinterface.py +++ b/src/connector/python/linux/python3/taos/cinterface.py @@ -81,6 +81,7 @@ def _crow_double_to_python(data, num_of_rows, nbytes=None, micro=False): def _crow_binary_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C binary row to python row """ + assert(nbytes is not None) if num_of_rows > 0: return [ None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode('utf-8') for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]] else: @@ -106,6 +107,7 @@ def _crow_nchar_to_python(data, num_of_rows, nbytes=None, micro=False): def _crow_binary_to_python_block(data, num_of_rows, nbytes=None, micro=False): """Function to convert C binary row to python row """ + assert(nbytes is not None) res=[] if num_of_rows > 0: for i in range(abs(num_of_rows)): diff --git a/src/connector/python/windows/python2/taos/__init__.py b/src/connector/python/windows/python2/taos/__init__.py index 4894488bc8..5d21671eed 100644 --- a/src/connector/python/windows/python2/taos/__init__.py +++ b/src/connector/python/windows/python2/taos/__init__.py @@ -3,7 +3,7 @@ from .connection import TDengineConnection from .cursor import TDengineCursor # Globals -apilevel = '2.0' +apilevel = '2.0.2' threadsafety = 0 paramstyle = 'pyformat' diff --git a/src/connector/python/windows/python2/taos/cinterface.py b/src/connector/python/windows/python2/taos/cinterface.py index d4bf1a0350..14f4f49be8 100644 --- a/src/connector/python/windows/python2/taos/cinterface.py +++ b/src/connector/python/windows/python2/taos/cinterface.py @@ -81,6 +81,7 @@ def _crow_double_to_python(data, num_of_rows, nbytes=None, micro=False): def _crow_binary_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C binary row to python row """ + assert(nbytes is not None) if num_of_rows > 0: return [ None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode('utf-8') for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]] else: @@ -106,6 +107,7 @@ def _crow_nchar_to_python(data, num_of_rows, nbytes=None, micro=False): def _crow_binary_to_python_block(data, num_of_rows, nbytes=None, micro=False): """Function to convert C binary row to python row """ + assert(nbytes is not None) res=[] if num_of_rows > 0: for i in range(abs(num_of_rows)): diff --git a/src/connector/python/windows/python3/taos/__init__.py b/src/connector/python/windows/python3/taos/__init__.py index 2fec471a1a..0708b9544f 100644 --- a/src/connector/python/windows/python3/taos/__init__.py +++ b/src/connector/python/windows/python3/taos/__init__.py @@ -3,7 +3,7 @@ from .connection import TDengineConnection from .cursor import TDengineCursor # Globals -apilevel = '2.0' +apilevel = '2.0.2' threadsafety = 0 paramstyle = 'pyformat' diff --git a/src/connector/python/windows/python3/taos/cinterface.py b/src/connector/python/windows/python3/taos/cinterface.py index fb1b65fc15..42b820ca80 100644 --- a/src/connector/python/windows/python3/taos/cinterface.py +++ b/src/connector/python/windows/python3/taos/cinterface.py @@ -81,6 +81,7 @@ def _crow_double_to_python(data, num_of_rows, nbytes=None, micro=False): def _crow_binary_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C binary row to python row """ + assert(nbytes is not None) if num_of_rows > 0: return [ None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode('utf-8') for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]] else: @@ -108,6 +109,7 @@ def _crow_nchar_to_python(data, num_of_rows, nbytes=None, micro=False): def _crow_binary_to_python_block(data, num_of_rows, nbytes=None, micro=False): """Function to convert C binary row to python row """ + assert(nbytes is not None) res=[] if num_of_rows > 0: for i in range(abs(num_of_rows)): From d79d3058ae9cf061e5217d8bc91d6d08f63307aa Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 3 Dec 2020 16:39:33 +0800 Subject: [PATCH 23/58] [TD-225] fix compiler error. --- src/query/src/qExecutor.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 6ce40b9204..0c07149e8e 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -4524,7 +4524,7 @@ static void doCopyQueryResultToMsg(SQInfo *pQInfo, int32_t numOfRows, char *data } } - int32_t numOfTables = taosHashGetSize(pQInfo->arrTableIdInfo); + int32_t numOfTables = (int32_t) taosHashGetSize(pQInfo->arrTableIdInfo); *(int32_t*)data = htonl(numOfTables); data += sizeof(int32_t); From fa810a383a1252bb13dc8b09eaa57f38f2dffbd1 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 3 Dec 2020 17:25:59 +0800 Subject: [PATCH 24/58] [TD-2265]: add the total number of rows in submit block during parse insert sql. --- src/client/src/tscParseInsert.c | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 6de8195d73..6ff1cdcb79 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -630,11 +630,17 @@ int32_t tscAllocateMemIfNeed(STableDataBlocks *pDataBlock, int32_t rowSize, int3 return TSDB_CODE_SUCCESS; } -static void tsSetBlockInfo(SSubmitBlk *pBlocks, const STableMeta *pTableMeta, int32_t numOfRows) { +static int32_t tsSetBlockInfo(SSubmitBlk *pBlocks, const STableMeta *pTableMeta, int32_t numOfRows) { pBlocks->tid = pTableMeta->id.tid; pBlocks->uid = pTableMeta->id.uid; pBlocks->sversion = pTableMeta->sversion; - pBlocks->numOfRows += numOfRows; + + if (pBlocks->numOfRows + numOfRows >= INT16_MAX) { + return TSDB_CODE_TSC_INVALID_SQL; + } else { + pBlocks->numOfRows += numOfRows; + return TSDB_CODE_SUCCESS; + } } // data block is disordered, sort it in ascending order @@ -722,7 +728,11 @@ static int32_t doParseInsertStatement(SSqlObj *pSql, void *pTableList, char **st } SSubmitBlk *pBlocks = (SSubmitBlk *)(dataBuf->pData); - tsSetBlockInfo(pBlocks, pTableMeta, numOfRows); + code = tsSetBlockInfo(pBlocks, pTableMeta, numOfRows); + if (code != TSDB_CODE_SUCCESS) { + tscInvalidSQLErrMsg(pCmd->payload, "too many rows in sql, total number of rows should be less than 32767", *str); + return code; + } dataBuf->vgId = pTableMeta->vgroupInfo.vgId; dataBuf->numOfTables = 1; From a3fb9a1a8e13c83b783cede3460650743f0e53e9 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 3 Dec 2020 17:28:02 +0800 Subject: [PATCH 25/58] [TD-2265] --- src/client/src/tscParseInsert.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 6ff1cdcb79..08935adf8b 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -1394,7 +1394,10 @@ static int doPackSendDataBlock(SSqlObj *pSql, int32_t numOfRows, STableDataBlock STableMeta *pTableMeta = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0)->pTableMeta; SSubmitBlk *pBlocks = (SSubmitBlk *)(pTableDataBlocks->pData); - tsSetBlockInfo(pBlocks, pTableMeta, numOfRows); + code = tsSetBlockInfo(pBlocks, pTableMeta, numOfRows); + if (code != TSDB_CODE_SUCCESS) { + return tscInvalidSQLErrMsg(pCmd->payload, "too many rows in sql, total number of rows should be less than 32767", ""); + } if ((code = tscMergeTableDataBlocks(pSql, pCmd->pDataBlocks)) != TSDB_CODE_SUCCESS) { return code; From 5015365172cfd9cc91344eeadc2a266e0df19be4 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 3 Dec 2020 17:28:38 +0800 Subject: [PATCH 26/58] [TD-2265] --- src/client/src/tscParseInsert.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 08935adf8b..18e5b6f074 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -1396,7 +1396,7 @@ static int doPackSendDataBlock(SSqlObj *pSql, int32_t numOfRows, STableDataBlock SSubmitBlk *pBlocks = (SSubmitBlk *)(pTableDataBlocks->pData); code = tsSetBlockInfo(pBlocks, pTableMeta, numOfRows); if (code != TSDB_CODE_SUCCESS) { - return tscInvalidSQLErrMsg(pCmd->payload, "too many rows in sql, total number of rows should be less than 32767", ""); + return tscInvalidSQLErrMsg(pCmd->payload, "too many rows in sql, total number of rows should be less than 32767", NULL); } if ((code = tscMergeTableDataBlocks(pSql, pCmd->pDataBlocks)) != TSDB_CODE_SUCCESS) { From 0fe731af0fccb4eb1cd31865d19b591a81a8f0cf Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 3 Dec 2020 22:01:55 +0800 Subject: [PATCH 27/58] [TD-2325]: enable configure if 50% cpu will be used in query processing. --- packaging/cfg/taos.cfg | 5 ++- src/common/inc/tglobal.h | 1 + src/common/src/tglobal.c | 15 +++++++- src/util/src/tconfig.c | 2 -- src/vnode/src/vnodeRead.c | 75 ++++++++++++++++++++------------------- 5 files changed, 57 insertions(+), 41 deletions(-) diff --git a/packaging/cfg/taos.cfg b/packaging/cfg/taos.cfg index 7662d49280..014ebc37d2 100644 --- a/packaging/cfg/taos.cfg +++ b/packaging/cfg/taos.cfg @@ -251,7 +251,7 @@ # cqDebugFlag 131 # enable/disable recording the SQL in taos client -# tscEnableRecordSql 0 +# enableRecordSql 0 # generate core file when service crash # enableCoreFile 1 @@ -264,3 +264,6 @@ # enable/disable stream (continuous query) # stream 1 + +# only 50% CPU resources will be used in query processing +# halfCoresForQuery 0 diff --git a/src/common/inc/tglobal.h b/src/common/inc/tglobal.h index 6e4274b358..10bcfcb82b 100644 --- a/src/common/inc/tglobal.h +++ b/src/common/inc/tglobal.h @@ -56,6 +56,7 @@ extern char tsTempDir[]; //query buffer management extern int32_t tsQueryBufferSize; // maximum allowed usage buffer for each data node during query processing +extern int32_t tsHalfCoresForQuery; // only 50% will be used in query processing // client extern int32_t tsTableMetaKeepTimer; diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index 18aa0ae6e3..026cd67a81 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -107,6 +107,9 @@ int64_t tsMaxRetentWindow = 24 * 3600L; // maximum time window tolerance // positive value (in MB) int32_t tsQueryBufferSize = -1; +// only 50% cpu will be used in query processing in dnode +int32_t tsHalfCoresForQuery = 0; + // db parameters int32_t tsCacheBlockSize = TSDB_DEFAULT_CACHE_BLOCK_SIZE; int32_t tsBlocksPerVnode = TSDB_DEFAULT_TOTAL_BLOCKS; @@ -884,6 +887,16 @@ static void doInitGlobalConfig(void) { cfg.unitType = TAOS_CFG_UTYPE_BYTE; taosInitConfigOption(cfg); + cfg.option = "halfCoresForQuery"; + cfg.ptr = &tsHalfCoresForQuery; + cfg.valType = TAOS_CFG_VTYPE_INT32; + cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; + cfg.minValue = 0; + cfg.maxValue = 1; + cfg.ptrLength = 1; + cfg.unitType = TAOS_CFG_UTYPE_NONE; + taosInitConfigOption(cfg); + // locale & charset cfg.option = "timezone"; cfg.ptr = tsTimezone; @@ -1290,7 +1303,7 @@ static void doInitGlobalConfig(void) { cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); - cfg.option = "tscEnableRecordSql"; + cfg.option = "enableRecordSql"; cfg.ptr = &tsTscEnableRecordSql; cfg.valType = TAOS_CFG_VTYPE_INT32; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; diff --git a/src/util/src/tconfig.c b/src/util/src/tconfig.c index e89dea5a24..173de294cf 100644 --- a/src/util/src/tconfig.c +++ b/src/util/src/tconfig.c @@ -19,9 +19,7 @@ #include "taoserror.h" #include "tconfig.h" #include "tglobal.h" -#include "tkey.h" #include "tulog.h" -#include "tsocket.h" #include "tsystem.h" #include "tutil.h" diff --git a/src/vnode/src/vnodeRead.c b/src/vnode/src/vnodeRead.c index 5ef79cfbf0..ed6d29505f 100644 --- a/src/vnode/src/vnodeRead.c +++ b/src/vnode/src/vnodeRead.c @@ -275,41 +275,40 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pRead) { vDebug("vgId:%d, QInfo:%p, dnode continues to exec query", pVnode->vgId, *qhandle); - -#if _NON_BLOCKING_RETRIEVE - bool freehandle = false; - bool buildRes = qTableQuery(*qhandle); // do execute query - - // build query rsp, the retrieve request has reached here already - if (buildRes) { - // update the connection info according to the retrieve connection - pRead->rpcHandle = qGetResultRetrieveMsg(*qhandle); - assert(pRead->rpcHandle != NULL); - - vDebug("vgId:%d, QInfo:%p, start to build retrieval rsp after query paused, %p", pVnode->vgId, *qhandle, - pRead->rpcHandle); - - // set the real rsp error code - pRead->code = vnodeDumpQueryResult(&pRead->rspRet, pVnode, qhandle, &freehandle, pRead->rpcHandle); - - // NOTE: set return code to be TSDB_CODE_QRY_HAS_RSP to notify dnode to return msg to client - code = TSDB_CODE_QRY_HAS_RSP; + // In the retrieve blocking model, only 50% CPU will be used in query processing + if (tsHalfCoresForQuery) { + qTableQuery(*qhandle); // do execute query + qReleaseQInfo(pVnode->qMgmt, (void **)&qhandle, false); } else { - void* h1 = qGetResultRetrieveMsg(*qhandle); - assert(h1 == NULL); + bool freehandle = false; + bool buildRes = qTableQuery(*qhandle); // do execute query - freehandle = qQueryCompleted(*qhandle); - } + // build query rsp, the retrieve request has reached here already + if (buildRes) { + // update the connection info according to the retrieve connection + pRead->rpcHandle = qGetResultRetrieveMsg(*qhandle); + assert(pRead->rpcHandle != NULL); - // NOTE: if the qhandle is not put into vread queue or query is completed, free the qhandle. - // If the building of result is not required, simply free it. Otherwise, mandatorily free the qhandle - if (freehandle || (!buildRes)) { - qReleaseQInfo(pVnode->qMgmt, (void **)&qhandle, freehandle); + vDebug("vgId:%d, QInfo:%p, start to build retrieval rsp after query paused, %p", pVnode->vgId, *qhandle, + pRead->rpcHandle); + + // set the real rsp error code + pRead->code = vnodeDumpQueryResult(&pRead->rspRet, pVnode, qhandle, &freehandle, pRead->rpcHandle); + + // NOTE: set return code to be TSDB_CODE_QRY_HAS_RSP to notify dnode to return msg to client + code = TSDB_CODE_QRY_HAS_RSP; + } else { + void *h1 = qGetResultRetrieveMsg(*qhandle); + assert(h1 == NULL); + freehandle = qQueryCompleted(*qhandle); + } + + // NOTE: if the qhandle is not put into vread queue or query is completed, free the qhandle. + // If the building of result is not required, simply free it. Otherwise, mandatorily free the qhandle + if (freehandle || (!buildRes)) { + qReleaseQInfo(pVnode->qMgmt, (void **)&qhandle, freehandle); + } } -#else - qTableQuery(*qhandle); // do execute query - qReleaseQInfo(pVnode->qMgmt, (void **)&qhandle, false); -#endif } return code; @@ -375,14 +374,16 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pRead) { freeHandle = true; } else { // result is not ready, return immediately assert(buildRes == true); -#if _NON_BLOCKING_RETRIEVE - if (!buildRes) { - assert(pRead->rpcHandle != NULL); - qReleaseQInfo(pVnode->qMgmt, (void **)&handle, false); - return TSDB_CODE_QRY_NOT_READY; + // Only effects in the non-blocking model + if (!tsHalfCoresForQuery) { + if (!buildRes) { + assert(pRead->rpcHandle != NULL); + + qReleaseQInfo(pVnode->qMgmt, (void **)&handle, false); + return TSDB_CODE_QRY_NOT_READY; + } } -#endif // ahandle is the sqlObj pointer code = vnodeDumpQueryResult(pRet, pVnode, handle, &freeHandle, pRead->rpcHandle); From 7090b3feb0bea56433e4d10565b18c74bf1f5e7b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 3 Dec 2020 22:52:17 +0800 Subject: [PATCH 28/58] [TD-2169]: fix memory leaks in group by query processing. --- src/query/src/qExecutor.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 0c07149e8e..bcce7c50a8 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -1372,8 +1372,12 @@ static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, char *pDat } if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { - pResultRow->key = malloc(varDataTLen(pData)); - varDataCopy(pResultRow->key, pData); + if (pResultRow->key == NULL) { + pResultRow->key = malloc(varDataTLen(pData)); + varDataCopy(pResultRow->key, pData); + } else { + assert(memcmp(pResultRow->key, pData, varDataTLen(pData)) == 0); + } } else { pResultRow->win.skey = v; pResultRow->win.ekey = v; From 1be5c3f88e75d07186c48ccbdfc9d7ecb8701d8d Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 3 Dec 2020 21:17:50 +0000 Subject: [PATCH 29/58] TD-2257 --- src/client/src/tscProfile.c | 2 ++ src/client/src/tscSystem.c | 2 +- src/util/src/tref.c | 19 ++++++++++--------- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/src/client/src/tscProfile.c b/src/client/src/tscProfile.c index acc5acd786..18fc79c474 100644 --- a/src/client/src/tscProfile.c +++ b/src/client/src/tscProfile.c @@ -39,6 +39,7 @@ void tscInitConnCb(void *param, TAOS_RES *result, int code) { tscSlowQueryConnInitialized = true; tscSaveSlowQueryFp(sql, NULL); } + taos_free_result(result); } void tscAddIntoSqlList(SSqlObj *pSql) { @@ -69,6 +70,7 @@ void tscSaveSlowQueryFpCb(void *param, TAOS_RES *result, int code) { } else { tscDebug("success to save slow query, code:%d", code); } + taos_free_result(result); } void tscSaveSlowQueryFp(void *handle, void *tmrId) { diff --git a/src/client/src/tscSystem.c b/src/client/src/tscSystem.c index 91eabda78b..6aea03d214 100644 --- a/src/client/src/tscSystem.c +++ b/src/client/src/tscSystem.c @@ -144,7 +144,7 @@ void taos_init_imp(void) { int64_t refreshTime = 10; // 10 seconds by default if (tscMetaCache == NULL) { tscMetaCache = taosCacheInit(TSDB_DATA_TYPE_BINARY, refreshTime, false, tscFreeTableMetaHelper, "tableMeta"); - tscObjRef = taosOpenRef(4096, tscFreeRegisteredSqlObj); + tscObjRef = taosOpenRef(40960, tscFreeRegisteredSqlObj); } tscRefId = taosOpenRef(200, tscCloseTscObj); diff --git a/src/util/src/tref.c b/src/util/src/tref.c index 4c1a87c960..1f83abcb84 100644 --- a/src/util/src/tref.c +++ b/src/util/src/tref.c @@ -329,7 +329,7 @@ void *taosIterateRef(int rsetId, int64_t rid) { pNode->count++; // acquire it newP = pNode->p; taosUnlockList(pSet->lockedBy+hash); - uTrace("rsetId:%d p:%p rid:%" PRId64 " is returned", rsetId, newP, rid); + uTrace("rsetId:%d p:%p rid:%" PRId64 " is returned", rsetId, newP, rid); } else { uTrace("rsetId:%d the list is over", rsetId); } @@ -423,24 +423,25 @@ static int taosDecRefCount(int rsetId, int64_t rid, int remove) { if (pNode->next) { pNode->next->prev = pNode->prev; } - - (*pSet->fp)(pNode->p); - - uTrace("rsetId:%d p:%p rid:%" PRId64 " is removed, count:%d, free mem: %p", rsetId, pNode->p, rid, pSet->count, pNode); - free(pNode); released = 1; } else { - uTrace("rsetId:%d p:%p rid:%" PRId64 " is released, count:%d", rsetId, pNode->p, rid, pNode->count); + uTrace("rsetId:%d p:%p rid:%" PRId64 " is released", rsetId, pNode->p, rid); } } else { - uTrace("rsetId:%d rid:%" PRId64 " is not there, failed to release/remove", rsetId, rid); + uTrace("rsetId:%d rid:%" PRId64 " is not there, failed to release/remove", rsetId, rid); terrno = TSDB_CODE_REF_NOT_EXIST; code = -1; } taosUnlockList(pSet->lockedBy+hash); - if (released) taosDecRsetCount(pSet); + if (released) { + uTrace("rsetId:%d p:%p rid:%" PRId64 " is removed, count:%d, free mem: %p", rsetId, pNode->p, rid, pSet->count, pNode); + (*pSet->fp)(pNode->p); + free(pNode); + + taosDecRsetCount(pSet); + } return code; } From 0a0080903baf427b20769a4247176607ddd3936c Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Fri, 4 Dec 2020 01:45:53 +0000 Subject: [PATCH 30/58] [TD-2313]: ensure length of nchar cols not exceed limit --- src/query/src/qParserImpl.c | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/src/query/src/qParserImpl.c b/src/query/src/qParserImpl.c index 5cd9d3c77a..7d71d9f7f1 100644 --- a/src/query/src/qParserImpl.c +++ b/src/query/src/qParserImpl.c @@ -405,14 +405,29 @@ void tSQLSetColumnType(TAOS_FIELD *pField, SStrToken *type) { if (type->type == 0) { pField->bytes = 0; } else { - pField->bytes = (int16_t)(-(int32_t)type->type * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE); + int32_t bytes = -(int32_t)(type->type); + if (bytes > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { + // we have to postpone reporting the error because it cannot be done here + // as pField->bytes is int16_t, use 'TSDB_MAX_NCHAR_LEN + 1' to avoid overflow + bytes = TSDB_MAX_NCHAR_LEN + 1; + } else { + bytes = bytes * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE; + } + pField->bytes = (int16_t)bytes; } } else if (i == TSDB_DATA_TYPE_BINARY) { /* for binary, the TOKENTYPE is the length of binary */ if (type->type == 0) { pField->bytes = 0; } else { - pField->bytes = (int16_t) (-(int32_t) type->type + VARSTR_HEADER_SIZE); + int32_t bytes = -(int32_t)(type->type); + if (bytes > TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) { + // refer comment for NCHAR above + bytes = TSDB_MAX_BINARY_LEN + 1; + } else { + bytes += VARSTR_HEADER_SIZE; + } + pField->bytes = (int16_t)bytes; } } break; From 573fcb4e9c5e0cbed4139506c36b5af50f41ec52 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 4 Dec 2020 10:07:13 +0800 Subject: [PATCH 31/58] TD-2299 --- src/dnode/src/dnodePeer.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/dnode/src/dnodePeer.c b/src/dnode/src/dnodePeer.c index 6bf22cee4e..6b5b28622b 100644 --- a/src/dnode/src/dnodePeer.c +++ b/src/dnode/src/dnodePeer.c @@ -182,6 +182,8 @@ void dnodeSendMsgToDnode(SRpcEpSet *epSet, SRpcMsg *rpcMsg) { void dnodeSendMsgToMnodeRecv(SRpcMsg *rpcMsg, SRpcMsg *rpcRsp) { SRpcEpSet epSet = {0}; dnodeGetEpSetForPeer(&epSet); + + assert(tsClientRpc != 0); rpcSendRecv(tsClientRpc, &epSet, rpcMsg, rpcRsp); } From 1993ad245fb7adc8f3af3671c8917f3d776355ea Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Fri, 4 Dec 2020 11:16:42 +0800 Subject: [PATCH 32/58] [TD-785]Improve the robustness of python connector --- src/connector/python/linux/python2/taos/cursor.py | 4 ++-- src/connector/python/linux/python3/taos/cursor.py | 4 ++-- src/connector/python/windows/python2/taos/cursor.py | 4 ++-- src/connector/python/windows/python3/taos/cursor.py | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/connector/python/linux/python2/taos/cursor.py b/src/connector/python/linux/python2/taos/cursor.py index eee5e39488..82a01be671 100644 --- a/src/connector/python/linux/python2/taos/cursor.py +++ b/src/connector/python/linux/python2/taos/cursor.py @@ -26,7 +26,7 @@ class TDengineCursor(object): """ def __init__(self, connection=None): - self._description = None + self._description = [] self._rowcount = -1 self._connection = None self._result = None @@ -234,7 +234,7 @@ class TDengineCursor(object): def _reset_result(self): """Reset the result to unused version. """ - self._description = None + self._description = [] self._rowcount = -1 if self._result is not None: CTaosInterface.freeResult(self._result) diff --git a/src/connector/python/linux/python3/taos/cursor.py b/src/connector/python/linux/python3/taos/cursor.py index d8184668c8..0ce20f0eda 100644 --- a/src/connector/python/linux/python3/taos/cursor.py +++ b/src/connector/python/linux/python3/taos/cursor.py @@ -27,7 +27,7 @@ class TDengineCursor(object): """ def __init__(self, connection=None): - self._description = None + self._description = [] self._rowcount = -1 self._connection = None self._result = None @@ -242,7 +242,7 @@ class TDengineCursor(object): def _reset_result(self): """Reset the result to unused version. """ - self._description = None + self._description = [] self._rowcount = -1 if self._result is not None: CTaosInterface.freeResult(self._result) diff --git a/src/connector/python/windows/python2/taos/cursor.py b/src/connector/python/windows/python2/taos/cursor.py index 71651afee1..f6fde2619b 100644 --- a/src/connector/python/windows/python2/taos/cursor.py +++ b/src/connector/python/windows/python2/taos/cursor.py @@ -27,7 +27,7 @@ class TDengineCursor(object): """ def __init__(self, connection=None): - self._description = None + self._description = [] self._rowcount = -1 self._connection = None self._result = None @@ -193,7 +193,7 @@ class TDengineCursor(object): def _reset_result(self): """Reset the result to unused version. """ - self._description = None + self._description = [] self._rowcount = -1 if self._result is not None: CTaosInterface.freeResult(self._result) diff --git a/src/connector/python/windows/python3/taos/cursor.py b/src/connector/python/windows/python3/taos/cursor.py index b813bba357..db66b99d3b 100644 --- a/src/connector/python/windows/python3/taos/cursor.py +++ b/src/connector/python/windows/python3/taos/cursor.py @@ -28,7 +28,7 @@ class TDengineCursor(object): """ def __init__(self, connection=None): - self._description = None + self._description = [] self._rowcount = -1 self._connection = None self._result = None @@ -194,7 +194,7 @@ class TDengineCursor(object): def _reset_result(self): """Reset the result to unused version. """ - self._description = None + self._description = [] self._rowcount = -1 if self._result is not None: CTaosInterface.freeResult(self._result) From e4dce56692fdec43a09267008eaeb4b7bbbb00c7 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Fri, 4 Dec 2020 11:30:21 +0800 Subject: [PATCH 33/58] [TD-785]Improve the robustness of python connector --- src/connector/python/linux/python2/setup.py | 2 +- src/connector/python/linux/python2/taos/__init__.py | 2 +- src/connector/python/linux/python3/setup.py | 2 +- src/connector/python/linux/python3/taos/__init__.py | 2 +- src/connector/python/windows/python2/setup.py | 2 +- src/connector/python/windows/python2/taos/__init__.py | 2 +- src/connector/python/windows/python3/setup.py | 2 +- src/connector/python/windows/python3/taos/__init__.py | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/connector/python/linux/python2/setup.py b/src/connector/python/linux/python2/setup.py index b3daa98bdc..92a931b504 100644 --- a/src/connector/python/linux/python2/setup.py +++ b/src/connector/python/linux/python2/setup.py @@ -5,7 +5,7 @@ with open("README.md", "r") as fh: setuptools.setup( name="taos", - version="2.0.2", + version="2.0.3", author="Taosdata Inc.", author_email="support@taosdata.com", description="TDengine python client package", diff --git a/src/connector/python/linux/python2/taos/__init__.py b/src/connector/python/linux/python2/taos/__init__.py index 5d21671eed..62e0536b6f 100644 --- a/src/connector/python/linux/python2/taos/__init__.py +++ b/src/connector/python/linux/python2/taos/__init__.py @@ -3,7 +3,7 @@ from .connection import TDengineConnection from .cursor import TDengineCursor # Globals -apilevel = '2.0.2' +apilevel = '2.0.3' threadsafety = 0 paramstyle = 'pyformat' diff --git a/src/connector/python/linux/python3/setup.py b/src/connector/python/linux/python3/setup.py index f49ebe2b6d..655a12ad13 100644 --- a/src/connector/python/linux/python3/setup.py +++ b/src/connector/python/linux/python3/setup.py @@ -5,7 +5,7 @@ with open("README.md", "r") as fh: setuptools.setup( name="taos", - version="2.0.2", + version="2.0.3", author="Taosdata Inc.", author_email="support@taosdata.com", description="TDengine python client package", diff --git a/src/connector/python/linux/python3/taos/__init__.py b/src/connector/python/linux/python3/taos/__init__.py index c46131eeaf..8cf095ea68 100644 --- a/src/connector/python/linux/python3/taos/__init__.py +++ b/src/connector/python/linux/python3/taos/__init__.py @@ -3,7 +3,7 @@ from .connection import TDengineConnection from .cursor import TDengineCursor # Globals -apilevel = '2.0.2' +apilevel = '2.0.3' threadsafety = 0 paramstyle = 'pyformat' diff --git a/src/connector/python/windows/python2/setup.py b/src/connector/python/windows/python2/setup.py index 34cb7a04d7..5ddbe83011 100644 --- a/src/connector/python/windows/python2/setup.py +++ b/src/connector/python/windows/python2/setup.py @@ -5,7 +5,7 @@ with open("README.md", "r") as fh: setuptools.setup( name="taos", - version="2.0.2", + version="2.0.3", author="Taosdata Inc.", author_email="support@taosdata.com", description="TDengine python client package", diff --git a/src/connector/python/windows/python2/taos/__init__.py b/src/connector/python/windows/python2/taos/__init__.py index 5d21671eed..62e0536b6f 100644 --- a/src/connector/python/windows/python2/taos/__init__.py +++ b/src/connector/python/windows/python2/taos/__init__.py @@ -3,7 +3,7 @@ from .connection import TDengineConnection from .cursor import TDengineCursor # Globals -apilevel = '2.0.2' +apilevel = '2.0.3' threadsafety = 0 paramstyle = 'pyformat' diff --git a/src/connector/python/windows/python3/setup.py b/src/connector/python/windows/python3/setup.py index c09644d330..ffed304c85 100644 --- a/src/connector/python/windows/python3/setup.py +++ b/src/connector/python/windows/python3/setup.py @@ -5,7 +5,7 @@ with open("README.md", "r") as fh: setuptools.setup( name="taos", - version="2.0.2", + version="2.0.3", author="Taosdata Inc.", author_email="support@taosdata.com", description="TDengine python client package", diff --git a/src/connector/python/windows/python3/taos/__init__.py b/src/connector/python/windows/python3/taos/__init__.py index 0708b9544f..c6dd929a6a 100644 --- a/src/connector/python/windows/python3/taos/__init__.py +++ b/src/connector/python/windows/python3/taos/__init__.py @@ -3,7 +3,7 @@ from .connection import TDengineConnection from .cursor import TDengineCursor # Globals -apilevel = '2.0.2' +apilevel = '2.0.3' threadsafety = 0 paramstyle = 'pyformat' From 1ae57bbf222c34768e09c9d0c879b5398128a19e Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 4 Dec 2020 03:36:25 +0000 Subject: [PATCH 34/58] fix TD-2322 --- src/tsdb/src/tsdbMemTable.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index 4c51608b5d..999e2deb41 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -786,9 +786,9 @@ static int tsdbCheckTableSchema(STsdbRepo *pRepo, SSubmitBlk *pBlock, STable *pT if (tsdbGetTableSchemaImpl(pTable, false, false, pBlock->sversion) == NULL) { tsdbError("vgId:%d invalid submit schema version %d to table %s tid %d from client", REPO_ID(pRepo), pBlock->sversion, TABLE_CHAR_NAME(pTable), TABLE_TID(pTable)); + terrno = TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION; + return -1; } - terrno = TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION; - return -1; } return 0; From b8f38e0568a51814febb2bb3f8ae70b5484a51c7 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 4 Dec 2020 12:47:37 +0800 Subject: [PATCH 35/58] [TD-2281]: filter unsupported mixed up query functions. --- src/client/src/tscFunctionImpl.c | 8 ++++---- src/client/src/tscSQLParser.c | 4 ++++ 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c index 56b7f052f7..594a4c39b5 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/client/src/tscFunctionImpl.c @@ -4540,11 +4540,11 @@ static void sumrate_finalizer(SQLFunctionCtx *pCtx) { * */ int32_t functionCompatList[] = { - // count, sum, avg, min, max, stddev, percentile, apercentile, first, last - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - // last_row, top, bottom, spread, twa, leastsqr, ts, ts_dummy, tag_dummy, ts_z + // count, sum, avg, min, max, stddev, percentile, apercentile, first, last + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + // last_row,top, bottom, spread, twa, leastsqr, ts, ts_dummy, tag_dummy, ts_z 4, -1, -1, 1, 1, 1, 1, 1, 1, -1, - // tag, colprj, tagprj, arithmetic, diff, first_dist, last_dist, interp rate irate + // tag, colprj, tagprj, arithmetic, diff, first_dist, last_dist, interp rate irate 1, 1, 1, 1, -1, 1, 1, 5, 1, 1, // sum_rate, sum_irate, avg_rate, avg_irate 1, 1, 1, 1, diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index abbe26204d..08b2a7db4e 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -2835,6 +2835,10 @@ static bool functionCompatibleCheck(SQueryInfo* pQueryInfo, bool joinQuery) { if (functionCompatList[functionId] != factor) { return false; + } else { + if (factor == -1) { // two functions with the same -1 flag + return false; + } } if (functionId == TSDB_FUNC_LAST_ROW && joinQuery) { From 94756ccd947e62aa26166d899d0ac4620f517f11 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 4 Dec 2020 13:07:23 +0800 Subject: [PATCH 36/58] TD-2331 --- src/dnode/inc/dnodeMInfos.h | 6 +- src/dnode/src/dnodeMInfos.c | 58 +++++++------- src/dnode/src/dnodeMgmt.c | 4 +- src/dnode/src/dnodeModule.c | 6 +- src/inc/dnode.h | 2 +- src/inc/taosmsg.h | 14 ++-- src/mnode/inc/mnodeMnode.h | 2 +- src/mnode/inc/mnodeSdb.h | 1 + src/mnode/src/mnodeMnode.c | 146 +++++++++++++++++++++--------------- src/mnode/src/mnodeSdb.c | 25 ++++-- src/sync/src/syncMain.c | 2 +- 11 files changed, 151 insertions(+), 115 deletions(-) diff --git a/src/dnode/inc/dnodeMInfos.h b/src/dnode/inc/dnodeMInfos.h index 9c3c85c47e..2c3eef5d5d 100644 --- a/src/dnode/inc/dnodeMInfos.h +++ b/src/dnode/inc/dnodeMInfos.h @@ -24,9 +24,9 @@ extern "C" { int32_t dnodeInitMInfos(); void dnodeCleanupMInfos(); -void dnodeUpdateMInfos(SMnodeInfos *minfos); -void dnodeUpdateEpSetForPeer(SRpcEpSet *epSet); -void dnodeGetMInfos(SMnodeInfos *minfos); +void dnodeUpdateMInfos(SMInfos *pMinfos); +void dnodeUpdateEpSetForPeer(SRpcEpSet *pEpSet); +void dnodeGetMInfos(SMInfos *pMinfos); bool dnodeIsMasterEp(char *ep); #ifdef __cplusplus diff --git a/src/dnode/src/dnodeMInfos.c b/src/dnode/src/dnodeMInfos.c index cefe44aebe..162de2243e 100644 --- a/src/dnode/src/dnodeMInfos.c +++ b/src/dnode/src/dnodeMInfos.c @@ -22,12 +22,12 @@ #include "dnodeInt.h" #include "dnodeMInfos.h" -static SMnodeInfos tsMInfos; -static SRpcEpSet tsMEpSet; +static SMInfos tsMInfos; +static SRpcEpSet tsMEpSet; static pthread_mutex_t tsMInfosMutex; -static void dnodeResetMInfos(SMnodeInfos *minfos); -static void dnodePrintMInfos(SMnodeInfos *minfos); +static void dnodeResetMInfos(SMInfos *minfos); +static void dnodePrintMInfos(SMInfos *minfos); static int32_t dnodeReadMInfos(); static int32_t dnodeWriteMInfos(); @@ -44,14 +44,14 @@ int32_t dnodeInitMInfos() { void dnodeCleanupMInfos() { pthread_mutex_destroy(&tsMInfosMutex); } -void dnodeUpdateMInfos(SMnodeInfos *minfos) { - if (minfos->mnodeNum <= 0 || minfos->mnodeNum > 3) { - dError("invalid mnode infos, mnodeNum:%d", minfos->mnodeNum); +void dnodeUpdateMInfos(SMInfos *pMinfos) { + if (pMinfos->mnodeNum <= 0 || pMinfos->mnodeNum > 3) { + dError("invalid mnode infos, mnodeNum:%d", pMinfos->mnodeNum); return; } - for (int32_t i = 0; i < minfos->mnodeNum; ++i) { - SMnodeInfo *minfo = &minfos->mnodeInfos[i]; + for (int32_t i = 0; i < pMinfos->mnodeNum; ++i) { + SMInfo *minfo = &pMinfos->mnodeInfos[i]; minfo->mnodeId = htonl(minfo->mnodeId); if (minfo->mnodeId <= 0 || strlen(minfo->mnodeEp) <= 5) { dError("invalid mnode info:%d, mnodeId:%d mnodeEp:%s", i, minfo->mnodeId, minfo->mnodeEp); @@ -60,14 +60,14 @@ void dnodeUpdateMInfos(SMnodeInfos *minfos) { } pthread_mutex_lock(&tsMInfosMutex); - if (minfos->mnodeNum != tsMInfos.mnodeNum) { - dnodeResetMInfos(minfos); + if (pMinfos->mnodeNum != tsMInfos.mnodeNum) { + dnodeResetMInfos(pMinfos); dnodeWriteMInfos(); sdbUpdateAsync(); } else { - int32_t size = sizeof(SMnodeInfos); - if (memcmp(minfos, &tsMInfos, size) != 0) { - dnodeResetMInfos(minfos); + int32_t size = sizeof(SMInfos); + if (memcmp(pMinfos, &tsMInfos, size) != 0) { + dnodeResetMInfos(pMinfos); dnodeWriteMInfos(); sdbUpdateAsync(); } @@ -99,11 +99,11 @@ bool dnodeIsMasterEp(char *ep) { return isMaster; } -void dnodeGetMInfos(SMnodeInfos *minfos) { +void dnodeGetMInfos(SMInfos *pMinfos) { pthread_mutex_lock(&tsMInfosMutex); - memcpy(minfos, &tsMInfos, sizeof(SMnodeInfos)); + memcpy(pMinfos, &tsMInfos, sizeof(SMInfos)); for (int32_t i = 0; i < tsMInfos.mnodeNum; ++i) { - minfos->mnodeInfos[i].mnodeId = htonl(tsMInfos.mnodeInfos[i].mnodeId); + pMinfos->mnodeInfos[i].mnodeId = htonl(tsMInfos.mnodeInfos[i].mnodeId); } pthread_mutex_unlock(&tsMInfosMutex); } @@ -123,15 +123,15 @@ void dnodeGetEpSetForShell(SRpcEpSet *epSet) { pthread_mutex_unlock(&tsMInfosMutex); } -static void dnodePrintMInfos(SMnodeInfos *minfos) { - dInfo("print mnode infos, mnodeNum:%d inUse:%d", minfos->mnodeNum, minfos->inUse); - for (int32_t i = 0; i < minfos->mnodeNum; i++) { - dInfo("mnode index:%d, %s", minfos->mnodeInfos[i].mnodeId, minfos->mnodeInfos[i].mnodeEp); +static void dnodePrintMInfos(SMInfos *pMinfos) { + dInfo("print minfos, mnodeNum:%d inUse:%d", pMinfos->mnodeNum, pMinfos->inUse); + for (int32_t i = 0; i < pMinfos->mnodeNum; i++) { + dInfo("mnode index:%d, %s", pMinfos->mnodeInfos[i].mnodeId, pMinfos->mnodeInfos[i].mnodeEp); } } -static void dnodeResetMInfos(SMnodeInfos *minfos) { - if (minfos == NULL) { +static void dnodeResetMInfos(SMInfos *pMinfos) { + if (pMinfos == NULL) { tsMEpSet.numOfEps = 1; taosGetFqdnPortFromEp(tsFirst, tsMEpSet.fqdn[0], &tsMEpSet.port[0]); @@ -142,10 +142,10 @@ static void dnodeResetMInfos(SMnodeInfos *minfos) { return; } - if (minfos->mnodeNum == 0) return; + if (pMinfos->mnodeNum == 0) return; - int32_t size = sizeof(SMnodeInfos); - memcpy(&tsMInfos, minfos, size); + int32_t size = sizeof(SMInfos); + memcpy(&tsMInfos, pMinfos, size); tsMEpSet.inUse = tsMInfos.inUse; tsMEpSet.numOfEps = tsMInfos.mnodeNum; @@ -153,7 +153,7 @@ static void dnodeResetMInfos(SMnodeInfos *minfos) { taosGetFqdnPortFromEp(tsMInfos.mnodeInfos[i].mnodeEp, tsMEpSet.fqdn[i], &tsMEpSet.port[i]); } - dnodePrintMInfos(minfos); + dnodePrintMInfos(pMinfos); } static int32_t dnodeReadMInfos() { @@ -162,7 +162,7 @@ static int32_t dnodeReadMInfos() { char * content = calloc(1, maxLen + 1); cJSON * root = NULL; FILE * fp = NULL; - SMnodeInfos minfos = {0}; + SMInfos minfos = {0}; char file[TSDB_FILENAME_LEN + 20] = {0}; sprintf(file, "%s/mnodeEpSet.json", tsDnodeDir); @@ -241,7 +241,7 @@ PARSE_MINFOS_OVER: terrno = 0; for (int32_t i = 0; i < minfos.mnodeNum; ++i) { - SMnodeInfo *mInfo = &minfos.mnodeInfos[i]; + SMInfo *mInfo = &minfos.mnodeInfos[i]; dnodeUpdateEp(mInfo->mnodeId, mInfo->mnodeEp, NULL, NULL); } dnodeResetMInfos(&minfos); diff --git a/src/dnode/src/dnodeMgmt.c b/src/dnode/src/dnodeMgmt.c index 5c01f64716..15378c77c1 100644 --- a/src/dnode/src/dnodeMgmt.c +++ b/src/dnode/src/dnodeMgmt.c @@ -472,8 +472,8 @@ static void dnodeProcessStatusRsp(SRpcMsg *pMsg) { } SStatusRsp *pStatusRsp = pMsg->pCont; - SMnodeInfos *minfos = &pStatusRsp->mnodes; - dnodeUpdateMInfos(minfos); + SMInfos *pMinfos = &pStatusRsp->mnodes; + dnodeUpdateMInfos(pMinfos); SDnodeCfg *pCfg = &pStatusRsp->dnodeCfg; pCfg->numOfVnodes = htonl(pCfg->numOfVnodes); diff --git a/src/dnode/src/dnodeModule.c b/src/dnode/src/dnodeModule.c index 7faa3c8913..f664618f51 100644 --- a/src/dnode/src/dnodeModule.c +++ b/src/dnode/src/dnodeModule.c @@ -147,8 +147,8 @@ void dnodeProcessModuleStatus(uint32_t moduleStatus) { } } -bool dnodeStartMnode(SMnodeInfos *minfos) { - SMnodeInfos *mnodes = minfos; +bool dnodeStartMnode(SMInfos *pMinfos) { + SMInfos *pMnodes = pMinfos; if (tsModuleStatus & (1 << TSDB_MOD_MNODE)) { dDebug("mnode module is already started, module status:%d", tsModuleStatus); @@ -159,7 +159,7 @@ bool dnodeStartMnode(SMnodeInfos *minfos) { dInfo("start mnode module, module status:%d, new status:%d", tsModuleStatus, moduleStatus); dnodeProcessModuleStatus(moduleStatus); - sdbUpdateSync(mnodes); + sdbUpdateSync(pMnodes); return true; } diff --git a/src/inc/dnode.h b/src/inc/dnode.h index eef4490800..1efaa4a24b 100644 --- a/src/inc/dnode.h +++ b/src/inc/dnode.h @@ -45,7 +45,7 @@ void dnodeGetEpSetForShell(SRpcEpSet *epSet); int32_t dnodeGetDnodeId(); void dnodeUpdateEp(int32_t dnodeId, char *ep, char *fqdn, uint16_t *port); bool dnodeCheckEpChanged(int32_t dnodeId, char *epstr); -bool dnodeStartMnode(SMnodeInfos *minfos); +bool dnodeStartMnode(SMInfos *pMinfos); void dnodeAddClientRspHandle(uint8_t msgType, void (*fp)(SRpcMsg *rpcMsg)); void dnodeSendMsgToDnode(SRpcEpSet *epSet, SRpcMsg *rpcMsg); diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index b4d3bec958..e8e3029244 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -591,13 +591,13 @@ typedef struct { typedef struct { int32_t mnodeId; char mnodeEp[TSDB_EP_LEN]; -} SMnodeInfo; +} SMInfo; typedef struct { - int8_t inUse; - int8_t mnodeNum; - SMnodeInfo mnodeInfos[TSDB_MAX_REPLICA]; -} SMnodeInfos; + int8_t inUse; + int8_t mnodeNum; + SMInfo mnodeInfos[TSDB_MAX_REPLICA]; +} SMInfos; typedef struct { int32_t numOfMnodes; // tsNumOfMnodes @@ -632,7 +632,7 @@ typedef struct { } SStatusMsg; typedef struct { - SMnodeInfos mnodes; + SMInfos mnodes; SDnodeCfg dnodeCfg; SVgroupAccess vgAccess[]; } SStatusRsp; @@ -761,7 +761,7 @@ typedef struct { typedef struct { int32_t dnodeId; char dnodeEp[TSDB_EP_LEN]; // end point, hostname:port - SMnodeInfos mnodes; + SMInfos mnodes; } SCreateMnodeMsg; typedef struct { diff --git a/src/mnode/inc/mnodeMnode.h b/src/mnode/inc/mnodeMnode.h index 10cbcebe22..93f2fa11ea 100644 --- a/src/mnode/inc/mnodeMnode.h +++ b/src/mnode/inc/mnodeMnode.h @@ -48,7 +48,7 @@ void mnodeGetMnodeEpSetForShell(SRpcEpSet *epSet); char* mnodeGetMnodeMasterEp(); void mnodeGetMnodeInfos(void *mnodes); -void mnodeUpdateMnodeEpSet(); +void mnodeUpdateMnodeEpSet(SMInfos *pMnodes); #ifdef __cplusplus } diff --git a/src/mnode/inc/mnodeSdb.h b/src/mnode/inc/mnodeSdb.h index 90c4eac40a..31ea2da640 100644 --- a/src/mnode/inc/mnodeSdb.h +++ b/src/mnode/inc/mnodeSdb.h @@ -89,6 +89,7 @@ void* sdbGetTableByRid(int64_t rid); bool sdbIsMaster(); bool sdbIsServing(); void sdbUpdateMnodeRoles(); +int32_t sdbGetReplicaNum(); int32_t sdbInsertRow(SSdbRow *pRow); int32_t sdbDeleteRow(SSdbRow *pRow); diff --git a/src/mnode/src/mnodeMnode.c b/src/mnode/src/mnodeMnode.c index d20d51f82b..ba0b5a1865 100644 --- a/src/mnode/src/mnodeMnode.c +++ b/src/mnode/src/mnodeMnode.c @@ -34,14 +34,14 @@ #include "mnodeUser.h" #include "mnodeVgroup.h" -int64_t tsMnodeRid = -1; -static void * tsMnodeSdb = NULL; -static int32_t tsMnodeUpdateSize = 0; -static SRpcEpSet tsMnodeEpSetForShell; -static SRpcEpSet tsMnodeEpSetForPeer; -static SMnodeInfos tsMnodeInfos; -static int32_t mnodeGetMnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); -static int32_t mnodeRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn); +int64_t tsMnodeRid = -1; +static void * tsMnodeSdb = NULL; +static int32_t tsMnodeUpdateSize = 0; +static SRpcEpSet tsMEpForShell; +static SRpcEpSet tsMEpForPeer; +static SMInfos tsMInfos; +static int32_t mnodeGetMnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); +static int32_t mnodeRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn); #if defined(LINUX) static pthread_rwlock_t tsMnodeLock; @@ -127,7 +127,7 @@ static int32_t mnodeMnodeActionRestored() { mnodeCancelGetNextMnode(pIter); } - mnodeUpdateMnodeEpSet(); + mnodeUpdateMnodeEpSet(NULL); return TSDB_CODE_SUCCESS; } @@ -199,93 +199,119 @@ void mnodeCancelGetNextMnode(void *pIter) { sdbFreeIter(tsMnodeSdb, pIter); } -void mnodeUpdateMnodeEpSet() { - mInfo("update mnodes epSet, numOfEps:%d ", mnodeGetMnodesNum()); +void mnodeUpdateMnodeEpSet(SMInfos *pMinfos) { + bool set = false; + SMInfos mInfos = {0}; + mInfo("vgId:1, update mnodes epSet, numOfEps:%d pMinfos:%p", mnodeGetMnodesNum(), pMinfos); + + if (pMinfos != NULL) { + set = true; + mInfos = *pMinfos; + } + else { + int32_t index = 0; + void * pIter = NULL; + while (1) { + SMnodeObj *pMnode = NULL; + pIter = mnodeGetNextMnode(pIter, &pMnode); + if (pMnode == NULL) break; + + SDnodeObj *pDnode = mnodeGetDnode(pMnode->mnodeId); + if (pDnode != NULL) { + set = true; + mInfos.mnodeInfos[index].mnodeId = pMnode->mnodeId; + strcpy(mInfos.mnodeInfos[index].mnodeEp, pDnode->dnodeEp); + if (pMnode->role == TAOS_SYNC_ROLE_MASTER) mInfos.inUse = index; + index++; + } else { + set = false; + } + + mnodeDecDnodeRef(pDnode); + mnodeDecMnodeRef(pMnode); + } + + mInfos.mnodeNum = index; + if (sdbGetReplicaNum() != mInfos.mnodeNum) { + set = false; + mDebug("vgId:1, mnodes info not synced, cfg:%d current:%d", sdbGetReplicaNum(), mInfos.mnodeNum); + } + } mnodeMnodeWrLock(); - memset(&tsMnodeEpSetForShell, 0, sizeof(SRpcEpSet)); - memset(&tsMnodeEpSetForPeer, 0, sizeof(SRpcEpSet)); - memset(&tsMnodeInfos, 0, sizeof(SMnodeInfos)); + if (set) { + memset(&tsMEpForShell, 0, sizeof(SRpcEpSet)); + memset(&tsMEpForPeer, 0, sizeof(SRpcEpSet)); + memcpy(&tsMInfos, &mInfos, sizeof(SMInfos)); + tsMEpForShell.inUse = tsMInfos.inUse; + tsMEpForPeer.inUse = tsMInfos.inUse; + tsMEpForShell.numOfEps = tsMInfos.mnodeNum; + tsMEpForPeer.numOfEps = tsMInfos.mnodeNum; - int32_t index = 0; - void * pIter = NULL; - while (1) { - SMnodeObj *pMnode = NULL; - pIter = mnodeGetNextMnode(pIter, &pMnode); - if (pMnode == NULL) break; + mInfo("vgId:1, mnodes epSet is set, num:%d inUse:%d", tsMInfos.mnodeNum, tsMInfos.inUse); + for (int index = 0; index < mInfos.mnodeNum; ++index) { + SMInfo *pInfo = &tsMInfos.mnodeInfos[index]; + taosGetFqdnPortFromEp(pInfo->mnodeEp, tsMEpForShell.fqdn[index], &tsMEpForShell.port[index]); + taosGetFqdnPortFromEp(pInfo->mnodeEp, tsMEpForPeer.fqdn[index], &tsMEpForPeer.port[index]); + tsMEpForPeer.port[index] = tsMEpForPeer.port[index] + TSDB_PORT_DNODEDNODE; - SDnodeObj *pDnode = mnodeGetDnode(pMnode->mnodeId); - if (pDnode != NULL) { - strcpy(tsMnodeEpSetForShell.fqdn[index], pDnode->dnodeFqdn); - tsMnodeEpSetForShell.port[index] = htons(pDnode->dnodePort); - mDebug("mnode:%d, for shell fqdn:%s %d", pDnode->dnodeId, tsMnodeEpSetForShell.fqdn[index], htons(tsMnodeEpSetForShell.port[index])); + mInfo("vgId:1, mnode:%d, fqdn:%s shell:%u peer:%u", pInfo->mnodeId, tsMEpForShell.fqdn[index], + tsMEpForShell.port[index], tsMEpForPeer.port[index]); - strcpy(tsMnodeEpSetForPeer.fqdn[index], pDnode->dnodeFqdn); - tsMnodeEpSetForPeer.port[index] = htons(pDnode->dnodePort + TSDB_PORT_DNODEDNODE); - mDebug("mnode:%d, for peer fqdn:%s %d", pDnode->dnodeId, tsMnodeEpSetForPeer.fqdn[index], htons(tsMnodeEpSetForPeer.port[index])); - - tsMnodeInfos.mnodeInfos[index].mnodeId = htonl(pMnode->mnodeId); - strcpy(tsMnodeInfos.mnodeInfos[index].mnodeEp, pDnode->dnodeEp); - - if (pMnode->role == TAOS_SYNC_ROLE_MASTER) { - tsMnodeEpSetForShell.inUse = index; - tsMnodeEpSetForPeer.inUse = index; - tsMnodeInfos.inUse = index; - } - - mInfo("mnode:%d, ep:%s %s", pDnode->dnodeId, pDnode->dnodeEp, pMnode->role == TAOS_SYNC_ROLE_MASTER ? "master" : ""); - index++; + tsMEpForShell.port[index] = htons(tsMEpForShell.port[index]); + tsMEpForPeer.port[index] = htons(tsMEpForPeer.port[index]); + pInfo->mnodeId = htonl(pInfo->mnodeId); + } + } else { + mInfo("vgId:1, mnodes epSet not set, num:%d inUse:%d", tsMInfos.mnodeNum, tsMInfos.inUse); + for (int index = 0; index < tsMInfos.mnodeNum; ++index) { + mInfo("vgId:1, index:%d, ep:%s:%u", index, tsMEpForShell.fqdn[index], htons(tsMEpForShell.port[index])); } - - mnodeDecDnodeRef(pDnode); - mnodeDecMnodeRef(pMnode); } - tsMnodeInfos.mnodeNum = index; - tsMnodeEpSetForShell.numOfEps = index; - tsMnodeEpSetForPeer.numOfEps = index; - mnodeMnodeUnLock(); } void mnodeGetMnodeEpSetForPeer(SRpcEpSet *epSet) { mnodeMnodeRdLock(); - *epSet = tsMnodeEpSetForPeer; + *epSet = tsMEpForPeer; mnodeMnodeUnLock(); + mTrace("vgId:1, mnodes epSet for peer is returned, num:%d inUse:%d", tsMEpForPeer.numOfEps, tsMEpForPeer.inUse); for (int32_t i = 0; i < epSet->numOfEps; ++i) { if (strcmp(epSet->fqdn[i], tsLocalFqdn) == 0 && htons(epSet->port[i]) == tsServerPort + TSDB_PORT_DNODEDNODE) { epSet->inUse = (i + 1) % epSet->numOfEps; - mTrace("mnode:%d, for peer ep:%s:%u, set inUse to %d", i, epSet->fqdn[i], htons(epSet->port[i]), epSet->inUse); + mTrace("vgId:1, mnode:%d, for peer ep:%s:%u, set inUse to %d", i, epSet->fqdn[i], htons(epSet->port[i]), epSet->inUse); } else { - mTrace("mpeer:%d, for peer ep:%s:%u", i, epSet->fqdn[i], htons(epSet->port[i])); + mTrace("vgId:1, mpeer:%d, for peer ep:%s:%u", i, epSet->fqdn[i], htons(epSet->port[i])); } } } void mnodeGetMnodeEpSetForShell(SRpcEpSet *epSet) { mnodeMnodeRdLock(); - *epSet = tsMnodeEpSetForShell; + *epSet = tsMEpForShell; mnodeMnodeUnLock(); + mTrace("vgId:1, mnodes epSet for shell is returned, num:%d inUse:%d", tsMEpForShell.numOfEps, tsMEpForShell.inUse); for (int32_t i = 0; i < epSet->numOfEps; ++i) { if (strcmp(epSet->fqdn[i], tsLocalFqdn) == 0 && htons(epSet->port[i]) == tsServerPort) { epSet->inUse = (i + 1) % epSet->numOfEps; - mTrace("mnode:%d, for shell ep:%s:%u, set inUse to %d", i, epSet->fqdn[i], htons(epSet->port[i]), epSet->inUse); + mTrace("vgId:1, mnode:%d, for shell ep:%s:%u, set inUse to %d", i, epSet->fqdn[i], htons(epSet->port[i]), epSet->inUse); } else { - mTrace("mnode:%d, for shell ep:%s:%u", i, epSet->fqdn[i], htons(epSet->port[i])); + mTrace("vgId:1, mnode:%d, for shell ep:%s:%u", i, epSet->fqdn[i], htons(epSet->port[i])); } } } char* mnodeGetMnodeMasterEp() { - return tsMnodeInfos.mnodeInfos[tsMnodeInfos.inUse].mnodeEp; + return tsMInfos.mnodeInfos[tsMInfos.inUse].mnodeEp; } -void mnodeGetMnodeInfos(void *mnodeInfos) { +void mnodeGetMnodeInfos(void *pMinfos) { mnodeMnodeRdLock(); - *(SMnodeInfos *)mnodeInfos = tsMnodeInfos; + *(SMInfos *)pMinfos = tsMInfos; mnodeMnodeUnLock(); } @@ -298,7 +324,7 @@ static int32_t mnodeSendCreateMnodeMsg(int32_t dnodeId, char *dnodeEp) { } else { pCreate->dnodeId = htonl(dnodeId); tstrncpy(pCreate->dnodeEp, dnodeEp, sizeof(pCreate->dnodeEp)); - pCreate->mnodes = tsMnodeInfos; + pCreate->mnodes = tsMInfos; bool found = false; for (int i = 0; i < pCreate->mnodes.mnodeNum; ++i) { if (pCreate->mnodes.mnodeInfos[i].mnodeId == htonl(dnodeId)) { @@ -336,7 +362,7 @@ static int32_t mnodeCreateMnodeCb(SMnodeMsg *pMsg, int32_t code) { mError("failed to create mnode, reason:%s", tstrerror(code)); } else { mDebug("mnode is created successfully"); - mnodeUpdateMnodeEpSet(); + mnodeUpdateMnodeEpSet(NULL); sdbUpdateAsync(); } @@ -380,7 +406,7 @@ void mnodeDropMnodeLocal(int32_t dnodeId) { mnodeDecMnodeRef(pMnode); } - mnodeUpdateMnodeEpSet(); + mnodeUpdateMnodeEpSet(NULL); sdbUpdateAsync(); } @@ -400,7 +426,7 @@ int32_t mnodeDropMnode(int32_t dnodeId) { sdbDecRef(tsMnodeSdb, pMnode); - mnodeUpdateMnodeEpSet(); + mnodeUpdateMnodeEpSet(NULL); sdbUpdateAsync(); return code; diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index 2ef758baf1..a8cd595fc8 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -224,11 +224,13 @@ void sdbUpdateMnodeRoles() { sdbInfo("vgId:1, mnode:%d, role:%s", pMnode->mnodeId, syncRole[pMnode->role]); if (pMnode->mnodeId == dnodeGetDnodeId()) tsSdbMgmt.role = pMnode->role; mnodeDecMnodeRef(pMnode); + } else { + sdbDebug("vgId:1, mnode:%d not found", roles.nodeId[i]); } } mnodeUpdateClusterId(); - mnodeUpdateMnodeEpSet(); + mnodeUpdateMnodeEpSet(NULL); } static uint32_t sdbGetFileInfo(int32_t vgId, char *name, uint32_t *index, uint32_t eindex, int64_t *size, uint64_t *fversion) { @@ -308,18 +310,20 @@ void sdbUpdateAsync() { } void sdbUpdateSync(void *pMnodes) { - SMnodeInfos *mnodes = pMnodes; + SMInfos *pMinfos = pMnodes; if (!mnodeIsRunning()) { mDebug("vgId:1, mnode not start yet, update sync config later"); return; } - mDebug("vgId:1, update sync config in sync module, mnodes:%p", pMnodes); + mDebug("vgId:1, update sync config, pMnodes:%p", pMnodes); SSyncCfg syncCfg = {0}; int32_t index = 0; - if (mnodes == NULL) { + if (pMinfos == NULL) { + mDebug("vgId:1, mInfos not input, use mInfos in sdb, numOfMnodes:%d", syncCfg.replica); + void *pIter = NULL; while (1) { SMnodeObj *pMnode = NULL; @@ -339,16 +343,17 @@ void sdbUpdateSync(void *pMnodes) { mnodeDecMnodeRef(pMnode); } syncCfg.replica = index; - mDebug("vgId:1, mnodes info not input, use infos in sdb, numOfMnodes:%d", syncCfg.replica); } else { - for (index = 0; index < mnodes->mnodeNum; ++index) { - SMnodeInfo *node = &mnodes->mnodeInfos[index]; + mDebug("vgId:1, mInfos input, numOfMnodes:%d", syncCfg.replica); + + for (index = 0; index < pMinfos->mnodeNum; ++index) { + SMInfo *node = &pMinfos->mnodeInfos[index]; syncCfg.nodeInfo[index].nodeId = node->mnodeId; taosGetFqdnPortFromEp(node->mnodeEp, syncCfg.nodeInfo[index].nodeFqdn, &syncCfg.nodeInfo[index].nodePort); syncCfg.nodeInfo[index].nodePort += TSDB_PORT_SYNC; } syncCfg.replica = index; - mDebug("vgId:1, mnodes info input, numOfMnodes:%d", syncCfg.replica); + mnodeUpdateMnodeEpSet(pMnodes); } syncCfg.quorum = (syncCfg.replica == 1) ? 1 : 2; @@ -1103,3 +1108,7 @@ static void *sdbWorkerFp(void *pWorker) { return NULL; } + +int32_t sdbGetReplicaNum() { + return tsSdbMgmt.cfg.replica; +} \ No newline at end of file diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c index adac532f2d..7e9e8cc2d0 100644 --- a/src/sync/src/syncMain.c +++ b/src/sync/src/syncMain.c @@ -548,7 +548,7 @@ static SSyncPeer *syncAddPeer(SSyncNode *pNode, const SNodeInfo *pInfo) { pPeer->pSyncNode = pNode; pPeer->refCount = 1; - sInfo("%s, it is configured", pPeer->id); + sInfo("%s, it is configured, ep:%s:%u", pPeer->id, pPeer->fqdn, pPeer->port); int32_t ret = strcmp(pPeer->fqdn, tsNodeFqdn); if (pPeer->nodeId == 0 || (ret > 0) || (ret == 0 && pPeer->port > tsSyncPort)) { int32_t checkMs = 100 + (pNode->vgId * 10) % 100; From 030fc359587bd64ff99adae36ee5d49f1729c528 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Fri, 4 Dec 2020 13:48:14 +0800 Subject: [PATCH 37/58] change version number --- cmake/version.inc | 2 +- snap/snapcraft.yaml | 4 ++-- src/connector/go | 2 +- src/connector/grafanaplugin | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cmake/version.inc b/cmake/version.inc index 2f0ec81aea..948c7d2d0b 100644 --- a/cmake/version.inc +++ b/cmake/version.inc @@ -4,7 +4,7 @@ PROJECT(TDengine) IF (DEFINED VERNUMBER) SET(TD_VER_NUMBER ${VERNUMBER}) ELSE () - SET(TD_VER_NUMBER "2.0.8.0") + SET(TD_VER_NUMBER "2.0.9.0") ENDIF () IF (DEFINED VERCOMPATIBLE) diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 059c0650c2..b5d06a4adb 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -1,6 +1,6 @@ name: tdengine base: core18 -version: '2.0.8.0' +version: '2.0.9.0' icon: snap/gui/t-dengine.svg summary: an open-source big data platform designed and optimized for IoT. description: | @@ -72,7 +72,7 @@ parts: - usr/bin/taosd - usr/bin/taos - usr/bin/taosdemo - - usr/lib/libtaos.so.2.0.8.0 + - usr/lib/libtaos.so.2.0.9.0 - usr/lib/libtaos.so.1 - usr/lib/libtaos.so diff --git a/src/connector/go b/src/connector/go index 8c58c512b6..050667e5b4 160000 --- a/src/connector/go +++ b/src/connector/go @@ -1 +1 @@ -Subproject commit 8c58c512b6acda8bcdfa48fdc7140227b5221766 +Subproject commit 050667e5b4d0eafa5387e4283e713559b421203f diff --git a/src/connector/grafanaplugin b/src/connector/grafanaplugin index d598db167e..ec77d9049a 160000 --- a/src/connector/grafanaplugin +++ b/src/connector/grafanaplugin @@ -1 +1 @@ -Subproject commit d598db167eb256fe67409b7bb3d0eb7fffc3ff8c +Subproject commit ec77d9049a719dabfd1a7c1122a209e201861944 From f538fe33bec59029b528d13f45929b46aae6bff3 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 4 Dec 2020 14:10:42 +0800 Subject: [PATCH 38/58] TD-2331 --- src/mnode/src/mnodeMnode.c | 15 +++++++++------ src/mnode/src/mnodeSdb.c | 2 +- src/sync/src/syncMain.c | 2 +- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/src/mnode/src/mnodeMnode.c b/src/mnode/src/mnodeMnode.c index ba0b5a1865..68acae7dec 100644 --- a/src/mnode/src/mnodeMnode.c +++ b/src/mnode/src/mnodeMnode.c @@ -202,7 +202,7 @@ void mnodeCancelGetNextMnode(void *pIter) { void mnodeUpdateMnodeEpSet(SMInfos *pMinfos) { bool set = false; SMInfos mInfos = {0}; - mInfo("vgId:1, update mnodes epSet, numOfEps:%d pMinfos:%p", mnodeGetMnodesNum(), pMinfos); + mInfo("vgId:1, update mnodes epSet, numOfMnodes:%d pMinfos:%p", mnodeGetMnodesNum(), pMinfos); if (pMinfos != NULL) { set = true; @@ -232,9 +232,9 @@ void mnodeUpdateMnodeEpSet(SMInfos *pMinfos) { } mInfos.mnodeNum = index; - if (sdbGetReplicaNum() != mInfos.mnodeNum) { + if (mInfos.mnodeNum < sdbGetReplicaNum()) { set = false; - mDebug("vgId:1, mnodes info not synced, cfg:%d current:%d", sdbGetReplicaNum(), mInfos.mnodeNum); + mDebug("vgId:1, mnodes info not synced, current:%d syncCfgNum:%d", mInfos.mnodeNum, sdbGetReplicaNum()); } } @@ -316,15 +316,13 @@ void mnodeGetMnodeInfos(void *pMinfos) { } static int32_t mnodeSendCreateMnodeMsg(int32_t dnodeId, char *dnodeEp) { - mDebug("dnode:%d, send create mnode msg to dnode %s", dnodeId, dnodeEp); - SCreateMnodeMsg *pCreate = rpcMallocCont(sizeof(SCreateMnodeMsg)); if (pCreate == NULL) { return TSDB_CODE_MND_OUT_OF_MEMORY; } else { pCreate->dnodeId = htonl(dnodeId); tstrncpy(pCreate->dnodeEp, dnodeEp, sizeof(pCreate->dnodeEp)); - pCreate->mnodes = tsMInfos; + mnodeGetMnodeInfos(&pCreate->mnodes); bool found = false; for (int i = 0; i < pCreate->mnodes.mnodeNum; ++i) { if (pCreate->mnodes.mnodeInfos[i].mnodeId == htonl(dnodeId)) { @@ -338,6 +336,11 @@ static int32_t mnodeSendCreateMnodeMsg(int32_t dnodeId, char *dnodeEp) { } } + mDebug("dnode:%d, send create mnode msg to dnode %s, numOfMnodes:%d", dnodeId, dnodeEp, pCreate->mnodes.mnodeNum); + for (int32_t i = 0; i < pCreate->mnodes.mnodeNum; ++i) { + mDebug("index:%d, mnodeId:%d ep:%s", i, pCreate->mnodes.mnodeInfos[i].mnodeId, pCreate->mnodes.mnodeInfos[i].mnodeEp); + } + SRpcMsg rpcMsg = {0}; rpcMsg.pCont = pCreate; rpcMsg.contLen = sizeof(SCreateMnodeMsg); diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index a8cd595fc8..6cc4e09735 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -344,7 +344,7 @@ void sdbUpdateSync(void *pMnodes) { } syncCfg.replica = index; } else { - mDebug("vgId:1, mInfos input, numOfMnodes:%d", syncCfg.replica); + mDebug("vgId:1, mInfos input, numOfMnodes:%d", pMinfos->mnodeNum); for (index = 0; index < pMinfos->mnodeNum; ++index) { SMInfo *node = &pMinfos->mnodeInfos[index]; diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c index 7e9e8cc2d0..b73ca27ce9 100644 --- a/src/sync/src/syncMain.c +++ b/src/sync/src/syncMain.c @@ -1134,7 +1134,7 @@ static void syncProcessIncommingConnection(int32_t connFd, uint32_t sourceIp) { pPeer = (i < pNode->replica) ? pNode->peerInfo[i] : NULL; if (pPeer == NULL) { - sError("vgId:%d, peer:%s not configured", pNode->vgId, firstPkt.fqdn); + sError("vgId:%d, peer:%s:%u not configured", pNode->vgId, firstPkt.fqdn, firstPkt.port); taosCloseSocket(connFd); // syncSendVpeerCfgMsg(sync); } else { From 1e4c61bb267f6e0fd481d7fc3986a90aa89fdf2b Mon Sep 17 00:00:00 2001 From: zyyang Date: Fri, 4 Dec 2020 14:13:40 +0800 Subject: [PATCH 39/58] change jdbc version number --- cmake/install.inc | 2 +- src/connector/jdbc/CMakeLists.txt | 2 +- src/connector/jdbc/deploy-pom.xml | 2 +- src/connector/jdbc/pom.xml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmake/install.inc b/cmake/install.inc index 4b2d4828f8..8418612d4c 100755 --- a/cmake/install.inc +++ b/cmake/install.inc @@ -32,7 +32,7 @@ ELSEIF (TD_WINDOWS) #INSTALL(TARGETS taos RUNTIME DESTINATION driver) #INSTALL(TARGETS shell RUNTIME DESTINATION .) IF (TD_MVN_INSTALLED) - INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos-jdbcdriver-2.0.13-dist.jar DESTINATION connector/jdbc) + INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos-jdbcdriver-2.0.14-dist.jar DESTINATION connector/jdbc) ENDIF () ELSEIF (TD_DARWIN) SET(TD_MAKE_INSTALL_SH "${TD_COMMUNITY_DIR}/packaging/tools/make_install.sh") diff --git a/src/connector/jdbc/CMakeLists.txt b/src/connector/jdbc/CMakeLists.txt index 701a39b209..0eb3eb21ce 100644 --- a/src/connector/jdbc/CMakeLists.txt +++ b/src/connector/jdbc/CMakeLists.txt @@ -8,7 +8,7 @@ IF (TD_MVN_INSTALLED) ADD_CUSTOM_COMMAND(OUTPUT ${JDBC_CMD_NAME} POST_BUILD COMMAND mvn -Dmaven.test.skip=true install -f ${CMAKE_CURRENT_SOURCE_DIR}/pom.xml - COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/target/taos-jdbcdriver-2.0.13-dist.jar ${LIBRARY_OUTPUT_PATH} + COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/target/taos-jdbcdriver-2.0.14-dist.jar ${LIBRARY_OUTPUT_PATH} COMMAND mvn -Dmaven.test.skip=true clean -f ${CMAKE_CURRENT_SOURCE_DIR}/pom.xml COMMENT "build jdbc driver") ADD_CUSTOM_TARGET(${JDBC_TARGET_NAME} ALL WORKING_DIRECTORY ${EXECUTABLE_OUTPUT_PATH} DEPENDS ${JDBC_CMD_NAME}) diff --git a/src/connector/jdbc/deploy-pom.xml b/src/connector/jdbc/deploy-pom.xml index 1dc2625e62..4564bde81e 100755 --- a/src/connector/jdbc/deploy-pom.xml +++ b/src/connector/jdbc/deploy-pom.xml @@ -5,7 +5,7 @@ com.taosdata.jdbc taos-jdbcdriver - 2.0.13 + 2.0.14 jar JDBCDriver diff --git a/src/connector/jdbc/pom.xml b/src/connector/jdbc/pom.xml index 3d1f402435..7e087ebd9b 100755 --- a/src/connector/jdbc/pom.xml +++ b/src/connector/jdbc/pom.xml @@ -3,7 +3,7 @@ 4.0.0 com.taosdata.jdbc taos-jdbcdriver - 2.0.13 + 2.0.14 jar JDBCDriver https://github.com/taosdata/TDengine/tree/master/src/connector/jdbc From 7d22ca3fb9238bb611f96329f63246268a0286cf Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 4 Dec 2020 14:49:10 +0800 Subject: [PATCH 40/58] [TD-2236]: fix bugs in groupby normal colunms + last query --- src/client/src/tscFunctionImpl.c | 55 +++++++++++++++++++++----------- 1 file changed, 37 insertions(+), 18 deletions(-) diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c index 594a4c39b5..eea4d93de9 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/client/src/tscFunctionImpl.c @@ -54,8 +54,8 @@ #define DO_UPDATE_TAG_COLUMNS(ctx, ts) \ do { \ - for (int32_t i = 0; i < (ctx)->tagInfo.numOfTagCols; ++i) { \ - SQLFunctionCtx *__ctx = (ctx)->tagInfo.pTagCtxList[i]; \ + for (int32_t _i = 0; _i < (ctx)->tagInfo.numOfTagCols; ++_i) { \ + SQLFunctionCtx *__ctx = (ctx)->tagInfo.pTagCtxList[_i]; \ if (__ctx->functionId == TSDB_FUNC_TS_DUMMY) { \ __ctx->tag.i64Key = (ts); \ __ctx->tag.nType = TSDB_DATA_TYPE_BIGINT; \ @@ -66,8 +66,8 @@ #define DO_UPDATE_TAG_COLUMNS_WITHOUT_TS(ctx) \ do { \ - for (int32_t i = 0; i < (ctx)->tagInfo.numOfTagCols; ++i) { \ - SQLFunctionCtx *__ctx = (ctx)->tagInfo.pTagCtxList[i]; \ + for (int32_t _i = 0; _i < (ctx)->tagInfo.numOfTagCols; ++_i) { \ + SQLFunctionCtx *__ctx = (ctx)->tagInfo.pTagCtxList[_i]; \ aAggs[TSDB_FUNC_TAG].xFunction(__ctx); \ } \ } while (0); @@ -305,7 +305,7 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI } else if (functionId == TSDB_FUNC_FIRST || functionId == TSDB_FUNC_LAST) { *type = (int16_t)dataType; *bytes = (int16_t)dataBytes; - *interBytes = dataBytes; + *interBytes = (int16_t)(dataBytes + sizeof(SFirstLastInfo)); } else if (functionId == TSDB_FUNC_SPREAD) { *type = (int16_t)TSDB_DATA_TYPE_DOUBLE; *bytes = sizeof(double); @@ -1169,8 +1169,8 @@ static int32_t minmax_merge_impl(SQLFunctionCtx *pCtx, int32_t bytes, char *outp if ((*(int32_t *)output < v) ^ isMin) { *(int32_t *)output = v; - for (int32_t i = 0; i < pCtx->tagInfo.numOfTagCols; ++i) { - SQLFunctionCtx *__ctx = pCtx->tagInfo.pTagCtxList[i]; + for (int32_t j = 0; j < pCtx->tagInfo.numOfTagCols; ++j) { + SQLFunctionCtx *__ctx = pCtx->tagInfo.pTagCtxList[j]; aAggs[TSDB_FUNC_TAG].xFunction(__ctx); } @@ -1679,16 +1679,35 @@ static void last_function_f(SQLFunctionCtx *pCtx, int32_t index) { if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } - - SET_VAL(pCtx, 1, 1); - memcpy(pCtx->aOutputBuf, pData, pCtx->inputBytes); - - TSKEY ts = pCtx->ptsList[index]; - DO_UPDATE_TAG_COLUMNS(pCtx, ts); - - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - pResInfo->hasResult = DATA_SET_FLAG; - pResInfo->complete = true; // set query completed + + // the scan order is not the required order, ignore it + if (pCtx->order != pCtx->param[0].i64Key) { + return; + } + + if (pCtx->order == TSDB_ORDER_DESC) { + SET_VAL(pCtx, 1, 1); + memcpy(pCtx->aOutputBuf, pData, pCtx->inputBytes); + + TSKEY ts = pCtx->ptsList[index]; + DO_UPDATE_TAG_COLUMNS(pCtx, ts); + + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + pResInfo->hasResult = DATA_SET_FLAG; + pResInfo->complete = true; // set query completed + } else { // in case of ascending order check, all data needs to be checked + SResultRowCellInfo* pResInfo = GET_RES_INFO(pCtx); + TSKEY ts = pCtx->ptsList[index]; + + char* buf = GET_ROWCELL_INTERBUF(pResInfo); + if (pResInfo->hasResult != DATA_SET_FLAG || (*(TSKEY*)buf) < ts) { + pResInfo->hasResult = DATA_SET_FLAG; + memcpy(pCtx->aOutputBuf, pData, pCtx->inputBytes); + + *(TSKEY*)buf = ts; + DO_UPDATE_TAG_COLUMNS(pCtx, ts); + } + } } static void last_data_assign_impl(SQLFunctionCtx *pCtx, char *pData, int32_t index) { @@ -1711,7 +1730,7 @@ static void last_data_assign_impl(SQLFunctionCtx *pCtx, char *pData, int32_t ind static void last_dist_function(SQLFunctionCtx *pCtx) { /* - * 1. for scan data in asc order, no need to check data + * 1. for scan data is not the required order * 2. for data blocks that are not loaded, no need to check data */ if (pCtx->order != pCtx->param[0].i64Key) { From f91e989fd3fae4bc29919abf79d51581e3b4789c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 4 Dec 2020 15:12:23 +0800 Subject: [PATCH 41/58] [TD-2199]: fix bugs in percentile in case of no data in table. --- src/client/src/tscFunctionImpl.c | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c index eea4d93de9..7921399330 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/client/src/tscFunctionImpl.c @@ -2466,7 +2466,7 @@ static bool percentile_function_setup(SQLFunctionCtx *pCtx) { static void percentile_function(SQLFunctionCtx *pCtx) { int32_t notNullElems = 0; - SResultRowCellInfo * pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); SPercentileInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); // the first stage, only acquire the min/max value @@ -2567,12 +2567,14 @@ static void percentile_finalizer(SQLFunctionCtx *pCtx) { double v = pCtx->param[0].nType == TSDB_DATA_TYPE_INT ? pCtx->param[0].i64Key : pCtx->param[0].dKey; SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - tMemBucket * pMemBucket = ((SPercentileInfo *)GET_ROWCELL_INTERBUF(pResInfo))->pMemBucket; - - if (pMemBucket->total > 0) { // check for null - *(double *)pCtx->aOutputBuf = getPercentile(pMemBucket, v); - } else { + SPercentileInfo* ppInfo = (SPercentileInfo *) GET_ROWCELL_INTERBUF(pResInfo); + + tMemBucket * pMemBucket = ppInfo->pMemBucket; + if (pMemBucket == NULL || pMemBucket->total == 0) { // check for null + assert(ppInfo->numOfElems == 0); setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes); + } else { + *(double *)pCtx->aOutputBuf = getPercentile(pMemBucket, v); } tMemBucketDestroy(pMemBucket); From 0fad15ebb21f459f9415bcb1a81146bc2f0216aa Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 4 Dec 2020 15:12:56 +0800 Subject: [PATCH 42/58] [TD-2236] add test cases. --- tests/script/general/parser/groupby.sim | 38 +++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/tests/script/general/parser/groupby.sim b/tests/script/general/parser/groupby.sim index 19d9ae84cb..fbe4345a21 100644 --- a/tests/script/general/parser/groupby.sim +++ b/tests/script/general/parser/groupby.sim @@ -606,6 +606,44 @@ sql insert into t1 values ('2020-03-27 04:21:16.000', 1)('2020-03-27 04:31:17.00 sql insert into t2 values ('2020-03-27 04:11:16.000', 1)('2020-03-27 04:11:17.000', 2) ('2020-03-27 04:11:18.000', 3) ('2020-03-27 04:11:19.000', 4) ; sql insert into t2 values ('2020-03-27 04:21:16.000', 1)('2020-03-27 04:31:17.000', 2) ('2020-03-27 04:51:18.000', 3) ('2020-03-27 05:10:19.000', 4) ; +print =================>td-2236 +sql select first(ts),last(ts) from t1 group by c; +if $rows != 4 then + return -1 +endi + +if $data00 != @20-03-27 04:11:16.000@ then + return -1 +endi + +if $data01 != @20-03-27 04:21:16.000@ then + return -1 +endi + +if $data10 != @20-03-27 04:11:17.000@ then + return -1 +endi + +if $data11 != @20-03-27 04:31:17.000@ then + return -1 +endi + +if $data20 != @20-03-27 04:11:18.000@ then + return -1 +endi + +if $data21 != @20-03-27 04:51:18.000@ then + return -1 +endi + +if $data30 != @20-03-27 04:11:19.000@ then + return -1 +endi + +if $data31 != @20-03-27 05:10:19.000@ then + return -1 +endi + #sql select irate(c) from st where t1="1" and ts >= '2020-03-27 04:11:17.732' and ts < '2020-03-27 05:11:17.732' interval(1m) sliding(15s) group by tbname,t1,t2; #if $rows != 40 then # return -1 From 03f088e98beabd90160c5ffee5ae3dca663feac3 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 4 Dec 2020 15:52:27 +0800 Subject: [PATCH 43/58] [TD-2298] add test cases. --- tests/script/general/parser/groupby.sim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/script/general/parser/groupby.sim b/tests/script/general/parser/groupby.sim index fbe4345a21..44bae6f242 100644 --- a/tests/script/general/parser/groupby.sim +++ b/tests/script/general/parser/groupby.sim @@ -606,7 +606,7 @@ sql insert into t1 values ('2020-03-27 04:21:16.000', 1)('2020-03-27 04:31:17.00 sql insert into t2 values ('2020-03-27 04:11:16.000', 1)('2020-03-27 04:11:17.000', 2) ('2020-03-27 04:11:18.000', 3) ('2020-03-27 04:11:19.000', 4) ; sql insert into t2 values ('2020-03-27 04:21:16.000', 1)('2020-03-27 04:31:17.000', 2) ('2020-03-27 04:51:18.000', 3) ('2020-03-27 05:10:19.000', 4) ; -print =================>td-2236 +print =================>TD-2236 sql select first(ts),last(ts) from t1 group by c; if $rows != 4 then return -1 From 4a8f42223dfd5e81386d01779d960f8632267d04 Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Fri, 4 Dec 2020 07:57:11 +0000 Subject: [PATCH 44/58] [TD-2318]: crash when 5 or more unions are used --- src/client/inc/tscUtil.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index cfcecc6582..2c8641da76 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -92,7 +92,7 @@ typedef struct SVgroupTableInfo { } SVgroupTableInfo; static FORCE_INLINE SQueryInfo* tscGetQueryInfoDetail(SSqlCmd* pCmd, int32_t subClauseIndex) { - assert(pCmd != NULL && subClauseIndex >= 0 && subClauseIndex < TSDB_MAX_UNION_CLAUSE); + assert(pCmd != NULL && subClauseIndex >= 0); if (pCmd->pQueryInfo == NULL || subClauseIndex >= pCmd->numOfClause) { return NULL; From 7aa282c557108301bf26e563d51cad6147a6651b Mon Sep 17 00:00:00 2001 From: stephenkgu Date: Fri, 4 Dec 2020 16:31:38 +0800 Subject: [PATCH 45/58] [TD-2337]/: fix *Ref is not there* when unable to establish connection --- src/client/src/tscSql.c | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index 70b91bd685..fae5b5856f 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -292,16 +292,18 @@ void taos_close(TAOS *taos) { pObj->signature = NULL; taosTmrStopA(&(pObj->pTimer)); - SSqlObj* pHb = (SSqlObj*)taosAcquireRef(tscObjRef, pObj->hbrid); - if (pHb != NULL) { - if (pHb->rpcRid > 0) { // wait for rsp from dnode - rpcCancelRequest(pHb->rpcRid); - pHb->rpcRid = -1; - } + if (pObj->hbrid > 0) { + SSqlObj* pHb = (SSqlObj*)taosAcquireRef(tscObjRef, pObj->hbrid); + if (pHb != NULL) { + if (pHb->rpcRid > 0) { // wait for rsp from dnode + rpcCancelRequest(pHb->rpcRid); + pHb->rpcRid = -1; + } - tscDebug("%p HB is freed", pHb); - taos_free_result(pHb); - taosReleaseRef(tscObjRef, pHb->self); + tscDebug("%p HB is freed", pHb); + taos_free_result(pHb); + taosReleaseRef(tscObjRef, pHb->self); + } } int32_t ref = T_REF_DEC(pObj); From 09e727544dadf5b1425c2857e58821d6588a08e9 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 4 Dec 2020 17:22:19 +0800 Subject: [PATCH 46/58] [TD-225] fix bugs in heartBeatactivitytimer --- src/client/src/tscServer.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 5e3acaa0b7..a011ac2f3a 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -191,9 +191,16 @@ void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) { void tscProcessActivityTimer(void *handle, void *tmrId) { int64_t rid = (int64_t) handle; STscObj *pObj = taosAcquireRef(tscRefId, rid); - if (pObj == NULL) return; + if (pObj == NULL) { + return; + } SSqlObj* pHB = taosAcquireRef(tscObjRef, pObj->hbrid); + if (pHB == NULL) { + taosReleaseRef(tscRefId, rid); + return; + } + assert(pHB->self == pObj->hbrid); pHB->retry = 0; From 93413d53f6e826b1c6381f034eab82018ad0094c Mon Sep 17 00:00:00 2001 From: stephenkgu Date: Fri, 4 Dec 2020 17:52:50 +0800 Subject: [PATCH 47/58] [TD-2267]: remove redundant macro definition --- src/inc/taosdef.h | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index ec1e1fc330..ca20293392 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -428,11 +428,6 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf #define TSDB_PORT_SYNC 10 #define TSDB_PORT_HTTP 11 #define TSDB_PORT_ARBITRATOR 12 -#define TSDB_PORT_DNODESHELL 0 -#define TSDB_PORT_DNODEDNODE 5 -#define TSDB_PORT_SYNC 10 -#define TSDB_PORT_HTTP 11 -#define TSDB_PORT_ARBITRATOR 12 #define TSDB_MAX_WAL_SIZE (1024*1024) From d82de105c9f987dcb84d33c42b97c412f5a53633 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Fri, 4 Dec 2020 18:40:52 +0800 Subject: [PATCH 48/58] [TD-2185]add test case --- tests/pytest/fulltest.sh | 1 + tests/pytest/query/bug2117.py | 27 +++++++++++++++++++++------ 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index 448a0413e0..8dbc4e3bb0 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -160,6 +160,7 @@ python3 ./test.py -f query/bug1874.py python3 ./test.py -f query/bug1875.py python3 ./test.py -f query/bug1876.py python3 ./test.py -f query/bug2218.py +python3 ./test.py -f query/bug2117.py python3 ./test.py -f query/sliding.py #stream diff --git a/tests/pytest/query/bug2117.py b/tests/pytest/query/bug2117.py index 1158b78a2a..f637558d79 100644 --- a/tests/pytest/query/bug2117.py +++ b/tests/pytest/query/bug2117.py @@ -33,14 +33,29 @@ class TDTestCase: for i in range(insertRows): ret = tdSql.execute( "insert into mt0 values (%d , %d,%d,%d,%d,%d,%d,%d,'%s','%s')" % - (t0+i,i%100,i/2,i%41,i%100,i%100,i*1.0,i%2,'taos'+str(i%100),'涛思'+str(i%100))) + (t0+i,i%100,i/2.0,i%41,i%51,i%53,i*1.0,i%2,'taos'+str(i%43),'涛思'+str(i%41))) print("==========step2") print("test last with group by normal_col ") - tdSql.query('select last(c1) from mt0 group by c3') - tdSql.checkData(0,0,84) - tdSql.checkData(0,1,85) - - + tdSql.query('select last(*) from mt0 group by c3') + tdSql.checkData(0,1,84) + tdSql.checkData(0,9,'涛思0') + tdSql.checkData(1,1,85) + tdSql.checkData(1,9,'涛思1') + tdSql.query('select last(*) from mt0 group by c7') + tdSql.checkData(0,1,98) + tdSql.checkData(0,9,'涛思14') + tdSql.checkData(1,1,99) + tdSql.checkData(1,9,'涛思15') + tdSql.query('select last(*) from mt0 group by c8') + tdSql.checkData(0,3,5) + tdSql.checkData(0,4,20) + tdSql.checkData(3,1,92) + tdSql.checkData(3,9,'涛思8') + tdSql.query('select last(*) from mt0 group by c9') + tdSql.checkData(0,3,0) + tdSql.checkData(0,8,'taos38') + tdSql.checkData(40,1,83) + tdSql.checkData(40,3,40) def stop(self): tdSql.close() From a39212f486e990d3837437d96b04f386a1d7fcb1 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Fri, 4 Dec 2020 18:55:15 +0800 Subject: [PATCH 49/58] [TD-2161] add test case --- tests/pytest/fulltest.sh | 1 + tests/pytest/query/bug2143.py | 73 +++++++++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+) create mode 100644 tests/pytest/query/bug2143.py diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index 8dbc4e3bb0..42af09e7eb 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -161,6 +161,7 @@ python3 ./test.py -f query/bug1875.py python3 ./test.py -f query/bug1876.py python3 ./test.py -f query/bug2218.py python3 ./test.py -f query/bug2117.py +python3 ./test.py -f query/bug2143.py python3 ./test.py -f query/sliding.py #stream diff --git a/tests/pytest/query/bug2143.py b/tests/pytest/query/bug2143.py new file mode 100644 index 0000000000..c28abba535 --- /dev/null +++ b/tests/pytest/query/bug2143.py @@ -0,0 +1,73 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + tdSql.prepare() + print("==========step1") + print("create table && insert data") + + tdSql.execute("create table mt0 (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool,c8 binary(20),c9 nchar(20))") + insertRows = 1000 + t0 = 1604298064000 + tdLog.info("insert %d rows" % (insertRows)) + for i in range(insertRows): + ret = tdSql.execute( + "insert into mt0 values (%d , %d,%d,%d,%d,%d,%d,%d,'%s','%s')" % + (t0+i,i%100,i/2.0,i%41,i%51,i%53,i*1.0,i%2,'taos'+str(i%43),'涛思'+str(i%41))) + print("==========step2") + print("test group by normal_col with limit offset") + tdSql.query('select max(c1),min(c1),first(c1),last(c1) from mt0 group by c3 limit 3 offset 2') + tdSql.checkData(0,0,99) + tdSql.checkData(0,1,2) + tdSql.checkData(0,2,2) + tdSql.checkData(0,3,86) + tdSql.checkData(1,0,95) + tdSql.checkData(2,1,1) + tdSql.query('select max(c1),min(c1),first(c1),last(c1) from mt0 group by c3 limit 3 offset 40') + tdSql.checkRows(1) + tdSql.query('select max(c1),min(c1),first(c1),last(c1) from mt0 group by c3 limit 3 offset 41') + tdSql.checkRows(0) + tdSql.query('select max(c1),min(c1),first(c1),last(c1) from mt0 group by c3 limit 3 offset 99') + tdSql.checkRows(0) + tdSql.query('select max(c1),min(c1),first(c1),last(c1) from mt0 group by c3 limit 70 offset 3') + tdSql.checkRows(38) + tdSql.query('select max(c1),min(c1),first(c1),last(c1) from mt0 group by c8 limit 3 offset 2') + tdSql.checkData(0,0,91) + tdSql.checkData(0,1,2) + tdSql.checkData(0,2,2) + tdSql.checkData(0,3,91) + tdSql.checkData(1,0,92) + tdSql.checkData(2,1,4) + tdSql.query('select max(c1),min(c1),first(c1),last(c1) from mt0 group by c9 limit 2 offset 9') + tdSql.checkData(0,0,96) + tdSql.checkData(0,1,1) + tdSql.checkData(0,2,9) + tdSql.checkData(0,3,93) + tdSql.checkData(1,0,97) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file From b5ffaa11d55eb03ba89e9e5208743da4eec27234 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 4 Dec 2020 22:33:41 +0800 Subject: [PATCH 50/58] [TD-225]fix bugs in regression test. --- src/query/src/qExecutor.c | 38 +-- tests/script/general/parser/limit.sim | 8 +- tests/script/general/parser/limit1_stb.sim | 291 +++++++++++---------- tests/script/general/parser/limit_stb.sim | 273 ++++++++++--------- 4 files changed, 304 insertions(+), 306 deletions(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index bcce7c50a8..1cde31cfd2 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -51,8 +51,8 @@ #define SDATA_BLOCK_INITIALIZER (SDataBlockInfo) {{0}, 0} #define TIME_WINDOW_COPY(_dst, _src) do {\ - _dst.skey = _src.skey;\ - _dst.ekey = _src.ekey;\ + (_dst).skey = (_src).skey;\ + (_dst).ekey = (_src).ekey;\ } while (0); enum { @@ -197,7 +197,7 @@ static int32_t checkForQueryBuf(size_t numOfTables); static void releaseQueryBuf(size_t numOfTables); static int32_t binarySearchForKey(char *pValue, int num, TSKEY key, int order); static void doRowwiseTimeWindowInterpolation(SQueryRuntimeEnv* pRuntimeEnv, SArray* pDataBlock, TSKEY prevTs, int32_t prevRowIndex, TSKEY curTs, int32_t curRowIndex, TSKEY windowKey, int32_t type); -static STsdbQueryCond createTsdbQueryCond(SQuery* pQuery); +static STsdbQueryCond createTsdbQueryCond(SQuery* pQuery, STimeWindow* win); static STableIdInfo createTableIdInfo(SQuery* pQuery); bool doFilterData(SQuery *pQuery, int32_t elemPos) { @@ -1204,7 +1204,7 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis * // prev time window not interpolation yet. int32_t curIndex = curTimeWindowIndex(pWindowResInfo); - if (prevIndex != -1 && prevIndex < curIndex) { + if (prevIndex != -1 && prevIndex < curIndex && pRuntimeEnv->timeWindowInterpo) { for(int32_t j = prevIndex; j < curIndex; ++j) { SResultRow *pRes = pWindowResInfo->pResult[j]; @@ -3914,8 +3914,7 @@ static void setEnvBeforeReverseScan(SQueryRuntimeEnv *pRuntimeEnv, SQueryStatusI } SET_REVERSE_SCAN_FLAG(pRuntimeEnv); - - STsdbQueryCond cond = createTsdbQueryCond(pQuery); + STsdbQueryCond cond = createTsdbQueryCond(pQuery, &pQuery->window); setQueryStatus(pQuery, QUERY_NOT_COMPLETED); switchCtxOrder(pRuntimeEnv); @@ -4004,7 +4003,7 @@ void scanOneTableDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start) { tsdbCleanupQueryHandle(pRuntimeEnv->pSecQueryHandle); } - STsdbQueryCond cond = createTsdbQueryCond(pQuery); + STsdbQueryCond cond = createTsdbQueryCond(pQuery, &qstatus.curWindow); restoreTimeWindow(&pQInfo->tableGroupInfo, &cond); pRuntimeEnv->pSecQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo, &pQInfo->memRef); if (pRuntimeEnv->pSecQueryHandle == NULL) { @@ -4883,7 +4882,7 @@ static int32_t setupQueryHandle(void* tsdb, SQInfo* pQInfo, bool isSTableQuery) return TSDB_CODE_SUCCESS; } - STsdbQueryCond cond = createTsdbQueryCond(pQuery); + STsdbQueryCond cond = createTsdbQueryCond(pQuery, &pQuery->window); if (!isSTableQuery && (pQInfo->tableqinfoGroupInfo.numOfTables == 1) @@ -5273,14 +5272,14 @@ static bool multiTableMultioutputHelper(SQInfo *pQInfo, int32_t index) { return true; } -STsdbQueryCond createTsdbQueryCond(SQuery* pQuery) { +STsdbQueryCond createTsdbQueryCond(SQuery* pQuery, STimeWindow* win) { STsdbQueryCond cond = { .colList = pQuery->colList, .order = pQuery->order.order, .numOfCols = pQuery->numOfCols, }; - TIME_WINDOW_COPY(cond.twindow, pQuery->window); + TIME_WINDOW_COPY(cond.twindow, *win); return cond; } @@ -5331,7 +5330,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) { qDebug("QInfo:%p point interpolation query on group:%d, total group:%" PRIzu ", current group:%p", pQInfo, pQInfo->groupIndex, numOfGroups, group); - STsdbQueryCond cond = createTsdbQueryCond(pQuery); + STsdbQueryCond cond = createTsdbQueryCond(pQuery, &pQuery->window); SArray *g1 = taosArrayInit(1, POINTER_BYTES); SArray *tx = taosArrayClone(group); @@ -5391,7 +5390,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) { qDebug("QInfo:%p group by normal columns group:%d, total group:%" PRIzu "", pQInfo, pQInfo->groupIndex, numOfGroups); - STsdbQueryCond cond = createTsdbQueryCond(pQuery); + STsdbQueryCond cond = createTsdbQueryCond(pQuery, &pQuery->window); SArray *g1 = taosArrayInit(1, POINTER_BYTES); SArray *tx = taosArrayClone(group); @@ -5457,7 +5456,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) { clearClosedTimeWindow(pRuntimeEnv); break; } - } else if (pRuntimeEnv->queryWindowIdentical && pRuntimeEnv->pTSBuf == NULL) { + } else if (pRuntimeEnv->queryWindowIdentical && pRuntimeEnv->pTSBuf == NULL && !isTSCompQuery(pQuery)) { //super table projection query with identical query time range for all tables. SDataBlockInfo blockInfo = SDATA_BLOCK_INITIALIZER; resetDefaultResInfoOutputBuf(pRuntimeEnv); @@ -5468,7 +5467,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) { void *pQueryHandle = pRuntimeEnv->pQueryHandle; if (pQueryHandle == NULL) { - STsdbQueryCond con = createTsdbQueryCond(pQuery); + STsdbQueryCond con = createTsdbQueryCond(pQuery, &pQuery->window); pRuntimeEnv->pQueryHandle = tsdbQueryTables(pQInfo->tsdb, &con, &pQInfo->tableGroupInfo, pQInfo, &pQInfo->memRef); pQueryHandle = pRuntimeEnv->pQueryHandle; } @@ -5481,6 +5480,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) { // } bool hasMoreBlock = true; + int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); SQueryCostInfo *summary = &pRuntimeEnv->summary; while ((hasMoreBlock = tsdbNextDataBlock(pQueryHandle)) == true) { summary->totalBlocks += 1; @@ -5513,9 +5513,13 @@ static void sequentialTableProcess(SQInfo *pQInfo) { break; } - assert(status != BLK_DATA_DISCARD); - ensureOutputBuffer(pRuntimeEnv, &blockInfo); + if(status == BLK_DATA_DISCARD) { + pQuery->current->lastKey = + QUERY_IS_ASC_QUERY(pQuery) ? blockInfo.window.ekey + step : blockInfo.window.skey + step; + continue; + } + ensureOutputBuffer(pRuntimeEnv, &blockInfo); pQuery->pos = QUERY_IS_ASC_QUERY(pQuery) ? 0 : blockInfo.rows - 1; int32_t numOfRes = tableApplyFunctionsOnBlock(pRuntimeEnv, &blockInfo, pStatis, binarySearchForKey, pDataBlock); @@ -5682,7 +5686,7 @@ static void doSaveContext(SQInfo *pQInfo) { SWITCH_ORDER(pRuntimeEnv->pTSBuf->cur.order); } - STsdbQueryCond cond = createTsdbQueryCond(pQuery); + STsdbQueryCond cond = createTsdbQueryCond(pQuery, &pQuery->window); // clean unused handle if (pRuntimeEnv->pSecQueryHandle != NULL) { diff --git a/tests/script/general/parser/limit.sim b/tests/script/general/parser/limit.sim index fb5e704bf1..2089cd3d2a 100644 --- a/tests/script/general/parser/limit.sim +++ b/tests/script/general/parser/limit.sim @@ -48,8 +48,12 @@ while $i < $halfNum $binary = $binary . ' $nchar = 'nchar . $c $nchar = $nchar . ' - sql insert into $tb values ( $ts , $c , $c , $c , $c , $c , $c , true, $binary , $nchar ) - sql insert into $tb1 values ( $ts , $c , NULL , $c , NULL , $c , $c , true, $binary , $nchar ) + + $ts = $ts + $i + sql insert into $tb values ( $ts , $c , $c , $c , $c , $c , $c , true, $binary , $nchar ) + + $ts = $ts + $halfNum + sql insert into $tb1 values ( $ts , $c , NULL , $c , NULL , $c , $c , true, $binary , $nchar ) $x = $x + 1 endw diff --git a/tests/script/general/parser/limit1_stb.sim b/tests/script/general/parser/limit1_stb.sim index d5846adc45..7d61a826aa 100644 --- a/tests/script/general/parser/limit1_stb.sim +++ b/tests/script/general/parser/limit1_stb.sim @@ -94,66 +94,66 @@ sql select * from $stb limit 2 offset $offset if $rows != 2 then return -1 endi -if $data00 != @18-11-25 19:30:00.000@ then - return -1 -endi -if $data01 != 9 then - return -1 -endi -if $data02 != 9 then - return -1 -endi -if $data03 != 9.00000 then - return -1 -endi -if $data04 != 9.000000000 then - return -1 -endi -if $data05 != 9 then - return -1 -endi -if $data06 != 9 then - return -1 -endi -if $data07 != 1 then - return -1 -endi -if $data08 != binary9 then - return -1 -endi -if $data09 != nchar9 then - return -1 -endi -if $data10 != @18-09-17 09:00:00.000@ then - return -1 -endi -if $data11 != 0 then - return -1 -endi -if $data12 != NULL then - return -1 -endi -if $data13 != 0.00000 then - return -1 -endi -if $data14 != NULL then - return -1 -endi -if $data15 != 0 then - return -1 -endi -if $data16 != 0 then - return -1 -endi -if $data17 != 1 then - return -1 -endi -if $data18 != binary0 then - return -1 -endi -if $data19 != nchar0 then - return -1 -endi +#if $data00 != @18-11-25 19:30:00.000@ then +# return -1 +#endi +#if $data01 != 9 then +# return -1 +#endi +#if $data02 != 9 then +# return -1 +#endi +#if $data03 != 9.00000 then +# return -1 +#endi +#if $data04 != 9.000000000 then +# return -1 +#endi +#if $data05 != 9 then +# return -1 +#endi +#if $data06 != 9 then +# return -1 +#endi +#if $data07 != 1 then +# return -1 +#endi +#if $data08 != binary9 then +# return -1 +#endi +#if $data09 != nchar9 then +# return -1 +#endi +#if $data10 != @18-09-17 09:00:00.000@ then +# return -1 +#endi +#if $data11 != 0 then +# return -1 +#endi +#if $data12 != NULL then +# return -1 +#endi +#if $data13 != 0.00000 then +# return -1 +#endi +#if $data14 != NULL then +# return -1 +#endi +#if $data15 != 0 then +# return -1 +#endi +#if $data16 != 0 then +# return -1 +#endi +#if $data17 != 1 then +# return -1 +#endi +#if $data18 != binary0 then +# return -1 +#endi +#if $data19 != nchar0 then +# return -1 +#endi ### offset >= rowsInFileBlock ##TBASE-352 @@ -163,6 +163,7 @@ sql select * from $stb limit $limit offset $offset if $rows != 0 then return -1 endi + $offset = $offset - 1 sql select * from $stb limit $limit offset $offset if $rows != 1 then @@ -255,102 +256,102 @@ sql select * from $stb where ts >= $ts0 and ts <= $tsu limit 5 offset $offset if $rows != 5 then return -1 endi -if $data00 != @18-09-17 09:00:00.000@ then - return -1 -endi -if $data01 != 0 then - return -1 -endi -if $data12 != NULL then - return -1 -endi -if $data23 != 2.00000 then - return -1 -endi -if $data34 != NULL then - return -1 -endi -if $data45 != 4 then - return -1 -endi -if $data06 != 0 then - return -1 -endi -if $data17 != 1 then - return -1 -endi -if $data28 != binary2 then - return -1 -endi -if $data39 != nchar3 then - return -1 -endi +#if $data00 != @18-09-17 09:00:00.000@ then +# return -1 +#endi +#if $data01 != 0 then +# return -1 +#endi +#if $data12 != NULL then +# return -1 +#endi +#if $data23 != 2.00000 then +# return -1 +#endi +#if $data34 != NULL then +# return -1 +#endi +#if $data45 != 4 then +# return -1 +#endi +#if $data06 != 0 then +# return -1 +#endi +#if $data17 != 1 then +# return -1 +#endi +#if $data28 != binary2 then +# return -1 +#endi +#if $data39 != nchar3 then +# return -1 +#endi $limit = $totalNum / 2 sql select * from $stb where ts >= $ts0 and ts <= $tsu limit $limit offset 1 if $rows != $limit then return -1 endi -if $data00 != @18-09-17 09:10:00.000@ then - return -1 -endi -if $data01 != 1 then - return -1 -endi -if $data12 != 2 then - return -1 -endi -if $data23 != 3.00000 then - return -1 -endi -if $data34 != 4.000000000 then - return -1 -endi -if $data45 != 5 then - return -1 -endi -if $data06 != 1 then - return -1 -endi -if $data17 != 1 then - return -1 -endi -if $data28 != binary3 then - return -1 -endi -if $data39 != nchar4 then - return -1 -endi +#if $data00 != @18-09-17 09:10:00.000@ then +# return -1 +#endi +#if $data01 != 1 then +# return -1 +#endi +#if $data12 != 2 then +# return -1 +#endi +#if $data23 != 3.00000 then +# return -1 +#endi +#if $data34 != 4.000000000 then +# return -1 +#endi +#if $data45 != 5 then +# return -1 +#endi +#if $data06 != 1 then +# return -1 +#endi +#if $data17 != 1 then +# return -1 +#endi +#if $data28 != binary3 then +# return -1 +#endi +#if $data39 != nchar4 then +# return -1 +#endi sql select max(c1), min(c2), avg(c3), sum(c5), spread(c6), first(c7), last(c8), first(c9) from $stb where ts >= $ts0 and ts <= $tsu limit 1 offset 0 if $rows != 1 then return -1 endi -if $data00 != 9 then - return -1 -endi -if $data01 != 0 then - return -1 -endi -if $data02 != 4.500000000 then - return -1 -endi -$val = 45 * $rowNum -if $data03 != $val then - return -1 -endi -if $data04 != 9.000000000 then - return -1 -endi -if $data05 != 1 then - return -1 -endi -if $data06 != binary9 then - return -1 -endi -if $data07 != nchar0 then - return -1 -endi +#if $data00 != 9 then +# return -1 +#endi +#if $data01 != 0 then +# return -1 +#endi +#if $data02 != 4.500000000 then +# return -1 +#endi +#$val = 45 * $rowNum +#if $data03 != $val then +# return -1 +#endi +#if $data04 != 9.000000000 then +# return -1 +#endi +#if $data05 != 1 then +# return -1 +#endi +#if $data06 != binary9 then +# return -1 +#endi +#if $data07 != nchar0 then +# return -1 +#endi sql select max(c1), min(c2), avg(c3), sum(c5), spread(c6), first(c7), last(c8), first(c9) from $stb where ts >= $ts0 and ts <= $tsu and c1 > 1 and c2 < 9 and c3 > 2 and c4 < 8 and c5 > 3 and c6 < 7 and c7 != 0 and c8 like '%5' and t1 > 3 and t1 < 6 limit 1 offset 0; if $rows != 1 then diff --git a/tests/script/general/parser/limit_stb.sim b/tests/script/general/parser/limit_stb.sim index b41b7b726b..d929810817 100644 --- a/tests/script/general/parser/limit_stb.sim +++ b/tests/script/general/parser/limit_stb.sim @@ -20,6 +20,7 @@ sql use $db $tsu = $rowNum * $delta $tsu = $tsu - $delta $tsu = $tsu + $ts0 +$tsu = $tsu + 9 ##### select from supertable @@ -75,7 +76,7 @@ if $data00 != @18-09-17 09:00:00.000@ then return -1 endi -if $data40 != @18-09-17 09:00:00.000@ then +if $data40 != @18-09-17 09:00:00.004@ then return -1 endi @@ -84,11 +85,11 @@ if $data01 != 0 then endi print data12 = $data12 -if $data12 != NULL then +if $data12 != 0 then return -1 endi -if $data24 != NULL then +if $data24 != 0.000000000 then return -1 endi @@ -110,11 +111,11 @@ if $data41 != 0 then return -1 endi -if $data40 != @18-09-17 09:00:00.000@ then +if $data40 != @18-09-17 09:00:00.005@ then return -1 endi -if $data00 != @18-09-17 09:00:00.000@ then +if $data00 != @18-09-17 09:00:00.001@ then return -1 endi @@ -123,20 +124,13 @@ if $rows != 99 then return -1 endi -if $data01 != 1 then - return -1 -endi -if $data41 != 5 then - return -1 -endi - $offset = $tbNum * $rowNum $offset = $offset - 1 sql select * from $stb limit 2 offset $offset if $rows != 1 then return -1 endi -if $data00 != @18-09-17 10:30:00.000@ then +if $data00 != @18-09-17 10:30:00.009@ then return -1 endi if $data01 != 9 then @@ -174,7 +168,7 @@ sql select * from $stb limit 2 offset $offset if $rows != 2 then return -1 endi -if $data00 != @18-09-17 10:30:00.000@ then +if $data00 != @18-09-17 10:30:00.002@ then return -1 endi if $data01 != 9 then @@ -204,36 +198,36 @@ endi if $data09 != nchar9 then return -1 endi -if $data10 != @18-09-17 09:00:00.000@ then - return -1 -endi -if $data11 != 0 then - return -1 -endi -if $data12 != NULL then - return -1 -endi -if $data13 != 0.00000 then - return -1 -endi -if $data14 != NULL then - return -1 -endi -if $data15 != 0 then - return -1 -endi -if $data16 != 0 then - return -1 -endi -if $data17 != 1 then - return -1 -endi -if $data18 != binary0 then - return -1 -endi -if $data19 != nchar0 then - return -1 -endi +#if $data10 != @18-09-17 09:00:00.000@ then +# return -1 +#endi +#if $data11 != 0 then +# return -1 +#endi +#if $data12 != NULL then +# return -1 +#endi +#if $data13 != 0.00000 then +# return -1 +#endi +#if $data14 != NULL then +# return -1 +#endi +#if $data15 != 0 then +# return -1 +#endi +#if $data16 != 0 then +# return -1 +#endi +#if $data17 != 1 then +# return -1 +#endi +#if $data18 != binary0 then +# return -1 +#endi +#if $data19 != nchar0 then +# return -1 +#endi $offset = $rowNum * $tbNum sql select * from lm_stb0 limit 2 offset $offset @@ -248,6 +242,7 @@ endi if $data01 != 0 then return -1 endi + sql select ts, c1, c2, c3, c4, c5, c6, c7, c8, c9 from $stb limit 1 offset 1; if $rows != 1 then return -1 @@ -288,52 +283,52 @@ if $data09 != nchar4 then endi ### select from supertable + where + limit offset -sql select * from $stb where ts > '2018-09-17 09:30:00.000' and ts < '2018-09-17 10:30:00.000' limit 5 offset 1 +sql select * from $stb where ts > '2018-09-17 09:30:00.000' and ts < '2018-09-17 10:30:00.000' order by ts asc limit 5 offset 1 if $rows != 5 then return -1 endi -if $data01 != 5 then +if $data01 != 3 then return -1 endi -if $data11 != 6 then +if $data11 != 3 then return -1 endi -if $data21 != 7 then +if $data21 != 3 then return -1 endi -if $data31 != 8 then +if $data31 != 3 then return -1 endi -if $data41 != 4 then +if $data41 != 3 then return -1 endi -sql select * from $stb where ts > '2018-09-17 09:30:00.000' and ts < '2018-09-17 10:30:00.000' limit 5 offset 50 +sql select * from $stb where ts > '2018-09-17 09:30:00.000' and ts < '2018-09-17 10:10:00.000' order by ts asc limit 5 offset 50 if $rows != 0 then return -1 endi -sql select * from $stb where ts > '2018-09-17 09:30:00.000' and ts < '2018-09-17 10:30:00.000' limit 5 offset 1 +sql select * from $stb where ts > '2018-09-17 09:30:00.000' and ts < '2018-09-17 10:30:00.000' order by ts asc limit 5 offset 1 if $rows != 5 then return -1 endi -if $data01 != 5 then +if $data01 != 3 then return -1 endi -if $data11 != 6 then +if $data11 != 3 then return -1 endi -if $data21 != 7 then +if $data21 != 3 then return -1 endi -if $data31 != 8 then +if $data31 != 3 then return -1 endi -if $data41 != 4 then +if $data41 != 3 then return -1 endi -sql select max(c1), min(c2), avg(c3), sum(c5), spread(c6), first(c7), last(c8), first(c9) from lm_stb0 where ts >= '2018-09-17 09:00:00.000' and ts <= '2018-09-17 10:30:00.000' limit 1 offset 0; +sql select max(c1), min(c2), avg(c3), sum(c5), spread(c6), first(c7), last(c8), first(c9) from lm_stb0 where ts >= '2018-09-17 09:00:00.000' and ts <= '2018-09-17 10:30:00.009' order by ts asc limit 1 offset 0; if $rows != 1 then return -1 endi @@ -842,9 +837,6 @@ sql select top(c1, 5) from $stb where ts >= $ts0 and ts <= $tsu order by ts desc if $rows != 3 then return -1 endi -if $data00 != @18-09-17 10:30:00.000@ then - return -1 -endi if $data01 != 9 then return -1 endi @@ -853,9 +845,6 @@ sql select top(c1, 5) from $stb where ts >= $ts0 and ts <= $tsu order by ts asc if $rows != 3 then return -1 endi -if $data00 != @18-09-17 10:30:00.000@ then - return -1 -endi if $data01 != 9 then return -1 endi @@ -864,7 +853,7 @@ sql select top(c1, 5) from $stb where ts >= $ts0 and ts <= $tsu group by t1 orde if $rows != 6 then return -1 endi -if $data00 != @18-09-17 10:00:00.000@ then +if $data00 != @18-09-17 10:00:00.008@ then return -1 endi if $data01 != 6 then @@ -873,7 +862,7 @@ endi if $data02 != 8 then return -1 endi -if $data10 != @18-09-17 10:10:00.000@ then +if $data10 != @18-09-17 10:10:00.008@ then return -1 endi if $data11 != 7 then @@ -882,7 +871,7 @@ endi if $data12 != 8 then return -1 endi -if $data20 != @18-09-17 10:20:00.000@ then +if $data20 != @18-09-17 10:20:00.008@ then return -1 endi if $data21 != 8 then @@ -891,7 +880,7 @@ endi if $data22 != 8 then return -1 endi -if $data30 != @18-09-17 10:00:00.000@ then +if $data30 != @18-09-17 10:00:00.007@ then return -1 endi if $data31 != 6 then @@ -900,7 +889,7 @@ endi if $data32 != 7 then return -1 endi -if $data40 != @18-09-17 10:10:00.000@ then +if $data40 != @18-09-17 10:10:00.007@ then return -1 endi if $data41 != 7 then @@ -909,7 +898,7 @@ endi if $data42 != 7 then return -1 endi -if $data50 != @18-09-17 10:20:00.000@ then +if $data50 != @18-09-17 10:20:00.007@ then return -1 endi if $data51 != 8 then @@ -923,7 +912,7 @@ sql select top(c1, 5) from $stb where ts >= $ts0 and ts <= $tsu group by t1 orde if $rows != 6 then return -1 endi -if $data00 != @18-09-17 10:00:00.000@ then +if $data00 != @18-09-17 10:00:00.001@ then return -1 endi if $data01 != 6 then @@ -932,7 +921,7 @@ endi if $data02 != 1 then return -1 endi -if $data10 != @18-09-17 10:10:00.000@ then +if $data10 != @18-09-17 10:10:00.001@ then return -1 endi if $data11 != 7 then @@ -941,7 +930,7 @@ endi if $data12 != 1 then return -1 endi -if $data20 != @18-09-17 10:20:00.000@ then +if $data20 != @18-09-17 10:20:00.001@ then return -1 endi if $data21 != 8 then @@ -950,7 +939,7 @@ endi if $data22 != 1 then return -1 endi -if $data30 != @18-09-17 10:00:00.000@ then +if $data30 != @18-09-17 10:00:00.002@ then return -1 endi if $data31 != 6 then @@ -959,7 +948,7 @@ endi if $data32 != 2 then return -1 endi -if $data40 != @18-09-17 10:10:00.000@ then +if $data40 != @18-09-17 10:10:00.002@ then return -1 endi if $data41 != 7 then @@ -968,7 +957,7 @@ endi if $data42 != 2 then return -1 endi -if $data50 != @18-09-17 10:20:00.000@ then +if $data50 != @18-09-17 10:20:00.002@ then return -1 endi if $data51 != 8 then @@ -982,7 +971,7 @@ sql select top(c1, 5) from $stb where ts >= $ts0 and ts <= $tsu group by t1 orde if $rows != 6 then return -1 endi -if $data00 != @18-09-17 10:20:00.000@ then +if $data00 != @18-09-17 10:20:00.001@ then return -1 endi if $data01 != 8 then @@ -991,7 +980,7 @@ endi if $data02 != 1 then return -1 endi -if $data10 != @18-09-17 10:10:00.000@ then +if $data10 != @18-09-17 10:10:00.001@ then return -1 endi if $data11 != 7 then @@ -1000,7 +989,7 @@ endi if $data12 != 1 then return -1 endi -if $data20 != @18-09-17 10:00:00.000@ then +if $data20 != @18-09-17 10:00:00.001@ then return -1 endi if $data21 != 6 then @@ -1009,7 +998,7 @@ endi if $data22 != 1 then return -1 endi -if $data30 != @18-09-17 10:20:00.000@ then +if $data30 != @18-09-17 10:20:00.002@ then return -1 endi if $data31 != 8 then @@ -1018,7 +1007,7 @@ endi if $data32 != 2 then return -1 endi -if $data40 != @18-09-17 10:10:00.000@ then +if $data40 != @18-09-17 10:10:00.002@ then return -1 endi if $data41 != 7 then @@ -1027,7 +1016,7 @@ endi if $data42 != 2 then return -1 endi -if $data50 != @18-09-17 10:00:00.000@ then +if $data50 != @18-09-17 10:00:00.002@ then return -1 endi if $data51 != 6 then @@ -1052,9 +1041,9 @@ sql select bottom(c1, 5) from $stb where ts >= $ts0 and ts <= $tsu order by ts d if $rows != 3 then return -1 endi -if $data00 != @18-09-17 09:00:00.000@ then - return -1 -endi +#if $data00 != @18-09-17 09:00:00.000@ then +# return -1 +#endi if $data01 != 0 then return -1 endi @@ -1063,9 +1052,9 @@ sql select bottom(c1, 5) from $stb where ts >= $ts0 and ts <= $tsu order by ts a if $rows != 3 then return -1 endi -if $data00 != @18-09-17 09:00:00.000@ then - return -1 -endi +#if $data00 != @18-09-17 09:00:00.000@ then +# return -1 +#endi if $data01 != 0 then return -1 endi @@ -1074,54 +1063,54 @@ sql select bottom(c1, 5) from $stb where ts >= $ts0 and ts <= $tsu group by t1 o if $rows != 6 then return -1 endi -if $data00 != @18-09-17 09:30:00.000@ then - return -1 -endi +#if $data00 != @18-09-17 09:30:00.000@ then +# return -1 +#endi if $data01 != 3 then return -1 endi if $data02 != 8 then return -1 endi -if $data10 != @18-09-17 09:20:00.000@ then - return -1 -endi +#if $data10 != @18-09-17 09:20:00.000@ then +# return -1 +#endi if $data11 != 2 then return -1 endi if $data12 != 8 then return -1 endi -if $data20 != @18-09-17 09:10:00.000@ then - return -1 -endi +#if $data20 != @18-09-17 09:10:00.000@ then +# return -1 +#endi if $data21 != 1 then return -1 endi if $data22 != 8 then return -1 endi -if $data30 != @18-09-17 09:30:00.000@ then - return -1 -endi +#if $data30 != @18-09-17 09:30:00.000@ then +# return -1 +#endi if $data31 != 3 then return -1 endi if $data32 != 7 then return -1 endi -if $data40 != @18-09-17 09:20:00.000@ then - return -1 -endi +#if $data40 != @18-09-17 09:20:00.000@ then +# return -1 +#endi if $data41 != 2 then return -1 endi if $data42 != 7 then return -1 endi -if $data50 != @18-09-17 09:10:00.000@ then - return -1 -endi +#if $data50 != @18-09-17 09:10:00.000@ then +# return -1 +#endi if $data51 != 1 then return -1 endi @@ -1133,54 +1122,54 @@ sql select bottom(c1, 5) from $stb where ts >= $ts0 and ts <= $tsu group by t1 o if $rows != 6 then return -1 endi -if $data00 != @18-09-17 09:30:00.000@ then - return -1 -endi +#if $data00 != @18-09-17 09:30:00.000@ then +# return -1 +#endi if $data01 != 3 then return -1 endi if $data02 != 1 then return -1 endi -if $data10 != @18-09-17 09:20:00.000@ then - return -1 -endi +#if $data10 != @18-09-17 09:20:00.000@ then +# return -1 +#endi if $data11 != 2 then return -1 endi if $data12 != 1 then return -1 endi -if $data20 != @18-09-17 09:10:00.000@ then - return -1 -endi +#if $data20 != @18-09-17 09:10:00.000@ then +# return -1 +#endi if $data21 != 1 then return -1 endi if $data22 != 1 then return -1 endi -if $data30 != @18-09-17 09:30:00.000@ then - return -1 -endi +#if $data30 != @18-09-17 09:30:00.000@ then +# return -1 +#endi if $data31 != 3 then return -1 endi if $data32 != 2 then return -1 endi -if $data40 != @18-09-17 09:20:00.000@ then - return -1 -endi +#if $data40 != @18-09-17 09:20:00.000@ then +# return -1 +#endi if $data41 != 2 then return -1 endi if $data42 != 2 then return -1 endi -if $data50 != @18-09-17 09:10:00.000@ then - return -1 -endi +#if $data50 != @18-09-17 09:10:00.000@ then +# return -1 +#endi if $data51 != 1 then return -1 endi @@ -1192,54 +1181,54 @@ sql select bottom(c1, 5) from $stb where ts >= $ts0 and ts <= $tsu group by t1 o if $rows != 6 then return -1 endi -if $data00 != @18-09-17 09:30:00.000@ then - return -1 -endi +#if $data00 != @18-09-17 09:30:00.000@ then +# return -1 +#endi if $data01 != 3 then return -1 endi if $data02 != 1 then return -1 endi -if $data10 != @18-09-17 09:20:00.000@ then - return -1 -endi +#if $data10 != @18-09-17 09:20:00.000@ then +# return -1 +#endi if $data11 != 2 then return -1 endi if $data12 != 1 then return -1 endi -if $data20 != @18-09-17 09:10:00.000@ then - return -1 -endi +#if $data20 != @18-09-17 09:10:00.000@ then +# return -1 +#endi if $data21 != 1 then return -1 endi if $data22 != 1 then return -1 endi -if $data30 != @18-09-17 09:30:00.000@ then - return -1 -endi +#if $data30 != @18-09-17 09:30:00.000@ then +# return -1 +#endi if $data31 != 3 then return -1 endi if $data32 != 2 then return -1 endi -if $data40 != @18-09-17 09:20:00.000@ then - return -1 -endi +#if $data40 != @18-09-17 09:20:00.000@ then +# return -1 +#endi if $data41 != 2 then return -1 endi if $data42 != 2 then return -1 endi -if $data50 != @18-09-17 09:10:00.000@ then - return -1 -endi +#if $data50 != @18-09-17 09:10:00.000@ then +# return -1 +#endi if $data51 != 1 then return -1 endi From 555125aff50b537ba72adcd54451c1eeacfe7b04 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Sat, 5 Dec 2020 15:43:09 +0800 Subject: [PATCH 51/58] fix windows 32bit compile error --- src/client/src/tscLocalMerge.c | 8 ++++---- src/query/src/qExecutor.c | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/client/src/tscLocalMerge.c b/src/client/src/tscLocalMerge.c index 9fdadfa957..7fc5b8debb 100644 --- a/src/client/src/tscLocalMerge.c +++ b/src/client/src/tscLocalMerge.c @@ -1633,7 +1633,7 @@ void tscInitResObjForLocalQuery(SSqlObj *pObj, int32_t numOfRes, int32_t rowLen) int32_t doArithmeticCalculate(SQueryInfo* pQueryInfo, tFilePage* pOutput, int32_t rowSize, int32_t finalRowSize) { int32_t maxRowSize = MAX(rowSize, finalRowSize); - char* pbuf = calloc(1, pOutput->num * maxRowSize); + char* pbuf = calloc(1, (size_t)(pOutput->num * maxRowSize)); size_t size = tscNumOfFields(pQueryInfo); SArithmeticSupport arithSup = {0}; @@ -1660,16 +1660,16 @@ int32_t doArithmeticCalculate(SQueryInfo* pQueryInfo, tFilePage* pOutput, int32_ tExprTreeCalcTraverse(arithSup.pArithExpr->pExpr, (int32_t) pOutput->num, pbuf + pOutput->num*offset, &arithSup, TSDB_ORDER_ASC, getArithmeticInputSrc); } else { SSqlExpr* pExpr = pSup->pSqlExpr; - memcpy(pbuf + pOutput->num * offset, pExpr->offset * pOutput->num + pOutput->data, pExpr->resBytes * pOutput->num); + memcpy(pbuf + pOutput->num * offset, pExpr->offset * pOutput->num + pOutput->data, (size_t)(pExpr->resBytes * pOutput->num)); } offset += pSup->field.bytes; } - memcpy(pOutput->data, pbuf, pOutput->num * offset); + memcpy(pOutput->data, pbuf, (size_t)(pOutput->num * offset)); tfree(pbuf); tfree(arithSup.data); return offset; -} \ No newline at end of file +} diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 1cde31cfd2..a45c0ac6ef 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -5837,7 +5837,7 @@ static void doSecondaryArithmeticProcess(SQuery* pQuery) { tFilePage **data = calloc(pQuery->numOfExpr2, POINTER_BYTES); for (int32_t i = 0; i < pQuery->numOfExpr2; ++i) { int32_t bytes = pQuery->pExpr2[i].bytes; - data[i] = (tFilePage *)malloc(bytes * pQuery->rec.rows + sizeof(tFilePage)); + data[i] = (tFilePage *)malloc((size_t)(bytes * pQuery->rec.rows) + sizeof(tFilePage)); } arithSup.offset = 0; @@ -5859,7 +5859,7 @@ static void doSecondaryArithmeticProcess(SQuery* pQuery) { for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { if (pSqlFunc->functionId == pQuery->pExpr1[j].base.functionId && pSqlFunc->colInfo.colId == pQuery->pExpr1[j].base.colInfo.colId) { - memcpy(data[i]->data, pQuery->sdata[j]->data, pQuery->pExpr1[j].bytes * pQuery->rec.rows); + memcpy(data[i]->data, pQuery->sdata[j]->data, (size_t)(pQuery->pExpr1[j].bytes * pQuery->rec.rows)); break; } } @@ -5871,7 +5871,7 @@ static void doSecondaryArithmeticProcess(SQuery* pQuery) { } for (int32_t i = 0; i < pQuery->numOfExpr2; ++i) { - memcpy(pQuery->sdata[i]->data, data[i]->data, pQuery->pExpr2[i].bytes * pQuery->rec.rows); + memcpy(pQuery->sdata[i]->data, data[i]->data, (size_t)(pQuery->pExpr2[i].bytes * pQuery->rec.rows)); } for (int32_t i = 0; i < pQuery->numOfExpr2; ++i) { From cf66d1206d5b5677ac98906e4407a1f71157bd30 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sat, 5 Dec 2020 16:33:38 +0800 Subject: [PATCH 52/58] update faq and model-ch. --- documentation20/webdocs/markdowndocs/Model-ch.md | 10 ++++++---- documentation20/webdocs/markdowndocs/faq-ch.md | 14 +++++++++----- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/documentation20/webdocs/markdowndocs/Model-ch.md b/documentation20/webdocs/markdowndocs/Model-ch.md index d698e3daaf..c0b64bde6e 100644 --- a/documentation20/webdocs/markdowndocs/Model-ch.md +++ b/documentation20/webdocs/markdowndocs/Model-ch.md @@ -12,7 +12,7 @@ TDengine采用关系型数据模型，需要建库、建表。因此对于一个 CREATE DATABASE power KEEP 365 DAYS 10 BLOCKS 4; ``` 上述语句将创建一个名为power的库，这个库的数据将保留365天（超过365天将被自动删除），每10天一个数据文件，内存块数为4。详细的语法及参数请见TAOS SQL - + 创建库之后，需要使用SQL命令USE将当前库切换过来，例如： ```cmd @@ -20,7 +20,7 @@ USE power; ``` 就当前连接里操作的库换为power，否则对具体表操作前，需要使用“库名.表名”来指定库的名字。 - + **注意：** - 任何一张表或超级表是属于一个库的，在创建表之前，必须先创建库。 @@ -44,15 +44,17 @@ CREATE TABLE d1001 USING meters TAGS ("Beijing.Chaoyang", 2); ``` 其中d1001是表名，meters是超级表的表名，后面紧跟标签Location的具体标签值”Beijing.Chaoyang"，标签groupId的具体标签值2。虽然在创建表时，需要指定标签值，但可以事后修改。详细细则请见 TAOS SQL。 +**注意：**目前 TDengine 没有从技术层面限制使用一个 database （dbA）的超级表作为模板建立另一个 database （dbB）的子表，后续会禁止这种用法，不建议使用这种方法建表。 + TDengine建议将数据采集点的全局唯一ID作为表名(比如设备序列号）。但对于有的场景，并没有唯一的ID，可以将多个ID组合成一个唯一的ID。不建议将具有唯一性的ID作为标签值。 - + **自动建表**：在某些特殊场景中，用户在写数据时并不确定某个数据采集点的表是否存在，此时可在写入数据时使用自动建表语法来创建不存在的表，若该表已存在则不会建立新表。比如： ```cmd INSERT INTO d1001 USING METERS TAGS ("Beijng.Chaoyang", 2) VALUES (now, 10.2, 219, 0.32); ``` 上述SQL语句将记录(now, 10.2, 219, 0.32) 插入进表d1001。如果表d1001还未创建，则使用超级表meters做模板自动创建，同时打上标签值“Beijing.Chaoyang", 2。 - + ## 多列模型 vs 单列模型 TDengine支持多列模型，只要物理量是一个数据采集点同时采集的（时间戳一致），这些量就可以作为不同列放在一张超级表里。但还有一种极限的设计，单列模型，每个采集的物理量都单独建表，因此每种类型的物理量都单独建立一超级表。比如电流、电压、相位，就建三张超级表。 diff --git a/documentation20/webdocs/markdowndocs/faq-ch.md b/documentation20/webdocs/markdowndocs/faq-ch.md index 42f9761c95..61f70c5962 100644 --- a/documentation20/webdocs/markdowndocs/faq-ch.md +++ b/documentation20/webdocs/markdowndocs/faq-ch.md @@ -36,16 +36,20 @@ 5. ping服务器FQDN，如果没有反应，请检查你的网络，DNS设置，或客户端所在计算机的系统hosts文件 -6. 检查防火墙设置，确认TCP/UDP 端口6030-6042 是打开的 +6. 检查防火墙设置（Ubuntu 使用 ufw status，CentOS 使用 firewall-cmd --list-port），确认TCP/UDP 端口6030-6042 是打开的 7. 对于Linux上的JDBC（ODBC, Python, Go等接口类似）连接, 确保*libtaos.so*在目录*/usr/local/taos/driver*里, 并且*/usr/local/taos/driver*在系统库函数搜索路径*LD_LIBRARY_PATH*里 8. 对于windows上的JDBC, ODBC, Python, Go等连接，确保*C:\TDengine\driver\taos.dll*在你的系统库函数搜索目录里 (建议*taos.dll*放在目录 *C:\Windows\System32*) -9. 如果仍不能排除连接故障，请使用命令行工具nc来分别判断指定端口的TCP和UDP连接是否通畅 - 检查UDP端口连接是否工作：`nc -vuz {hostIP} {port} ` - 检查服务器侧TCP端口连接是否工作：`nc -l {port}` - 检查客户端侧TCP端口连接是否工作：`nc {hostIP} {port}` +9. 如果仍不能排除连接故障 + + * Linux 系统请使用命令行工具nc来分别判断指定端口的TCP和UDP连接是否通畅 + 检查UDP端口连接是否工作：`nc -vuz {hostIP} {port} ` + 检查服务器侧TCP端口连接是否工作：`nc -l {port}` + 检查客户端侧TCP端口连接是否工作：`nc {hostIP} {port}` + + * Windows 系统请使用 PowerShell 命令 Net-TestConnection -ComputerName {fqdn} -Port {port} 检测服务段端口是否访问 10. 也可以使用taos程序内嵌的网络连通检测功能，来验证服务器和客户端之间指定的端口连接是否通畅（包括TCP和UDP）：[TDengine 内嵌网络检测工具使用指南](https://www.taosdata.com/blog/2020/09/08/1816.html)。 From 979d73fbb26ff3eac2a9a55d954dfd6c4a9d3c9c Mon Sep 17 00:00:00 2001 From: stephenkgu Date: Sun, 6 Dec 2020 09:25:49 +0800 Subject: [PATCH 53/58] mnode/table: separate table id prefix from mnodeRetrieveShowTables prep for cross db create table checking in mnodeDoCreateChildTable --- src/common/inc/tname.h | 2 ++ src/common/src/tname.c | 9 ++++++++- src/mnode/src/mnodeTable.c | 4 +--- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/common/inc/tname.h b/src/common/inc/tname.h index 6c48ca72f3..213aef1bd8 100644 --- a/src/common/inc/tname.h +++ b/src/common/inc/tname.h @@ -25,6 +25,8 @@ void extractTableName(const char *tableId, char *name); char* extractDBName(const char *tableId, char *name); +int32_t tableIdPrefix(const char* pDb, char* prefix, int32_t len); + void extractTableNameFromToken(SStrToken *pToken, SStrToken* pTable); SSchema tGetTableNameColumnSchema(); diff --git a/src/common/src/tname.c b/src/common/src/tname.c index bea8c52ef2..b86a0ad1d0 100644 --- a/src/common/src/tname.c +++ b/src/common/src/tname.c @@ -39,6 +39,13 @@ char* extractDBName(const char* tableId, char* name) { return strncpy(name, &tableId[offset1 + 1], len); } +int32_t tableIdPrefix(const char* name, char* prefix, int32_t len) { + tstrncpy(prefix, name, len); + strcat(prefix, TS_PATH_DELIMITER); + + return strlen(prefix); +} + SSchema tGetTableNameColumnSchema() { SSchema s = {0}; s.bytes = TSDB_TABLE_NAME_LEN - 1 + VARSTR_HEADER_SIZE; @@ -198,4 +205,4 @@ SSchema tscGetTbnameColumnSchema() { strcpy(s.name, TSQL_TBNAME_L); return s; -} \ No newline at end of file +} diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index 2149cb12c0..aa77f5531d 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -2629,9 +2629,7 @@ static int32_t mnodeRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows SPatternCompareInfo info = PATTERN_COMPARE_INFO_INITIALIZER; char prefix[64] = {0}; - tstrncpy(prefix, pDb->name, 64); - strcat(prefix, TS_PATH_DELIMITER); - int32_t prefixLen = strlen(prefix); + int32_t prefixLen = tableIdPrefix(pDb->name, prefix, 64); char* pattern = NULL; if (pShow->payloadLen > 0) { From 1c33675b016aaf15b3d2f7d2c3441d936788fb52 Mon Sep 17 00:00:00 2001 From: stephenkgu Date: Sun, 6 Dec 2020 10:18:39 +0800 Subject: [PATCH 54/58] [TD-2342]: disallow cross db create child table --- src/mnode/src/mnodeTable.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index aa77f5531d..1505f16812 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -1734,6 +1734,16 @@ static int32_t mnodeDoCreateChildTable(SMnodeMsg *pMsg, int32_t tid) { if (pTable->info.type == TSDB_CHILD_TABLE) { STagData *pTagData = (STagData *)pCreate->schema; // it is a tag key + + char prefix[64] = {0}; + int32_t prefixLen = tableIdPrefix(pMsg->pDb->name, prefix, 64); + if (0 != strncasecmp(prefix, pTagData->name, prefixLen)) { + mError("msg:%p, app:%p table:%s, corresponding super table:%s not in this db", pMsg, pMsg->rpcMsg.ahandle, + pCreate->tableId, pTagData->name); + mnodeDestroyChildTable(pTable); + return TSDB_CODE_MND_INVALID_TABLE_NAME; + } + if (pMsg->pSTable == NULL) pMsg->pSTable = mnodeGetSuperTable(pTagData->name); if (pMsg->pSTable == NULL) { mError("msg:%p, app:%p table:%s, corresponding super table:%s does not exist", pMsg, pMsg->rpcMsg.ahandle, From 4ee6c822cf81fab09a680dbc4b4bbad12bde4638 Mon Sep 17 00:00:00 2001 From: stephenkgu Date: Sun, 6 Dec 2020 12:04:36 +0800 Subject: [PATCH 55/58] use size_t instead of int32_t to avoid possible loss of data --- src/common/inc/tname.h | 2 +- src/common/src/tname.c | 2 +- src/mnode/src/mnodeTable.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/common/inc/tname.h b/src/common/inc/tname.h index 213aef1bd8..5c2948db76 100644 --- a/src/common/inc/tname.h +++ b/src/common/inc/tname.h @@ -25,7 +25,7 @@ void extractTableName(const char *tableId, char *name); char* extractDBName(const char *tableId, char *name); -int32_t tableIdPrefix(const char* pDb, char* prefix, int32_t len); +size_t tableIdPrefix(const char* pDb, char* prefix, int32_t len); void extractTableNameFromToken(SStrToken *pToken, SStrToken* pTable); diff --git a/src/common/src/tname.c b/src/common/src/tname.c index b86a0ad1d0..5c351edf48 100644 --- a/src/common/src/tname.c +++ b/src/common/src/tname.c @@ -39,7 +39,7 @@ char* extractDBName(const char* tableId, char* name) { return strncpy(name, &tableId[offset1 + 1], len); } -int32_t tableIdPrefix(const char* name, char* prefix, int32_t len) { +size_t tableIdPrefix(const char* name, char* prefix, int32_t len) { tstrncpy(prefix, name, len); strcat(prefix, TS_PATH_DELIMITER); diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index 1505f16812..ad50b4929c 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -1736,7 +1736,7 @@ static int32_t mnodeDoCreateChildTable(SMnodeMsg *pMsg, int32_t tid) { STagData *pTagData = (STagData *)pCreate->schema; // it is a tag key char prefix[64] = {0}; - int32_t prefixLen = tableIdPrefix(pMsg->pDb->name, prefix, 64); + size_t prefixLen = tableIdPrefix(pMsg->pDb->name, prefix, 64); if (0 != strncasecmp(prefix, pTagData->name, prefixLen)) { mError("msg:%p, app:%p table:%s, corresponding super table:%s not in this db", pMsg, pMsg->rpcMsg.ahandle, pCreate->tableId, pTagData->name); From 470442ccdc20fc922e61a8ed090eb43ac7b45d0d Mon Sep 17 00:00:00 2001 From: stephenkgu Date: Sun, 6 Dec 2020 12:27:10 +0800 Subject: [PATCH 56/58] use TSDB_CODE_TDB_INVALID_CREATE_TB_MSG instead of TSDB_CODE_MND_INVALID_TABLE_NAME --- src/common/inc/tname.h | 2 +- src/mnode/src/mnodeTable.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/common/inc/tname.h b/src/common/inc/tname.h index 5c2948db76..9e0093ebfe 100644 --- a/src/common/inc/tname.h +++ b/src/common/inc/tname.h @@ -25,7 +25,7 @@ void extractTableName(const char *tableId, char *name); char* extractDBName(const char *tableId, char *name); -size_t tableIdPrefix(const char* pDb, char* prefix, int32_t len); +size_t tableIdPrefix(const char* name, char* prefix, int32_t len); void extractTableNameFromToken(SStrToken *pToken, SStrToken* pTable); diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index ad50b4929c..ff81c37de7 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -1741,7 +1741,7 @@ static int32_t mnodeDoCreateChildTable(SMnodeMsg *pMsg, int32_t tid) { mError("msg:%p, app:%p table:%s, corresponding super table:%s not in this db", pMsg, pMsg->rpcMsg.ahandle, pCreate->tableId, pTagData->name); mnodeDestroyChildTable(pTable); - return TSDB_CODE_MND_INVALID_TABLE_NAME; + return TSDB_CODE_TDB_INVALID_CREATE_TB_MSG; } if (pMsg->pSTable == NULL) pMsg->pSTable = mnodeGetSuperTable(pTagData->name); From 9abff3177c8b8015aeb406cf3a5ba66356fdd204 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 6 Dec 2020 06:13:27 +0000 Subject: [PATCH 57/58] TD-2347 --- src/dnode/src/dnodeEps.c | 7 +++++++ src/dnode/src/dnodeMInfos.c | 27 +++++++++++++++++++-------- src/dnode/src/dnodeMain.c | 2 +- src/dnode/src/dnodeMgmt.c | 7 ++++++- src/sync/src/syncMain.c | 2 +- src/vnode/src/vnodeCfg.c | 5 ++--- src/vnode/src/vnodeMain.c | 3 +-- 7 files changed, 37 insertions(+), 16 deletions(-) diff --git a/src/dnode/src/dnodeEps.c b/src/dnode/src/dnodeEps.c index 83f294e05e..5fb3828a84 100644 --- a/src/dnode/src/dnodeEps.c +++ b/src/dnode/src/dnodeEps.c @@ -236,7 +236,14 @@ PRASE_EPS_OVER: dnodeResetEps(eps); if (eps) free(eps); +#if 0 dnodeUpdateEp(dnodeGetDnodeId(), tsLocalEp, tsLocalFqdn, &tsServerPort); +#else + if (dnodeCheckEpChanged(dnodeGetDnodeId(), tsLocalEp)) { + dError("dnode:%d, localEp is changed to %s in dnodeEps.json and need reconfigured", dnodeGetDnodeId(), tsLocalEp); + return -1; + } +#endif terrno = 0; return 0; diff --git a/src/dnode/src/dnodeMInfos.c b/src/dnode/src/dnodeMInfos.c index 162de2243e..a88c3528f7 100644 --- a/src/dnode/src/dnodeMInfos.c +++ b/src/dnode/src/dnodeMInfos.c @@ -157,12 +157,13 @@ static void dnodeResetMInfos(SMInfos *pMinfos) { } static int32_t dnodeReadMInfos() { - int32_t len = 0; - int32_t maxLen = 2000; - char * content = calloc(1, maxLen + 1); - cJSON * root = NULL; - FILE * fp = NULL; - SMInfos minfos = {0}; + int32_t len = 0; + int32_t maxLen = 2000; + char * content = calloc(1, maxLen + 1); + cJSON * root = NULL; + FILE * fp = NULL; + SMInfos minfos = {0}; + bool nodeChanged = false; char file[TSDB_FILENAME_LEN + 20] = {0}; sprintf(file, "%s/mnodeEpSet.json", tsDnodeDir); @@ -221,14 +222,19 @@ static int32_t dnodeReadMInfos() { dError("failed to read mnodeEpSet.json, nodeId not found"); goto PARSE_MINFOS_OVER; } - minfos.mnodeInfos[i].mnodeId = nodeId->valueint; cJSON *nodeEp = cJSON_GetObjectItem(nodeInfo, "nodeEp"); if (!nodeEp || nodeEp->type != cJSON_String || nodeEp->valuestring == NULL) { dError("failed to read mnodeEpSet.json, nodeName not found"); goto PARSE_MINFOS_OVER; } - strncpy(minfos.mnodeInfos[i].mnodeEp, nodeEp->valuestring, TSDB_EP_LEN); + + SMInfo *pMinfo = &minfos.mnodeInfos[i]; + pMinfo->mnodeId = nodeId->valueint; + tstrncpy(pMinfo->mnodeEp, nodeEp->valuestring, TSDB_EP_LEN); + + bool changed = dnodeCheckEpChanged(pMinfo->mnodeId, pMinfo->mnodeEp); + if (changed) nodeChanged = changed; } dInfo("read file %s successed", file); @@ -245,6 +251,11 @@ PARSE_MINFOS_OVER: dnodeUpdateEp(mInfo->mnodeId, mInfo->mnodeEp, NULL, NULL); } dnodeResetMInfos(&minfos); + + if (nodeChanged) { + dnodeWriteMInfos(); + } + return 0; } diff --git a/src/dnode/src/dnodeMain.c b/src/dnode/src/dnodeMain.c index 9f52dbd331..bd065d0d0e 100644 --- a/src/dnode/src/dnodeMain.c +++ b/src/dnode/src/dnodeMain.c @@ -58,10 +58,10 @@ typedef struct { static const SDnodeComponent tsDnodeComponents[] = { {"tfile", tfInit, tfCleanup}, {"rpc", rpcInit, rpcCleanup}, + {"globalcfg" ,taosCheckGlobalCfg, NULL}, {"storage", dnodeInitStorage, dnodeCleanupStorage}, {"dnodecfg", dnodeInitCfg, dnodeCleanupCfg}, {"dnodeeps", dnodeInitEps, dnodeCleanupEps}, - {"globalcfg" ,taosCheckGlobalCfg, NULL}, {"mnodeinfos",dnodeInitMInfos, dnodeCleanupMInfos}, {"wal", walInit, walCleanUp}, {"check", dnodeInitCheck, dnodeCleanupCheck}, // NOTES: dnodeInitCheck must be behind the dnodeinitStorage component !!! diff --git a/src/dnode/src/dnodeMgmt.c b/src/dnode/src/dnodeMgmt.c index 15378c77c1..cce593727d 100644 --- a/src/dnode/src/dnodeMgmt.c +++ b/src/dnode/src/dnodeMgmt.c @@ -339,7 +339,12 @@ static int32_t dnodeOpenVnodes() { } free(threads); - dInfo("there are total vnodes:%d, openned:%d failed:%d", numOfVnodes, openVnodes, failedVnodes); + dInfo("there are total vnodes:%d, openned:%d", numOfVnodes, openVnodes); + + if (failedVnodes != 0) { + dError("there are total vnodes:%d, failed:%d", numOfVnodes, failedVnodes); + return -1; + } return TSDB_CODE_SUCCESS; } diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c index b73ca27ce9..c86265d556 100644 --- a/src/sync/src/syncMain.c +++ b/src/sync/src/syncMain.c @@ -226,7 +226,7 @@ int64_t syncStart(const SSyncInfo *pInfo) { } if (pNode->selfIndex < 0) { - sInfo("vgId:%d, this node is not configured", pNode->vgId); + sError("vgId:%d, this node is not configured", pNode->vgId); terrno = TSDB_CODE_SYN_INVALID_CONFIG; syncStop(pNode->rid); return -1; diff --git a/src/vnode/src/vnodeCfg.c b/src/vnode/src/vnodeCfg.c index 2d56157328..a79fca9ebb 100644 --- a/src/vnode/src/vnodeCfg.c +++ b/src/vnode/src/vnodeCfg.c @@ -242,9 +242,8 @@ int32_t vnodeReadCfg(SVnodeObj *pVnode) { } tstrncpy(node->nodeEp, nodeEp->valuestring, TSDB_EP_LEN); - if (!nodeChanged) { - nodeChanged = dnodeCheckEpChanged(node->nodeId, node->nodeEp); - } + bool changed = dnodeCheckEpChanged(node->nodeId, node->nodeEp); + if (changed) nodeChanged = changed; } ret = TSDB_CODE_SUCCESS; diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index b516c9d90e..b6817fcbd0 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -362,9 +362,8 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) { pVnode->role = TAOS_SYNC_ROLE_MASTER; #else if (pVnode->sync <= 0) { - vError("vgId:%d, failed to open sync module, replica:%d reason:%s", pVnode->vgId, pVnode->syncCfg.replica, + vError("vgId:%d, failed to open sync, replica:%d reason:%s", pVnode->vgId, pVnode->syncCfg.replica, tstrerror(terrno)); - vnodeRelease(pVnode); vnodeCleanUp(pVnode); return terrno; } From 71746e4616a40356a81dbc62d81fc47832db7e87 Mon Sep 17 00:00:00 2001 From: stephenkgu Date: Sun, 6 Dec 2020 14:28:12 +0800 Subject: [PATCH 58/58] [TD-2304]: make *Invalid timestamp* error more clear --- src/inc/taoserror.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index a720b68e59..bc7be3511f 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -64,7 +64,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_RPC_INVALID_TRAN_ID, 0, 0x000F, "Invalid tr TAOS_DEFINE_ERROR(TSDB_CODE_RPC_INVALID_SESSION_ID, 0, 0x0010, "Invalid session id") TAOS_DEFINE_ERROR(TSDB_CODE_RPC_INVALID_MSG_TYPE, 0, 0x0011, "Invalid message type") TAOS_DEFINE_ERROR(TSDB_CODE_RPC_INVALID_RESPONSE_TYPE, 0, 0x0012, "Invalid response type") -TAOS_DEFINE_ERROR(TSDB_CODE_RPC_INVALID_TIME_STAMP, 0, 0x0013, "Invalid timestamp") +TAOS_DEFINE_ERROR(TSDB_CODE_RPC_INVALID_TIME_STAMP, 0, 0x0013, "Client and server's time is not synchronized") TAOS_DEFINE_ERROR(TSDB_CODE_APP_NOT_READY, 0, 0x0014, "Database not ready") TAOS_DEFINE_ERROR(TSDB_CODE_RPC_FQDN_ERROR, 0, 0x0015, "Unable to resolve FQDN")