From 47a77371eea1d85518bbe94b6e04b405c1b5a1f3 Mon Sep 17 00:00:00 2001 From: freemine Date: Sun, 4 Oct 2020 14:54:31 +0800 Subject: [PATCH 01/51] prefer named-type rather than void-type to reduce typos --- src/client/inc/tscSubquery.h | 2 +- src/client/inc/tsclient.h | 12 +++--- src/client/src/tscLocal.c | 2 +- src/client/src/tscProfile.c | 4 +- src/client/src/tscServer.c | 2 +- src/client/src/tscSql.c | 23 +++++----- src/client/src/tscSubquery.c | 6 +-- src/inc/taos.h | 20 ++++++--- src/kit/shell/inc/shell.h | 4 +- src/kit/shell/src/shellEngine.c | 6 +-- src/kit/shell/src/shellImport.c | 4 +- src/plugins/http/inc/httpInt.h | 6 +-- src/plugins/http/src/httpGcJson.c | 2 +- src/plugins/http/src/httpRestJson.c | 4 +- src/plugins/http/src/httpSql.c | 2 - tests/comparisonTest/tdengine/tdengineTest.c | 45 ++++++++++++++------ tests/examples/c/demo.c | 20 ++++++--- 17 files changed, 101 insertions(+), 63 deletions(-) diff --git a/src/client/inc/tscSubquery.h b/src/client/inc/tscSubquery.h index 07e0580397..ae7f4a2f0c 100644 --- a/src/client/inc/tscSubquery.h +++ b/src/client/inc/tscSubquery.h @@ -39,7 +39,7 @@ int32_t tscHandleMultivnodeInsert(SSqlObj *pSql); int32_t tscHandleInsertRetry(SSqlObj* pSql); void tscBuildResFromSubqueries(SSqlObj *pSql); -void **doSetResultRowData(SSqlObj *pSql, bool finalResult); +TAOS_ROW doSetResultRowData(SSqlObj *pSql, bool finalResult); #ifdef __cplusplus } diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index 046bea2c27..3db4a97974 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -306,7 +306,7 @@ typedef struct { int32_t numOfGroups; SResRec * pGroupRec; char * data; - void ** tsrow; + TAOS_ROW tsrow; int32_t* length; // length for each field for current row char ** buffer; // Buffer used to put multibytes encoded using unicode (wchar_t) SColumnIndex * pColumnIndex; @@ -439,7 +439,7 @@ void tscFreeSqlObjInCache(void *pSql); void tscCloseTscObj(STscObj *pObj); TAOS *taos_connect_a(char *ip, char *user, char *pass, char *db, uint16_t port, void (*fp)(void *, TAOS_RES *, int), - void *param, void **taos); + void *param, TAOS **taos); void waitForQueryRsp(void *param, TAOS_RES *tres, int code) ; void doAsyncQuery(STscObj *pObj, SSqlObj *pSql, void (*fp)(), void *param, const char *sqlstr, size_t sqlLen); @@ -471,11 +471,11 @@ static FORCE_INLINE void tscGetResultColumnChr(SSqlRes* pRes, SFieldInfo* pField if (type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_BINARY) { pData = pInfo->pSqlExpr->param[1].pz; pRes->length[columnIndex] = pInfo->pSqlExpr->param[1].nLen; - pRes->tsrow[columnIndex] = (pInfo->pSqlExpr->param[1].nType == TSDB_DATA_TYPE_NULL) ? NULL : pData; + pRes->tsrow[columnIndex] = (pInfo->pSqlExpr->param[1].nType == TSDB_DATA_TYPE_NULL) ? NULL : (unsigned char*)pData; } else { assert(bytes == tDataTypeDesc[type].nSize); - pRes->tsrow[columnIndex] = isNull(pData, type) ? NULL : &pInfo->pSqlExpr->param[1].i64Key; + pRes->tsrow[columnIndex] = isNull(pData, type) ? NULL : (unsigned char*)&pInfo->pSqlExpr->param[1].i64Key; pRes->length[columnIndex] = bytes; } } else { @@ -483,7 +483,7 @@ static FORCE_INLINE void tscGetResultColumnChr(SSqlRes* pRes, SFieldInfo* pField int32_t realLen = varDataLen(pData); assert(realLen <= bytes - VARSTR_HEADER_SIZE); - pRes->tsrow[columnIndex] = (isNull(pData, type)) ? NULL : ((tstr *)pData)->data; + pRes->tsrow[columnIndex] = (isNull(pData, type)) ? NULL : (unsigned char*)((tstr *)pData)->data; if (realLen < pInfo->pSqlExpr->resBytes - VARSTR_HEADER_SIZE) { // todo refactor *(pData + realLen + VARSTR_HEADER_SIZE) = 0; } @@ -492,7 +492,7 @@ static FORCE_INLINE void tscGetResultColumnChr(SSqlRes* pRes, SFieldInfo* pField } else { assert(bytes == tDataTypeDesc[type].nSize); - pRes->tsrow[columnIndex] = isNull(pData, type) ? NULL : pData; + pRes->tsrow[columnIndex] = isNull(pData, type) ? NULL : (unsigned char*)pData; pRes->length[columnIndex] = bytes; } } diff --git a/src/client/src/tscLocal.c b/src/client/src/tscLocal.c index a15673bdb1..929b9ffe68 100644 --- a/src/client/src/tscLocal.c +++ b/src/client/src/tscLocal.c @@ -296,7 +296,7 @@ static int32_t tscProcessDescribeTable(SSqlObj *pSql) { return tscSetValueToResObj(pSql, rowLen); } static int32_t tscGetNthFieldResult(TAOS_ROW row, TAOS_FIELD* fields, int *lengths, int idx, char *result) { - const char *val = row[idx]; + const char *val = (const char*)row[idx]; if (val == NULL) { sprintf(result, "%s", TSDB_DATA_NULL_STR); return -1; diff --git a/src/client/src/tscProfile.c b/src/client/src/tscProfile.c index f2f997a594..5b3876d3dc 100644 --- a/src/client/src/tscProfile.c +++ b/src/client/src/tscProfile.c @@ -20,8 +20,10 @@ #include "tutil.h" #include "taosmsg.h" +#include "taos.h" + void tscSaveSlowQueryFp(void *handle, void *tmrId); -void *tscSlowQueryConn = NULL; +TAOS *tscSlowQueryConn = NULL; bool tscSlowQueryConnInitialized = false; void tscInitConnCb(void *param, TAOS_RES *result, int code) { diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index d3c0201169..d25f4b4a95 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -1392,7 +1392,7 @@ static int tscSetResultPointer(SQueryInfo *pQueryInfo, SSqlRes *pRes) { for (int i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { int16_t offset = tscFieldInfoGetOffset(pQueryInfo, i); - pRes->tsrow[i] = ((char*) pRes->data + offset * pRes->numOfRows); + pRes->tsrow[i] = (unsigned char*)((char*) pRes->data + offset * pRes->numOfRows); } return 0; diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index ac7081ba70..525333454f 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -49,8 +49,8 @@ static bool validPassword(const char* passwd) { return validImpl(passwd, TSDB_PASSWORD_LEN - 1); } -SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pass, const char *auth, const char *db, - uint16_t port, void (*fp)(void *, TAOS_RES *, int), void *param, void **taos) { +static SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pass, const char *auth, const char *db, + uint16_t port, void (*fp)(void *, TAOS_RES *, int), void *param, TAOS **taos) { taos_init(); if (!validUserName(user)) { @@ -172,7 +172,7 @@ static void syncConnCallback(void *param, TAOS_RES *tres, int code) { TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, const char *auth, const char *db, uint16_t port) { STscObj *pObj = NULL; - SSqlObj *pSql = taosConnectImpl(ip, user, pass, auth, db, port, syncConnCallback, NULL, (void **)&pObj); + SSqlObj *pSql = taosConnectImpl(ip, user, pass, auth, db, port, syncConnCallback, NULL, &pObj); if (pSql != NULL) { pSql->fp = syncConnCallback; pSql->param = pSql; @@ -241,16 +241,19 @@ static void asyncConnCallback(void *param, TAOS_RES *tres, int code) { } TAOS *taos_connect_a(char *ip, char *user, char *pass, char *db, uint16_t port, void (*fp)(void *, TAOS_RES *, int), - void *param, void **taos) { - SSqlObj* pSql = taosConnectImpl(ip, user, pass, NULL, db, port, asyncConnCallback, param, taos); + void *param, TAOS **taos) { + STscObj *pObj = NULL; + SSqlObj* pSql = taosConnectImpl(ip, user, pass, NULL, db, port, asyncConnCallback, param, &pObj); if (pSql == NULL) { return NULL; } - + + if (taos) *taos = pObj; + pSql->fetchFp = fp; pSql->res.code = tscProcessSql(pSql); tscDebug("%p DB async connection is opening", taos); - return taos; + return pObj; } void taos_close(TAOS *taos) { @@ -812,7 +815,7 @@ int taos_validate_sql(TAOS *taos, const char *sql) { if (pSql->sqlstr == NULL) { pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; tscError("%p failed to malloc sql string buffer", pSql); - tscDebug("%p Valid SQL result:%d, %s pObj:%p", pSql, pRes->code, taos_errstr(taos), pObj); + tscDebug("%p Valid SQL result:%d, %s pObj:%p", pSql, pRes->code, taos_errstr(pSql), pObj); taosTFree(pSql); return pRes->code; } @@ -835,7 +838,7 @@ int taos_validate_sql(TAOS *taos, const char *sql) { } if (code != TSDB_CODE_SUCCESS) { - tscDebug("%p Valid SQL result:%d, %s pObj:%p", pSql, code, taos_errstr(taos), pObj); + tscDebug("%p Valid SQL result:%d, %s pObj:%p", pSql, code, taos_errstr(pSql), pObj); } taos_free_result(pSql); @@ -979,7 +982,7 @@ int taos_load_table_info(TAOS *taos, const char *tableNameList) { tscDoQuery(pSql); - tscDebug("%p load multi table meta result:%d %s pObj:%p", pSql, pRes->code, taos_errstr(taos), pObj); + tscDebug("%p load multi table meta result:%d %s pObj:%p", pSql, pRes->code, taos_errstr(pSql), pObj); if ((code = pRes->code) != TSDB_CODE_SUCCESS) { tscFreeSqlObj(pSql); } diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 390c3ca92d..1527a82d68 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -2141,7 +2141,7 @@ static void transferNcharData(SSqlObj *pSql, int32_t columnIndex, TAOS_FIELD *pF int32_t length = taosUcs4ToMbs(pRes->tsrow[columnIndex], pRes->length[columnIndex], pRes->buffer[columnIndex]); if ( length >= 0 ) { - pRes->tsrow[columnIndex] = pRes->buffer[columnIndex]; + pRes->tsrow[columnIndex] = (unsigned char*)pRes->buffer[columnIndex]; pRes->length[columnIndex] = length; } else { tscError("%p charset:%s to %s. val:%s convert failed.", pSql, DEFAULT_UNICODE_ENCODEC, tsCharset, (char*)pRes->tsrow[columnIndex]); @@ -2169,7 +2169,7 @@ static char *getArithemicInputSrc(void *param, const char *name, int32_t colId) return pSupport->data[index] + pSupport->offset * pExpr->resBytes; } -void **doSetResultRowData(SSqlObj *pSql, bool finalResult) { +TAOS_ROW doSetResultRowData(SSqlObj *pSql, bool finalResult) { SSqlCmd *pCmd = &pSql->cmd; SSqlRes *pRes = &pSql->res; @@ -2222,7 +2222,7 @@ void **doSetResultRowData(SSqlObj *pSql, bool finalResult) { tExprTreeCalcTraverse(pRes->pArithSup->pArithExpr->pExpr, 1, pRes->buffer[i], pRes->pArithSup, TSDB_ORDER_ASC, getArithemicInputSrc); - pRes->tsrow[i] = pRes->buffer[i]; + pRes->tsrow[i] = (unsigned char*)pRes->buffer[i]; } } diff --git a/src/inc/taos.h b/src/inc/taos.h index 7e8f174b7c..9ea1d268ea 100644 --- a/src/inc/taos.h +++ b/src/inc/taos.h @@ -22,12 +22,20 @@ extern "C" { #endif -typedef void TAOS; -typedef void** TAOS_ROW; -typedef void TAOS_RES; -typedef void TAOS_SUB; -typedef void TAOS_STREAM; -typedef void TAOS_STMT; +typedef struct STscObj TAOS; +typedef struct STscStmt TAOS_STMT; +typedef struct SSqlObj TAOS_RES; +typedef struct SSqlStream TAOS_STREAM; +typedef struct SSub TAOS_SUB; +typedef unsigned char** TAOS_ROW; + + +// typedef void TAOS; +// typedef void** TAOS_ROW; +// typedef void TAOS_RES; +// typedef void TAOS_SUB; +// typedef void TAOS_STREAM; +// typedef void TAOS_STMT; // Data type definition #define TSDB_DATA_TYPE_NULL 0 // 1 bytes diff --git a/src/kit/shell/inc/shell.h b/src/kit/shell/inc/shell.h index dd62df170a..da2824d806 100644 --- a/src/kit/shell/inc/shell.h +++ b/src/kit/shell/inc/shell.h @@ -59,7 +59,7 @@ typedef struct SShellArguments { extern void shellParseArgument(int argc, char* argv[], SShellArguments* arguments); extern TAOS* shellInit(SShellArguments* args); extern void* shellLoopQuery(void* arg); -extern void taos_error(TAOS* con); +extern void taos_error(TAOS_RES* tres); extern int regex_match(const char* s, const char* reg, int cflags); void shellReadCommand(TAOS* con, char command[]); int32_t shellRunCommand(TAOS* con, char* command); @@ -71,7 +71,7 @@ void source_dir(TAOS* con, SShellArguments* args); void get_history_path(char* history); void cleanup_handler(void* arg); void exitShell(); -int shellDumpResult(TAOS* con, char* fname, int* error_no, bool printMode); +int shellDumpResult(TAOS_RES* con, char* fname, int* error_no, bool printMode); void shellGetGrantInfo(void *con); int isCommentLine(char *line); diff --git a/src/kit/shell/src/shellEngine.c b/src/kit/shell/src/shellEngine.c index d5e826fbaa..4c4d8913bb 100644 --- a/src/kit/shell/src/shellEngine.c +++ b/src/kit/shell/src/shellEngine.c @@ -495,7 +495,7 @@ static int dumpResultToFile(const char* fname, TAOS_RES* tres) { if (i > 0) { fputc(',', fp); } - dumpFieldToFile(fp, row[i], fields +i, length[i], precision); + dumpFieldToFile(fp, (const char*)row[i], fields +i, length[i], precision); } fputc('\n', fp); @@ -621,7 +621,7 @@ static int verticalPrintResult(TAOS_RES* tres) { int padding = (int)(maxColNameLen - strlen(field->name)); printf("%*.s%s: ", padding, " ", field->name); - printField(row[i], field, 0, length[i], precision); + printField((const char*)row[i], field, 0, length[i], precision); putchar('\n'); } @@ -722,7 +722,7 @@ static int horizontalPrintResult(TAOS_RES* tres) { int32_t* length = taos_fetch_lengths(tres); for (int i = 0; i < num_fields; i++) { putchar(' '); - printField(row[i], fields + i, width[i], length[i], precision); + printField((const char*)row[i], fields + i, width[i], length[i], precision); putchar(' '); putchar('|'); } diff --git a/src/kit/shell/src/shellImport.c b/src/kit/shell/src/shellImport.c index ee0a90757b..af61995c61 100644 --- a/src/kit/shell/src/shellImport.c +++ b/src/kit/shell/src/shellImport.c @@ -204,7 +204,7 @@ static void shellSourceFile(TAOS *con, char *fptr) { int32_t code = taos_errno(pSql); if (code != 0) { - fprintf(stderr, "DB error: %s: %s (%d)\n", taos_errstr(con), fname, lineNo); + fprintf(stderr, "DB error: %s: %s (%d)\n", taos_errstr(pSql), fname, lineNo); } /* free local resouce: allocated memory/metric-meta refcnt */ @@ -243,7 +243,7 @@ static void shellRunImportThreads(SShellArguments* args) pThread->totalThreads = args->threadNum; pThread->taos = taos_connect(args->host, args->user, args->password, args->database, tsDnodeShellPort); if (pThread->taos == NULL) { - fprintf(stderr, "ERROR: thread:%d failed connect to TDengine, error:%s\n", pThread->threadIndex, taos_errstr(pThread->taos)); + fprintf(stderr, "ERROR: thread:%d failed connect to TDengine, error:%s\n", pThread->threadIndex, "null taos"/*taos_errstr(pThread->taos)*/); exit(0); } diff --git a/src/plugins/http/inc/httpInt.h b/src/plugins/http/inc/httpInt.h index 36a022159f..affc0e838e 100644 --- a/src/plugins/http/inc/httpInt.h +++ b/src/plugins/http/inc/httpInt.h @@ -122,9 +122,9 @@ typedef struct { } HttpDecodeMethod; typedef struct { - void (*startJsonFp)(struct HttpContext *pContext, HttpSqlCmd *cmd, void *result); + void (*startJsonFp)(struct HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result); void (*stopJsonFp)(struct HttpContext *pContext, HttpSqlCmd *cmd); - bool (*buildQueryJsonFp)(struct HttpContext *pContext, HttpSqlCmd *cmd, void *result, int numOfRows); + bool (*buildQueryJsonFp)(struct HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, int numOfRows); void (*buildAffectRowJsonFp)(struct HttpContext *pContext, HttpSqlCmd *cmd, int affectRows); void (*initJsonFp)(struct HttpContext *pContext); void (*cleanJsonFp)(struct HttpContext *pContext); @@ -148,7 +148,7 @@ typedef struct HttpContext { char ipstr[22]; char user[TSDB_USER_LEN]; // parsed from auth token or login message char pass[TSDB_PASSWORD_LEN]; - void * taos; + TAOS * taos; void * ppContext; HttpSession *session; z_stream gzipStream; diff --git a/src/plugins/http/src/httpGcJson.c b/src/plugins/http/src/httpGcJson.c index 80e4ae3bc2..2c9eca11de 100644 --- a/src/plugins/http/src/httpGcJson.c +++ b/src/plugins/http/src/httpGcJson.c @@ -217,7 +217,7 @@ bool gcBuildQueryJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, break; case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_NCHAR: - httpJsonStringForTransMean(jsonBuf, row[i], fields[i].bytes); + httpJsonStringForTransMean(jsonBuf, (char*)row[i], fields[i].bytes); break; case TSDB_DATA_TYPE_TIMESTAMP: if (precision == TSDB_TIME_PRECISION_MILLI) { //ms diff --git a/src/plugins/http/src/httpRestJson.c b/src/plugins/http/src/httpRestJson.c index 954678c24c..a5b156bffc 100644 --- a/src/plugins/http/src/httpRestJson.c +++ b/src/plugins/http/src/httpRestJson.c @@ -131,7 +131,7 @@ bool restBuildSqlJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, break; case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_NCHAR: - httpJsonStringForTransMean(jsonBuf, row[i], length[i]); + httpJsonStringForTransMean(jsonBuf, (char*)row[i], length[i]); break; case TSDB_DATA_TYPE_TIMESTAMP: if (timestampFormat == REST_TIMESTAMP_FMT_LOCAL_STRING) { @@ -195,4 +195,4 @@ void restStopSqlJson(HttpContext *pContext, HttpSqlCmd *cmd) { httpJsonToken(jsonBuf, JsonObjEnd); httpWriteJsonBufEnd(jsonBuf); -} \ No newline at end of file +} diff --git a/src/plugins/http/src/httpSql.c b/src/plugins/http/src/httpSql.c index 883fa574ff..70d644146c 100644 --- a/src/plugins/http/src/httpSql.c +++ b/src/plugins/http/src/httpSql.c @@ -27,8 +27,6 @@ #include "httpSession.h" #include "httpQueue.h" -void *taos_connect_a(char *ip, char *user, char *pass, char *db, uint16_t port, void (*fp)(void *, TAOS_RES *, int), - void *param, void **taos); void httpProcessMultiSql(HttpContext *pContext); void httpProcessMultiSqlRetrieveCallBack(void *param, TAOS_RES *result, int numOfRows); diff --git a/tests/comparisonTest/tdengine/tdengineTest.c b/tests/comparisonTest/tdengine/tdengineTest.c index 237403f525..1298aa8323 100644 --- a/tests/comparisonTest/tdengine/tdengineTest.c +++ b/tests/comparisonTest/tdengine/tdengineTest.c @@ -108,9 +108,9 @@ void parseArg(int argc, char *argv[]) { } } -void taos_error(TAOS *con) { - printf("TDengine error: %s\n", taos_errstr(con)); - taos_close(con); +static void taos_error(TAOS_RES *tres, TAOS *conn) { + printf("TDengine error: %s\n", tres?taos_errstr(tres):"null result"); + taos_close(conn); exit(1); } @@ -125,13 +125,17 @@ void writeDataImp(void *param) { printf("Thread %d, writing sID %d, eID %d\n", pThread->threadId, pThread->sID, pThread->eID); void *taos = taos_connect("127.0.0.1", "root", "taosdata", NULL, 0); - if (taos == NULL) - taos_error(taos); + if (taos == NULL) { + // where to find errstr? + // taos_error(NULL, taos); + printf("TDengine error: %s\n", "failed to connect"); + exit(1); + } TAOS_RES* result = taos_query(taos, "use db"); int32_t code = taos_errno(result); if (code != 0) { - taos_error(taos); + taos_error(result, taos); } taos_free_result(result); @@ -227,12 +231,17 @@ void writeData() { taos_init(); void *taos = taos_connect("127.0.0.1", "root", "taosdata", NULL, 0); - if (taos == NULL) taos_error(taos); + if (taos == NULL) { + // where to find errstr? + // taos_error(NULL, taos); + printf("TDengine error: %s\n", "failed to connect"); + exit(1); + } TAOS_RES *result = taos_query(taos, "create database if not exists db"); int32_t code = taos_errno(result); if (code != 0) { - taos_error(taos); + taos_error(result, taos); } taos_free_result(result); @@ -241,7 +250,7 @@ void writeData() { "tags(devid int, devname binary(16), devgroup int)"); code = taos_errno(result); if (code != 0) { - taos_error(taos); + taos_error(result, taos); } taos_free_result(result); @@ -293,8 +302,12 @@ void readDataImp(void *param) printf("open file %s success\n", arguments.sql); void *taos = taos_connect("127.0.0.1", "root", "taosdata", NULL, 0); - if (taos == NULL) - taos_error(taos); + if (taos == NULL) { + // where to find errstr? + // taos_error(NULL, taos); + printf("TDengine error: %s\n", "failed to connect"); + exit(1); + } char *line = NULL; size_t len = 0; @@ -313,7 +326,7 @@ void readDataImp(void *param) TAOS_RES *result = taos_query(taos, line); int32_t code = taos_errno(result); if (code != 0) { - taos_error(taos); + taos_error(result, taos); } TAOS_ROW row; @@ -343,8 +356,12 @@ void readData() { printf("---- clients: %d\n", arguments.clients); void *taos = taos_connect("127.0.0.1", "root", "taosdata", NULL, 0); - if (taos == NULL) - taos_error(taos); + if (taos == NULL) { + // where to find errstr? + // taos_error(NULL, taos); + printf("TDengine error: %s\n", "failed to connect"); + exit(1); + } ThreadObj *threads = calloc((size_t)arguments.clients, sizeof(ThreadObj)); diff --git a/tests/examples/c/demo.c b/tests/examples/c/demo.c index d64c0de1ce..8f8a66a325 100644 --- a/tests/examples/c/demo.c +++ b/tests/examples/c/demo.c @@ -38,7 +38,7 @@ int main(int argc, char *argv[]) { taos = taos_connect(argv[1], "root", "taosdata", NULL, 0); if (taos == NULL) { - printf("failed to connect to server, reason:%s\n", taos_errstr(taos)); + printf("failed to connect to server, reason:%s\n", "null taos"/*taos_errstr(taos)*/); exit(1); } printf("success to connect to server\n"); @@ -48,7 +48,7 @@ int main(int argc, char *argv[]) { result = taos_query(taos, "create database demo"); if (result == NULL) { - printf("failed to create database, reason:%s\n", taos_errstr(taos)); + printf("failed to create database, reason:%s\n", "null result"/*taos_errstr(taos)*/); exit(1); } printf("success to create database\n"); @@ -57,7 +57,7 @@ int main(int argc, char *argv[]) { // create table if (taos_query(taos, "create table m1 (ts timestamp, ti tinyint, si smallint, i int, bi bigint, f float, d double, b binary(10))") == 0) { - printf("failed to create table, reason:%s\n", taos_errstr(taos)); + printf("failed to create table, reason:%s\n", taos_errstr(result)); exit(1); } printf("success to create table\n"); @@ -70,9 +70,19 @@ int main(int argc, char *argv[]) { for (i = 0; i < 10; ++i) { sprintf(qstr, "insert into m1 values (%" PRId64 ", %d, %d, %d, %d, %f, %lf, '%s')", 1546300800000 + i * 1000, i, i, i, i*10000000, i*1.0, i*2.0, "hello"); printf("qstr: %s\n", qstr); - if (taos_query(taos, qstr)) { - printf("insert row: %i, reason:%s\n", i, taos_errstr(taos)); + + // note: how do you wanna do if taos_query returns non-NULL + // if (taos_query(taos, qstr)) { + // printf("insert row: %i, reason:%s\n", i, taos_errstr(taos)); + // } + TAOS_RES *result = taos_query(taos, qstr); + if (result) { + printf("insert row: %i\n", i); + } else { + printf("failed to insert row: %i, reason:%s\n", i, "null result"/*taos_errstr(result)*/); + exit(1); } + //sleep(1); } printf("success to insert rows, total %d rows\n", i); From 231cc2963ebad5a0cfa9d2b8518ca90734d0857f Mon Sep 17 00:00:00 2001 From: freemine Date: Sun, 4 Oct 2020 20:27:04 +0800 Subject: [PATCH 02/51] remove void types --- src/inc/taos.h | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/inc/taos.h b/src/inc/taos.h index 9ea1d268ea..deba808d0a 100644 --- a/src/inc/taos.h +++ b/src/inc/taos.h @@ -30,13 +30,6 @@ typedef struct SSub TAOS_SUB; typedef unsigned char** TAOS_ROW; -// typedef void TAOS; -// typedef void** TAOS_ROW; -// typedef void TAOS_RES; -// typedef void TAOS_SUB; -// typedef void TAOS_STREAM; -// typedef void TAOS_STMT; - // Data type definition #define TSDB_DATA_TYPE_NULL 0 // 1 bytes #define TSDB_DATA_TYPE_BOOL 1 // 1 bytes From a87717bb7d712ddd39a6d3cd37ac48002ac77524 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 17 Oct 2020 16:35:12 +0800 Subject: [PATCH 03/51] [td-225] --- src/client/src/tscAsync.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c index 639de294e6..003eddc8d2 100644 --- a/src/client/src/tscAsync.c +++ b/src/client/src/tscAsync.c @@ -405,11 +405,11 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) { SSqlRes *pRes = &pSql->res; pRes->code = code; + const char* msg = (pCmd->command == TSDB_SQL_STABLEVGROUP)? "vgroup-list":"table-meta"; if (code != TSDB_CODE_SUCCESS) { - tscError("%p get tableMeta failed, code:%s", pSql, tstrerror(code)); + tscError("%p get %s failed, code:%s", pSql, msg, tstrerror(code)); goto _error; } else { - const char* msg = (pCmd->command == TSDB_SQL_STABLEVGROUP)? "vgroup-list":"table-meta"; tscDebug("%p get %s successfully", pSql, msg); } From d069a35683b015f4e65601943fee4e6a3b0ec4c9 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 17 Oct 2020 22:38:07 +0800 Subject: [PATCH 04/51] [td-225] add more info in log. --- src/vnode/src/vnodeRead.c | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/src/vnode/src/vnodeRead.c b/src/vnode/src/vnodeRead.c index baaeae2a81..f6e773c96b 100644 --- a/src/vnode/src/vnodeRead.c +++ b/src/vnode/src/vnodeRead.c @@ -67,12 +67,12 @@ int32_t vnodeProcessRead(void *param, SReadMsg *pReadMsg) { return (*vnodeProcessReadMsgFp[msgType])(pVnode, pReadMsg); } -static void vnodePutItemIntoReadQueue(SVnodeObj *pVnode, void **qhandle) { +static void vnodePutItemIntoReadQueue(SVnodeObj *pVnode, void **qhandle, void *ahandle) { SReadMsg *pRead = (SReadMsg *)taosAllocateQitem(sizeof(SReadMsg)); pRead->rpcMsg.msgType = TSDB_MSG_TYPE_QUERY; pRead->pCont = qhandle; pRead->contLen = 0; - pRead->rpcMsg.handle = NULL; + pRead->rpcMsg.ahandle = ahandle; atomic_add_fetch_32(&pVnode->refCount, 1); @@ -80,14 +80,23 @@ static void vnodePutItemIntoReadQueue(SVnodeObj *pVnode, void **qhandle) { taosWriteQitem(pVnode->rqueue, TAOS_QTYPE_QUERY, pRead); } -static int32_t vnodeDumpQueryResult(SRspRet *pRet, void *pVnode, void **handle, bool *freeHandle) { +/** + * + * @param pRet response message object + * @param pVnode the vnode object + * @param handle qhandle for executing query + * @param freeHandle free qhandle or not + * @param ahandle sqlObj address at client side + * @return + */ +static int32_t vnodeDumpQueryResult(SRspRet *pRet, void *pVnode, void **handle, bool *freeHandle, void *ahandle) { bool continueExec = false; int32_t code = TSDB_CODE_SUCCESS; if ((code = qDumpRetrieveResult(*handle, (SRetrieveTableRsp **)&pRet->rsp, &pRet->len, &continueExec)) == TSDB_CODE_SUCCESS) { if (continueExec) { *freeHandle = false; - vnodePutItemIntoReadQueue(pVnode, handle); + vnodePutItemIntoReadQueue(pVnode, handle, ahandle); pRet->qhandle = *handle; } else { *freeHandle = true; @@ -189,7 +198,7 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { if (handle != NULL) { vDebug("vgId:%d, QInfo:%p, dnode query msg disposed, create qhandle and returns to app", vgId, *handle); - vnodePutItemIntoReadQueue(pVnode, handle); + vnodePutItemIntoReadQueue(pVnode, handle, pReadMsg->rpcMsg.ahandle); } } else { assert(pCont != NULL); @@ -210,7 +219,7 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { pReadMsg->rpcMsg.handle); // set the real rsp error code - pReadMsg->rpcMsg.code = vnodeDumpQueryResult(&pReadMsg->rspRet, pVnode, qhandle, &freehandle); + pReadMsg->rpcMsg.code = vnodeDumpQueryResult(&pReadMsg->rspRet, pVnode, qhandle, &freehandle, pReadMsg->rpcMsg.ahandle); // NOTE: set return code to be TSDB_CODE_QRY_HAS_RSP to notify dnode to return msg to client code = TSDB_CODE_QRY_HAS_RSP; @@ -287,7 +296,8 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { return TSDB_CODE_QRY_NOT_READY; } - code = vnodeDumpQueryResult(pRet, pVnode, handle, &freeHandle); + // ahandle is the sqlObj pointer + code = vnodeDumpQueryResult(pRet, pVnode, handle, &freeHandle, pReadMsg->rpcMsg.ahandle); } // If qhandle is not added into vread queue, the query should be completed already or paused with error. From 95adf3ea611be0c62e4b50131d4ed9432eec2ef5 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 17 Oct 2020 23:47:40 +0800 Subject: [PATCH 05/51] [td-225] --- src/query/src/qExecutor.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index f742616b05..6072b83121 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -6692,12 +6692,12 @@ static bool doBuildResCheck(SQInfo* pQInfo) { pQInfo->dataReady = QUERY_RESULT_READY; buildRes = (pQInfo->rspContext != NULL); - pthread_mutex_unlock(&pQInfo->lock); - - // clear qhandle owner + // clear qhandle owner, it must be in the secure area. other thread may run ahead before current, after it is + // put into task to be executed. assert(pQInfo->owner == taosGetPthreadId()); pQInfo->owner = 0; + pthread_mutex_unlock(&pQInfo->lock); return buildRes; } From adcbb0fe4017456364ece39c0b634abed235181a Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 18 Oct 2020 00:23:19 +0800 Subject: [PATCH 06/51] [td-225] --- src/vnode/src/vnodeRead.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/vnode/src/vnodeRead.c b/src/vnode/src/vnodeRead.c index f6e773c96b..79e69b97a1 100644 --- a/src/vnode/src/vnodeRead.c +++ b/src/vnode/src/vnodeRead.c @@ -224,6 +224,9 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { // NOTE: set return code to be TSDB_CODE_QRY_HAS_RSP to notify dnode to return msg to client code = TSDB_CODE_QRY_HAS_RSP; } else { + void* h1 = qGetResultRetrieveMsg(*qhandle); + assert(h1 == NULL); + freehandle = qQueryCompleted(*qhandle); } From 5adcf14d6e3f5f6c9cc37639789480c133d6ead9 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 18 Oct 2020 00:45:49 +0800 Subject: [PATCH 07/51] [td-225] --- src/query/src/qExecutor.c | 5 +++++ src/vnode/src/vnodeRead.c | 2 ++ 2 files changed, 7 insertions(+) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 6072b83121..1070f65284 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -6227,6 +6227,7 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou // NOTE: pTableCheckInfo need to update the query time range and the lastKey info pQInfo->arrTableIdInfo = taosArrayInit(tableIndex, sizeof(STableIdInfo)); pQInfo->dataReady = QUERY_RESULT_NOT_READY; + pQInfo->rspContext = NULL; pthread_mutex_init(&pQInfo->lock, NULL); pQuery->pos = -1; @@ -6761,6 +6762,7 @@ int32_t qRetrieveQueryResultInfo(qinfo_t qinfo, bool* buildRes, void* pRspContex SQInfo *pQInfo = (SQInfo *)qinfo; if (pQInfo == NULL || !isValidQInfo(pQInfo)) { + qError("QInfo:%p invalid qhandle", pQInfo); return TSDB_CODE_QRY_INVALID_QHANDLE; } @@ -6773,6 +6775,8 @@ int32_t qRetrieveQueryResultInfo(qinfo_t qinfo, bool* buildRes, void* pRspContex int32_t code = TSDB_CODE_SUCCESS; pthread_mutex_lock(&pQInfo->lock); + assert(pQInfo->rspContext == NULL); + if (pQInfo->dataReady == QUERY_RESULT_READY) { *buildRes = true; qDebug("QInfo:%p retrieve result info, rowsize:%d, rows:%"PRId64", code:%d", pQInfo, pQuery->rowSize, pQuery->rec.rows, @@ -6781,6 +6785,7 @@ int32_t qRetrieveQueryResultInfo(qinfo_t qinfo, bool* buildRes, void* pRspContex *buildRes = false; qDebug("QInfo:%p retrieve req set query return result after paused", pQInfo); pQInfo->rspContext = pRspContext; + assert(pQInfo->rspContext != NULL); } code = pQInfo->code; diff --git a/src/vnode/src/vnodeRead.c b/src/vnode/src/vnodeRead.c index 79e69b97a1..a610a8b0df 100644 --- a/src/vnode/src/vnodeRead.c +++ b/src/vnode/src/vnodeRead.c @@ -295,6 +295,8 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { freeHandle = true; } else { // result is not ready, return immediately if (!buildRes) { + assert(pReadMsg->rpcMsg.handle != NULL); + qReleaseQInfo(pVnode->qMgmt, (void **)&handle, false); return TSDB_CODE_QRY_NOT_READY; } From 5df7a811604b04a2406455247430e22bdd2537aa Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 18 Oct 2020 01:14:01 +0800 Subject: [PATCH 08/51] [td-225] --- src/rpc/src/rpcMain.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index 63231bb1fa..6e9088d9fb 100644 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -412,7 +412,7 @@ void rpcSendResponse(const SRpcMsg *pRsp) { rpcLockConn(pConn); if ( pConn->inType == 0 || pConn->user[0] == 0 ) { - tDebug("%s, connection is already released, rsp wont be sent", pConn->info); + tError("%s, connection is already released, rsp wont be sent", pConn->info); rpcUnlockConn(pConn); rpcFreeCont(pMsg->pCont); rpcDecRef(pRpc); From 134d1f64343b93d29ede1afca148c6e746afd9b9 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 18 Oct 2020 02:03:10 +0800 Subject: [PATCH 09/51] [td-225] --- src/dnode/src/dnodeVRead.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/dnode/src/dnodeVRead.c b/src/dnode/src/dnodeVRead.c index 1a3d0ebc27..cdb03de555 100644 --- a/src/dnode/src/dnodeVRead.c +++ b/src/dnode/src/dnodeVRead.c @@ -213,7 +213,8 @@ static void *dnodeProcessReadQueue(void *param) { } else { if (code == TSDB_CODE_QRY_HAS_RSP) { dnodeSendRpcReadRsp(pVnode, pReadMsg, pReadMsg->rpcMsg.code); - } else { // code == TSDB_CODE_NOT_READY, do not return msg to client + } else { // code == TSDB_CODE_QRY_NOT_READY, do not return msg to client + assert(pReadMsg->rpcMsg.handle != NULL); dnodeDispatchNonRspMsg(pVnode, pReadMsg, code); } } From a0a274e8ba6a795661153312c4b8a5fe41909f0b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 18 Oct 2020 02:16:36 +0800 Subject: [PATCH 10/51] [td-225] --- src/dnode/src/dnodeVRead.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dnode/src/dnodeVRead.c b/src/dnode/src/dnodeVRead.c index cdb03de555..9dac65c765 100644 --- a/src/dnode/src/dnodeVRead.c +++ b/src/dnode/src/dnodeVRead.c @@ -214,7 +214,7 @@ static void *dnodeProcessReadQueue(void *param) { if (code == TSDB_CODE_QRY_HAS_RSP) { dnodeSendRpcReadRsp(pVnode, pReadMsg, pReadMsg->rpcMsg.code); } else { // code == TSDB_CODE_QRY_NOT_READY, do not return msg to client - assert(pReadMsg->rpcMsg.handle != NULL); + assert(pReadMsg->rpcMsg.handle == NULL); dnodeDispatchNonRspMsg(pVnode, pReadMsg, code); } } From 1e8461e66c9095a2975818a9924e19809f1e8730 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 18 Oct 2020 02:24:29 +0800 Subject: [PATCH 11/51] [td-225] --- src/dnode/src/dnodeVRead.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dnode/src/dnodeVRead.c b/src/dnode/src/dnodeVRead.c index 9dac65c765..f6909e54d4 100644 --- a/src/dnode/src/dnodeVRead.c +++ b/src/dnode/src/dnodeVRead.c @@ -214,7 +214,7 @@ static void *dnodeProcessReadQueue(void *param) { if (code == TSDB_CODE_QRY_HAS_RSP) { dnodeSendRpcReadRsp(pVnode, pReadMsg, pReadMsg->rpcMsg.code); } else { // code == TSDB_CODE_QRY_NOT_READY, do not return msg to client - assert(pReadMsg->rpcMsg.handle == NULL); + assert(pReadMsg->rpcMsg.handle == NULL || (pReadMsg->rpcMsg.handle != NULL && pReadMsg->rpcMsg.msgType == 5)); dnodeDispatchNonRspMsg(pVnode, pReadMsg, code); } } From b3165d4193819e7aaf719cd7048addb27a3fd37d Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 18 Oct 2020 15:38:18 +0800 Subject: [PATCH 12/51] [td-225] --- src/query/inc/qExecutor.h | 1 + src/query/src/qExecutor.c | 13 ++++++++++++- src/vnode/src/vnodeRead.c | 11 ++++++++++- 3 files changed, 23 insertions(+), 2 deletions(-) diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h index 5d570821cb..fbf699297a 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qExecutor.h @@ -206,6 +206,7 @@ typedef struct SQInfo { void* pBuf; // allocated buffer for STableQueryInfo, sizeof(STableQueryInfo)*numOfTables; pthread_mutex_t lock; // used to synchronize the rsp/query threads + tsem_t ready; int32_t dataReady; // denote if query result is ready or not void* rspContext; // response context } SQInfo; diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 1070f65284..b9a5295ed2 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -6699,6 +6699,8 @@ static bool doBuildResCheck(SQInfo* pQInfo) { pQInfo->owner = 0; pthread_mutex_unlock(&pQInfo->lock); + + tsem_post(&pQInfo->ready); return buildRes; } @@ -6767,13 +6769,16 @@ int32_t qRetrieveQueryResultInfo(qinfo_t qinfo, bool* buildRes, void* pRspContex } *buildRes = false; - SQuery *pQuery = pQInfo->runtimeEnv.pQuery; if (IS_QUERY_KILLED(pQInfo)) { qDebug("QInfo:%p query is killed, code:%d", pQInfo, pQInfo->code); return pQInfo->code; } int32_t code = TSDB_CODE_SUCCESS; + +#if 0 + SQuery *pQuery = pQInfo->runtimeEnv.pQuery; + pthread_mutex_lock(&pQInfo->lock); assert(pQInfo->rspContext == NULL); @@ -6790,6 +6795,12 @@ int32_t qRetrieveQueryResultInfo(qinfo_t qinfo, bool* buildRes, void* pRspContex code = pQInfo->code; pthread_mutex_unlock(&pQInfo->lock); +#else + tsem_wait(&pQInfo->ready); + *buildRes = true; + code = pQInfo->code; +#endif + return code; } diff --git a/src/vnode/src/vnodeRead.c b/src/vnode/src/vnodeRead.c index a610a8b0df..ede5921dc1 100644 --- a/src/vnode/src/vnodeRead.c +++ b/src/vnode/src/vnodeRead.c @@ -14,7 +14,8 @@ */ #define _DEFAULT_SOURCE -//#include +#define _NON_BLOCKING_RETRIEVE 0 + #include "os.h" #include "tglobal.h" @@ -206,6 +207,8 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { vDebug("vgId:%d, QInfo:%p, dnode continues to exec query", pVnode->vgId, *qhandle); + +#if _NON_BLOCKING_RETRIEVE bool freehandle = false; bool buildRes = qTableQuery(*qhandle); // do execute query @@ -235,6 +238,9 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { if (freehandle || (!buildRes)) { qReleaseQInfo(pVnode->qMgmt, (void **)&qhandle, freehandle); } +#else + qTableQuery(*qhandle); // do execute query +#endif } return code; @@ -294,12 +300,15 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { memset(pRet->rsp, 0, sizeof(SRetrieveTableRsp)); freeHandle = true; } else { // result is not ready, return immediately + assert(buildRes == true); +#if _NON_BLOCKING_RETRIEVE if (!buildRes) { assert(pReadMsg->rpcMsg.handle != NULL); qReleaseQInfo(pVnode->qMgmt, (void **)&handle, false); return TSDB_CODE_QRY_NOT_READY; } +#endif // ahandle is the sqlObj pointer code = vnodeDumpQueryResult(pRet, pVnode, handle, &freeHandle, pReadMsg->rpcMsg.ahandle); From 40c7208cc2ab92d7c45a999764730d49902f97a5 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 19 Oct 2020 09:53:20 +0800 Subject: [PATCH 13/51] [td-225] --- src/query/src/qExecutor.c | 1 + src/vnode/src/vnodeRead.c | 16 ++++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index b9a5295ed2..f6c2ce29d6 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -6229,6 +6229,7 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou pQInfo->dataReady = QUERY_RESULT_NOT_READY; pQInfo->rspContext = NULL; pthread_mutex_init(&pQInfo->lock, NULL); + tsem_init(&pQInfo->ready, 0, 0); pQuery->pos = -1; pQuery->window = pQueryMsg->window; diff --git a/src/vnode/src/vnodeRead.c b/src/vnode/src/vnodeRead.c index ede5921dc1..9c3fc1c7da 100644 --- a/src/vnode/src/vnodeRead.c +++ b/src/vnode/src/vnodeRead.c @@ -272,6 +272,11 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { if (pRetrieve->free == 1) { vWarn("vgId:%d, QInfo:%p, retrieve msg received to kill query and free qhandle", pVnode->vgId, *handle); qKillQuery(*handle); +#if !(_NON_BLOCKING_RETRIEVE) + void** p = handle; + qReleaseQInfo(pVnode->qMgmt, (void **)&p, false); +#endif + qReleaseQInfo(pVnode->qMgmt, (void **)&handle, true); vnodeBuildNoResultQueryRsp(pRet); @@ -285,6 +290,12 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { pReadMsg->rpcMsg.handle); code = TSDB_CODE_RPC_NETWORK_UNAVAIL; qKillQuery(*handle); + +#if !(_NON_BLOCKING_RETRIEVE) + void** p = handle; + qReleaseQInfo(pVnode->qMgmt, (void **)&p, false); +#endif + qReleaseQInfo(pVnode->qMgmt, (void **)&handle, true); return code; } @@ -314,6 +325,11 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { code = vnodeDumpQueryResult(pRet, pVnode, handle, &freeHandle, pReadMsg->rpcMsg.ahandle); } +#if !(_NON_BLOCKING_RETRIEVE) + void** p = handle; + qReleaseQInfo(pVnode->qMgmt, (void **)&p, false); +#endif + // If qhandle is not added into vread queue, the query should be completed already or paused with error. // Here free qhandle immediately if (freeHandle) { From 3abaaabeee3b081b0ce8253465e90e0a9a3933d0 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 19 Oct 2020 11:33:49 +0800 Subject: [PATCH 14/51] [td-225] --- src/vnode/src/vnodeRead.c | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/src/vnode/src/vnodeRead.c b/src/vnode/src/vnodeRead.c index 9c3fc1c7da..a86d83cf2d 100644 --- a/src/vnode/src/vnodeRead.c +++ b/src/vnode/src/vnodeRead.c @@ -240,6 +240,7 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { } #else qTableQuery(*qhandle); // do execute query + qReleaseQInfo(pVnode->qMgmt, (void **)&qhandle, false); #endif } @@ -272,11 +273,6 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { if (pRetrieve->free == 1) { vWarn("vgId:%d, QInfo:%p, retrieve msg received to kill query and free qhandle", pVnode->vgId, *handle); qKillQuery(*handle); -#if !(_NON_BLOCKING_RETRIEVE) - void** p = handle; - qReleaseQInfo(pVnode->qMgmt, (void **)&p, false); -#endif - qReleaseQInfo(pVnode->qMgmt, (void **)&handle, true); vnodeBuildNoResultQueryRsp(pRet); @@ -290,12 +286,6 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { pReadMsg->rpcMsg.handle); code = TSDB_CODE_RPC_NETWORK_UNAVAIL; qKillQuery(*handle); - -#if !(_NON_BLOCKING_RETRIEVE) - void** p = handle; - qReleaseQInfo(pVnode->qMgmt, (void **)&p, false); -#endif - qReleaseQInfo(pVnode->qMgmt, (void **)&handle, true); return code; } @@ -325,11 +315,6 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { code = vnodeDumpQueryResult(pRet, pVnode, handle, &freeHandle, pReadMsg->rpcMsg.ahandle); } -#if !(_NON_BLOCKING_RETRIEVE) - void** p = handle; - qReleaseQInfo(pVnode->qMgmt, (void **)&p, false); -#endif - // If qhandle is not added into vread queue, the query should be completed already or paused with error. // Here free qhandle immediately if (freeHandle) { From fb0456ca4cd88f5a08a97acd3242ada5d89080b0 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 19 Oct 2020 16:26:28 +0800 Subject: [PATCH 15/51] [td-1747] reduce tableMeta size --- src/client/inc/tsclient.h | 8 ++++---- src/client/src/tscSchemaUtil.c | 15 +++++++++++++-- src/client/src/tscServer.c | 18 ++++++++++++------ src/inc/taosmsg.h | 27 ++++++++++++++++++++++----- src/mnode/src/mnodeTable.c | 24 ++++++++++++------------ 5 files changed, 63 insertions(+), 29 deletions(-) diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index 8621f9d28b..f0d3e26a49 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -90,10 +90,10 @@ typedef struct STableComInfo { } STableComInfo; typedef struct SCMCorVgroupInfo { - int32_t version; - int8_t inUse; - int8_t numOfEps; - SEpAddr epAddr[TSDB_MAX_REPLICA]; + int32_t version; + int8_t inUse; + int8_t numOfEps; + SEpAddr1 epAddr[TSDB_MAX_REPLICA]; } SCMCorVgroupInfo; typedef struct STableMeta { diff --git a/src/client/src/tscSchemaUtil.c b/src/client/src/tscSchemaUtil.c index 1e841c68fd..fb9130b6c5 100644 --- a/src/client/src/tscSchemaUtil.c +++ b/src/client/src/tscSchemaUtil.c @@ -145,10 +145,11 @@ static void tscInitCorVgroupInfo(SCMCorVgroupInfo *corVgroupInfo, SCMVgroupInfo corVgroupInfo->inUse = 0; corVgroupInfo->numOfEps = vgroupInfo->numOfEps; for (int32_t i = 0; i < corVgroupInfo->numOfEps; i++) { - strncpy(corVgroupInfo->epAddr[i].fqdn, vgroupInfo->epAddr[i].fqdn, TSDB_FQDN_LEN); + corVgroupInfo->epAddr[i].fqdn = strdup(vgroupInfo->epAddr[i].fqdn); corVgroupInfo->epAddr[i].port = vgroupInfo->epAddr[i].port; } } + STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg, size_t* size) { assert(pTableMetaMsg != NULL); @@ -164,7 +165,17 @@ STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg, size_t* size pTableMeta->id.tid = pTableMetaMsg->sid; pTableMeta->id.uid = pTableMetaMsg->uid; - pTableMeta->vgroupInfo = pTableMetaMsg->vgroup; + + SCMVgroupInfo* pVgroupInfo = &pTableMeta->vgroupInfo; + pVgroupInfo->numOfEps = pTableMetaMsg->vgroup.numOfEps; + pVgroupInfo->vgId = pTableMetaMsg->vgroup.vgId; + + for(int32_t i = 0; i < pVgroupInfo->numOfEps; ++i) { + SEpAddrMsg* pEpMsg = &pTableMetaMsg->vgroup.epAddr[i]; + + pVgroupInfo->epAddr[i].fqdn = strndup(pEpMsg->fqdn, tListLen(pEpMsg->fqdn)); + pVgroupInfo->epAddr[i].port = pEpMsg->port; + } tscInitCorVgroupInfo(&pTableMeta->corVgroupInfo, &pTableMeta->vgroupInfo); diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 494a8a9c30..bfd913dcf5 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -1839,22 +1839,28 @@ int tscProcessSTableVgroupRsp(SSqlObj *pSql) { SSqlCmd* pCmd = &parent->cmd; for(int32_t i = 0; i < pStableVgroup->numOfTables; ++i) { STableMetaInfo *pInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, i); - SVgroupsInfo * pVgroupInfo = (SVgroupsInfo *)pMsg; - pVgroupInfo->numOfVgroups = htonl(pVgroupInfo->numOfVgroups); - size_t size = sizeof(SCMVgroupInfo) * pVgroupInfo->numOfVgroups + sizeof(SVgroupsInfo); + SVgroupsMsg * pVgroupMsg = (SVgroupsMsg *) pMsg; + pVgroupMsg->numOfVgroups = htonl(pVgroupMsg->numOfVgroups); + + size_t size = sizeof(SCMVgroupInfo) * pVgroupMsg->numOfVgroups + sizeof(SVgroupsInfo); pInfo->vgroupList = calloc(1, size); assert(pInfo->vgroupList != NULL); - memcpy(pInfo->vgroupList, pVgroupInfo, size); + pInfo->vgroupList->numOfVgroups = pVgroupMsg->numOfVgroups; for (int32_t j = 0; j < pInfo->vgroupList->numOfVgroups; ++j) { //just init, no need to lock SCMVgroupInfo *pVgroups = &pInfo->vgroupList->vgroups[j]; - pVgroups->vgId = htonl(pVgroups->vgId); + + SCMVgroupMsg *vgroupMsg = &pVgroupMsg->vgroups[j]; + pVgroups->vgId = htonl(vgroupMsg->vgId); + pVgroups->numOfEps = vgroupMsg->numOfEps; + assert(pVgroups->numOfEps >= 1); for (int32_t k = 0; k < pVgroups->numOfEps; ++k) { - pVgroups->epAddr[k].port = htons(pVgroups->epAddr[k].port); + pVgroups->epAddr[k].port = htons(vgroupMsg->epAddr[k].port); + pVgroups->epAddr[k].fqdn = strndup(vgroupMsg->epAddr[k].fqdn, tListLen(vgroupMsg->epAddr[k].fqdn)); } } diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index be66c76d71..635e2aba80 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -182,10 +182,16 @@ extern char *taosMsg[]; #pragma pack(push, 1) +// null-terminated string instead of char array to avoid too many memory consumption in case of more than 1M tableMeta typedef struct { char fqdn[TSDB_FQDN_LEN]; uint16_t port; -} SEpAddr; +} SEpAddrMsg; + +typedef struct { + char* fqdn; + uint16_t port; +} SEpAddr1; typedef struct { int32_t numOfVnodes; @@ -662,16 +668,27 @@ typedef struct SCMSTableVgroupMsg { } SCMSTableVgroupMsg, SCMSTableVgroupRspMsg; typedef struct { - int32_t vgId; - int8_t numOfEps; - SEpAddr epAddr[TSDB_MAX_REPLICA]; + int32_t vgId; + int8_t numOfEps; + SEpAddr1 epAddr[TSDB_MAX_REPLICA]; } SCMVgroupInfo; +typedef struct { + int32_t vgId; + int8_t numOfEps; + SEpAddrMsg epAddr[TSDB_MAX_REPLICA]; +} SCMVgroupMsg; + typedef struct { int32_t numOfVgroups; SCMVgroupInfo vgroups[]; } SVgroupsInfo; +typedef struct { + int32_t numOfVgroups; + SCMVgroupMsg vgroups[]; +} SVgroupsMsg; + typedef struct STableMetaMsg { int32_t contLen; char tableId[TSDB_TABLE_FNAME_LEN]; // table id @@ -684,7 +701,7 @@ typedef struct STableMetaMsg { int16_t tversion; int32_t sid; uint64_t uid; - SCMVgroupInfo vgroup; + SCMVgroupMsg vgroup; SSchema schema[]; } STableMetaMsg; diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index 1a1ff878a5..9965047a99 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -1472,12 +1472,12 @@ static int32_t mnodeProcessSuperTableVgroupMsg(SMnodeMsg *pMsg) { int32_t numOfTable = htonl(pInfo->numOfTables); // reserve space - int32_t contLen = sizeof(SCMSTableVgroupRspMsg) + 32 * sizeof(SCMVgroupInfo) + sizeof(SVgroupsInfo); + int32_t contLen = sizeof(SCMSTableVgroupRspMsg) + 32 * sizeof(SCMVgroupInfo) + sizeof(SVgroupsMsg); for (int32_t i = 0; i < numOfTable; ++i) { char *stableName = (char*)pInfo + sizeof(SCMSTableVgroupMsg) + (TSDB_TABLE_FNAME_LEN) * i; SSuperTableObj *pTable = mnodeGetSuperTable(stableName); if (pTable != NULL && pTable->vgHash != NULL) { - contLen += (taosHashGetSize(pTable->vgHash) * sizeof(SCMVgroupInfo) + sizeof(SVgroupsInfo)); + contLen += (taosHashGetSize(pTable->vgHash) * sizeof(SCMVgroupInfo) + sizeof(SVgroupsMsg)); } mnodeDecTableRef(pTable); } @@ -1506,12 +1506,12 @@ static int32_t mnodeProcessSuperTableVgroupMsg(SMnodeMsg *pMsg) { // even this super table has no corresponding table, still return pRsp->numOfTables++; - SVgroupsInfo *pVgroupInfo = (SVgroupsInfo *)msg; - pVgroupInfo->numOfVgroups = 0; + SVgroupsMsg *pVgroupMsg = (SVgroupsMsg *)msg; + pVgroupMsg->numOfVgroups = 0; - msg += sizeof(SVgroupsInfo); + msg += sizeof(SVgroupsMsg); } else { - SVgroupsInfo *pVgroupInfo = (SVgroupsInfo *)msg; + SVgroupsMsg *pVgroupMsg = (SVgroupsMsg *)msg; SHashMutableIterator *pIter = taosHashCreateIter(pTable->vgHash); int32_t vgSize = 0; @@ -1520,15 +1520,15 @@ static int32_t mnodeProcessSuperTableVgroupMsg(SMnodeMsg *pMsg) { SVgObj * pVgroup = mnodeGetVgroup(*pVgId); if (pVgroup == NULL) continue; - pVgroupInfo->vgroups[vgSize].vgId = htonl(pVgroup->vgId); + pVgroupMsg->vgroups[vgSize].vgId = htonl(pVgroup->vgId); for (int32_t vn = 0; vn < pVgroup->numOfVnodes; ++vn) { SDnodeObj *pDnode = pVgroup->vnodeGid[vn].pDnode; if (pDnode == NULL) break; - tstrncpy(pVgroupInfo->vgroups[vgSize].epAddr[vn].fqdn, pDnode->dnodeFqdn, TSDB_FQDN_LEN); - pVgroupInfo->vgroups[vgSize].epAddr[vn].port = htons(pDnode->dnodePort); + tstrncpy(pVgroupMsg->vgroups[vgSize].epAddr[vn].fqdn, pDnode->dnodeFqdn, TSDB_FQDN_LEN); + pVgroupMsg->vgroups[vgSize].epAddr[vn].port = htons(pDnode->dnodePort); - pVgroupInfo->vgroups[vgSize].numOfEps++; + pVgroupMsg->vgroups[vgSize].numOfEps++; } vgSize++; @@ -1538,10 +1538,10 @@ static int32_t mnodeProcessSuperTableVgroupMsg(SMnodeMsg *pMsg) { taosHashDestroyIter(pIter); mnodeDecTableRef(pTable); - pVgroupInfo->numOfVgroups = htonl(vgSize); + pVgroupMsg->numOfVgroups = htonl(vgSize); // one table is done, try the next table - msg += sizeof(SVgroupsInfo) + vgSize * sizeof(SCMVgroupInfo); + msg += sizeof(SVgroupsMsg) + vgSize * sizeof(SCMVgroupMsg); pRsp->numOfTables++; } } From 2cac39d66599050ea46016db23f451617d623b15 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 19 Oct 2020 18:20:33 +0800 Subject: [PATCH 16/51] [td-225] reduce tableMeta size --- src/client/src/tscServer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index bfd913dcf5..8a50299d51 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -1856,7 +1856,7 @@ int tscProcessSTableVgroupRsp(SSqlObj *pSql) { pVgroups->vgId = htonl(vgroupMsg->vgId); pVgroups->numOfEps = vgroupMsg->numOfEps; - assert(pVgroups->numOfEps >= 1); + assert(pVgroups->numOfEps >= 1 && pVgroups->vgId >= 1); for (int32_t k = 0; k < pVgroups->numOfEps; ++k) { pVgroups->epAddr[k].port = htons(vgroupMsg->epAddr[k].port); From aa55054e368aee22563a47c143e035108cb46248 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 19 Oct 2020 23:19:44 +0800 Subject: [PATCH 17/51] [td-1747] --- src/client/inc/tscUtil.h | 5 +- src/client/inc/tsclient.h | 1 + src/client/src/tscServer.c | 14 +++--- src/client/src/tscStream.c | 6 +-- src/client/src/tscSubquery.c | 8 +-- src/client/src/tscSystem.c | 8 +-- src/client/src/tscUtil.c | 95 +++++++++++++++++++++++++++++++----- src/mnode/src/mnodeTable.c | 4 +- 8 files changed, 111 insertions(+), 30 deletions(-) diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index 76a9bbac10..1bd86714c5 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -231,10 +231,11 @@ int tscGetTableMeta(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo); int tscGetMeterMetaEx(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo, bool createIfNotExists); void tscResetForNextRetrieve(SSqlRes* pRes); - -void tscAddTimestampColumn(SQueryInfo* pQueryInfo, int16_t functionId, int16_t tableIndex); void tscDoQuery(SSqlObj* pSql); +SVgroupsInfo* tscVgroupInfoClone(SVgroupsInfo *pInfo); +void* tscVgroupInfoClear(SVgroupsInfo *pInfo); +void tscSCMVgroupInfoCopy(SCMVgroupInfo* dst, const SCMVgroupInfo* src); /** * The create object function must be successful expect for the out of memory issue. * diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index f0d3e26a49..5d9abf852e 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -443,6 +443,7 @@ void tscPartiallyFreeSqlObj(SSqlObj *pSql); */ void tscFreeSqlObj(SSqlObj *pSql); void tscFreeRegisteredSqlObj(void *pSql); +void tscFreeTableMetaHelper(void *pTableMeta); void tscCloseTscObj(STscObj *pObj); diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 8a50299d51..ff798a9f99 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -124,9 +124,11 @@ static void tscUpdateVgroupInfo(SSqlObj *pObj, SRpcEpSet *pEpSet) { pVgroupInfo->inUse = pEpSet->inUse; pVgroupInfo->numOfEps = pEpSet->numOfEps; for (int32_t i = 0; i < pVgroupInfo->numOfEps; i++) { - tstrncpy(pVgroupInfo->epAddr[i].fqdn, pEpSet->fqdn[i], TSDB_FQDN_LEN); + taosTFree(pVgroupInfo->epAddr[i].fqdn); + pVgroupInfo->epAddr[i].fqdn = strndup(pEpSet->fqdn[i], tListLen(pEpSet->fqdn[i])); pVgroupInfo->epAddr[i].port = pEpSet->port[i]; } + tscDebug("after: EndPoint in use: %d", pVgroupInfo->inUse); taosCorEndWrite(&pVgroupInfo->version); } @@ -1852,15 +1854,15 @@ int tscProcessSTableVgroupRsp(SSqlObj *pSql) { //just init, no need to lock SCMVgroupInfo *pVgroups = &pInfo->vgroupList->vgroups[j]; - SCMVgroupMsg *vgroupMsg = &pVgroupMsg->vgroups[j]; - pVgroups->vgId = htonl(vgroupMsg->vgId); - pVgroups->numOfEps = vgroupMsg->numOfEps; + SCMVgroupMsg *vmsg = &pVgroupMsg->vgroups[j]; + pVgroups->vgId = htonl(vmsg->vgId); + pVgroups->numOfEps = vmsg->numOfEps; assert(pVgroups->numOfEps >= 1 && pVgroups->vgId >= 1); for (int32_t k = 0; k < pVgroups->numOfEps; ++k) { - pVgroups->epAddr[k].port = htons(vgroupMsg->epAddr[k].port); - pVgroups->epAddr[k].fqdn = strndup(vgroupMsg->epAddr[k].fqdn, tListLen(vgroupMsg->epAddr[k].fqdn)); + pVgroups->epAddr[k].port = htons(vmsg->epAddr[k].port); + pVgroups->epAddr[k].fqdn = strndup(vmsg->epAddr[k].fqdn, tListLen(vmsg->epAddr[k].fqdn)); } } diff --git a/src/client/src/tscStream.c b/src/client/src/tscStream.c index 61614b56fb..0f67911bbe 100644 --- a/src/client/src/tscStream.c +++ b/src/client/src/tscStream.c @@ -168,8 +168,8 @@ static void tscProcessStreamQueryCallback(void *param, TAOS_RES *tres, int numOf STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pStream->pSql->cmd, 0, 0); taosCacheRelease(tscMetaCache, (void**)&(pTableMetaInfo->pTableMeta), true); - taosTFree(pTableMetaInfo->vgroupList); - + pTableMetaInfo->vgroupList = tscVgroupInfoClear(pTableMetaInfo->vgroupList); + tscSetRetryTimer(pStream, pStream->pSql, retryDelay); return; } @@ -275,7 +275,7 @@ static void tscProcessStreamRetrieveResult(void *param, TAOS_RES *res, int numOf tscFreeSqlResult(pSql); taosTFree(pSql->pSubs); pSql->subState.numOfSub = 0; - taosTFree(pTableMetaInfo->vgroupList); + pTableMetaInfo->vgroupList = tscVgroupInfoClear(pTableMetaInfo->vgroupList); tscSetNextLaunchTimer(pStream, pSql); } } diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 49759bc4d3..7a2a242df5 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -449,7 +449,7 @@ void tscBuildVgroupTableInfo(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo, SArr SVgroupTableInfo info = {{0}}; for (int32_t m = 0; m < pvg->numOfVgroups; ++m) { if (tt->vgId == pvg->vgroups[m].vgId) { - info.vgInfo = pvg->vgroups[m]; + tscSCMVgroupInfoCopy(&info.vgInfo, &pvg->vgroups[m]); break; } } @@ -1645,9 +1645,9 @@ static void tscAllDataRetrievedFromDnode(SRetrieveSupport *trsupport, SSqlObj* p // data in from current vnode is stored in cache and disk uint32_t numOfRowsFromSubquery = (uint32_t)(trsupport->pExtMemBuffer[idx]->numOfTotalElems + trsupport->localBuffer->num); - tscDebug("%p sub:%p all data retrieved from ep:%s, vgId:%d, numOfRows:%d, orderOfSub:%d", pParentSql, pSql, - pTableMetaInfo->vgroupList->vgroups[0].epAddr[0].fqdn, pTableMetaInfo->vgroupList->vgroups[0].vgId, - numOfRowsFromSubquery, idx); + SVgroupsInfo* vgroupsInfo = pTableMetaInfo->vgroupList; + tscDebug("%p sub:%p all data retrieved from ep:%s, vgId:%d, numOfRows:%d, orderOfSub:%d", pParentSql, pSql, + vgroupsInfo->vgroups[0].epAddr[0].fqdn, vgroupsInfo->vgroups[0].vgId, numOfRowsFromSubquery, idx); tColModelCompact(pDesc->pColumnModel, trsupport->localBuffer, pDesc->pColumnModel->capacity); diff --git a/src/client/src/tscSystem.c b/src/client/src/tscSystem.c index 4c5dbb079f..47c2d35a75 100644 --- a/src/client/src/tscSystem.c +++ b/src/client/src/tscSystem.c @@ -77,6 +77,7 @@ int32_t tscInitRpc(const char *user, const char *secretEncrypt, void **pDnodeCon return 0; } + void taos_init_imp(void) { char temp[128]; @@ -124,8 +125,9 @@ void taos_init_imp(void) { double factor = (tscEmbedded == 0)? 2.0:4.0; tscNumOfThreads = (int)(tsNumOfCores * tsNumOfThreadsPerCore / factor); - - if (tscNumOfThreads < 2) tscNumOfThreads = 2; + if (tscNumOfThreads < 2) { + tscNumOfThreads = 2; + } tscQhandle = taosInitScheduler(queueSize, tscNumOfThreads, "tsc"); if (NULL == tscQhandle) { @@ -140,7 +142,7 @@ void taos_init_imp(void) { int64_t refreshTime = 10; // 10 seconds by default if (tscMetaCache == NULL) { - tscMetaCache = taosCacheInit(TSDB_DATA_TYPE_BINARY, refreshTime, false, NULL, "tableMeta"); + tscMetaCache = taosCacheInit(TSDB_DATA_TYPE_BINARY, refreshTime, false, tscFreeTableMetaHelper, "tableMeta"); tscObjCache = taosCacheInit(TSDB_CACHE_PTR_KEY, refreshTime / 2, false, tscFreeRegisteredSqlObj, "sqlObj"); } diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 0235f037bd..9e861d3f28 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -115,7 +115,7 @@ bool tscIsTwoStageSTableQuery(SQueryInfo* pQueryInfo, int32_t tableIndex) { // for select query super table, the super table vgroup list can not be null in any cases. if (pQueryInfo->command == TSDB_SQL_SELECT && UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { - assert(pTableMetaInfo->vgroupList != NULL); +// assert(pTableMetaInfo->vgroupList != NULL); // if retrieve vgroupInfo failed, the value may be null } if ((pQueryInfo->type & TSDB_QUERY_TYPE_FREE_RESOURCE) == TSDB_QUERY_TYPE_FREE_RESOURCE) { @@ -408,6 +408,24 @@ void tscFreeRegisteredSqlObj(void *pSql) { } } +void tscFreeTableMetaHelper(void *pTableMeta) { + STableMeta* p = (STableMeta*) pTableMeta; + + int32_t numOfEps = p->vgroupInfo.numOfEps; + assert(numOfEps >= 0 && numOfEps <= TSDB_MAX_REPLICA); + + for(int32_t i = 0; i < numOfEps; ++i) { + taosTFree(p->vgroupInfo.epAddr[i].fqdn); + } + + int32_t numOfEps1 = p->corVgroupInfo.numOfEps; + assert(numOfEps1 >= 0 && numOfEps1 <= TSDB_MAX_REPLICA); + + for(int32_t i = 0; i < numOfEps1; ++i) { + taosTFree(p->corVgroupInfo.epAddr[i].fqdn); + } +} + void tscFreeSqlObj(SSqlObj* pSql) { if (pSql == NULL || pSql->signature != pSql) { return; @@ -1682,8 +1700,14 @@ void tscClearSubqueryInfo(SSqlCmd* pCmd) { void tscFreeVgroupTableInfo(SArray* pVgroupTables) { if (pVgroupTables != NULL) { - for (size_t i = 0; i < taosArrayGetSize(pVgroupTables); i++) { + size_t num = taosArrayGetSize(pVgroupTables); + for (size_t i = 0; i < num; i++) { SVgroupTableInfo* pInfo = taosArrayGet(pVgroupTables, i); + + for(int32_t j = 0; j < pInfo->vgInfo.numOfEps; ++j) { + taosTFree(pInfo->vgInfo.epAddr[j].fqdn); + } + taosArrayDestroy(pInfo->itemList); } taosArrayDestroy(pVgroupTables); @@ -1695,6 +1719,7 @@ void clearAllTableMetaInfo(SQueryInfo* pQueryInfo, const char* address, bool rem for(int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, i); + tscFreeVgroupTableInfo(pTableMetaInfo->pVgroupTables); tscClearTableMetaInfo(pTableMetaInfo, removeFromCache); free(pTableMetaInfo); @@ -1727,13 +1752,7 @@ STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, const char* name, ST pTableMetaInfo->pTableMeta = pTableMeta; if (vgroupList != NULL) { - size_t size = sizeof(SVgroupsInfo) + sizeof(SCMVgroupInfo) * vgroupList->numOfVgroups; - pTableMetaInfo->vgroupList = malloc(size); - if (pTableMetaInfo->vgroupList == NULL) { - return NULL; - } - - memcpy(pTableMetaInfo->vgroupList, vgroupList, size); + pTableMetaInfo->vgroupList = tscVgroupInfoClone(vgroupList); } pTableMetaInfo->tagColList = taosArrayInit(4, POINTER_BYTES); @@ -1762,8 +1781,7 @@ void tscClearTableMetaInfo(STableMetaInfo* pTableMetaInfo, bool removeFromCache) taosCacheRelease(tscMetaCache, (void**)&(pTableMetaInfo->pTableMeta), removeFromCache); } - taosTFree(pTableMetaInfo->vgroupList); - + pTableMetaInfo->vgroupList = tscVgroupInfoClear(pTableMetaInfo->vgroupList); tscColumnListDestroy(pTableMetaInfo->tagColList); pTableMetaInfo->tagColList = NULL; } @@ -2403,3 +2421,58 @@ void tscClearSqlOwner(SSqlObj* pSql) { assert(taosCheckPthreadValid(pSql->owner)); atomic_store_64(&pSql->owner, 0); } + +SVgroupsInfo* tscVgroupInfoClone(SVgroupsInfo *vgroupList) { + if (vgroupList == NULL) { + return NULL; + } + + size_t size = sizeof(SVgroupsInfo) + sizeof(SCMVgroupInfo) * vgroupList->numOfVgroups; + SVgroupsInfo* pNew = calloc(1, size); + if (pNew == NULL) { + return NULL; + } + + pNew->numOfVgroups = vgroupList->numOfVgroups; + + for(int32_t i = 0; i < vgroupList->numOfVgroups; ++i) { + SCMVgroupInfo* pNewVInfo = &pNew->vgroups[i]; + + SCMVgroupInfo* pvInfo = &vgroupList->vgroups[i]; + pNewVInfo->vgId = pvInfo->vgId; + pNewVInfo->numOfEps = pvInfo->numOfEps; + + for(int32_t j = 0; j < pvInfo->numOfEps; ++j) { + pNewVInfo->epAddr[j].fqdn = strdup(pvInfo->epAddr[j].fqdn); + pNewVInfo->epAddr[j].port = pvInfo->epAddr[j].port; + } + } + + return pNew; +} + +void* tscVgroupInfoClear(SVgroupsInfo *vgroupList) { + if (vgroupList == NULL) { + return NULL; + } + + for(int32_t i = 0; i < vgroupList->numOfVgroups; ++i) { + SCMVgroupInfo* pVgroupInfo = &vgroupList->vgroups[i]; + + for(int32_t j = 0; j < pVgroupInfo->numOfEps; ++j) { + taosTFree(pVgroupInfo->epAddr[j].fqdn); + } + } + + taosTFree(vgroupList); + return NULL; +} + +void tscSCMVgroupInfoCopy(SCMVgroupInfo* dst, const SCMVgroupInfo* src) { + dst->vgId = src->vgId; + dst->numOfEps = src->numOfEps; + for(int32_t i = 0; i < dst->numOfEps; ++i) { + dst->epAddr[i].port = src->epAddr[i].port; + dst->epAddr[i].fqdn = strdup(src->epAddr[i].fqdn); + } +} diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index 9965047a99..143153be49 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -1472,7 +1472,7 @@ static int32_t mnodeProcessSuperTableVgroupMsg(SMnodeMsg *pMsg) { int32_t numOfTable = htonl(pInfo->numOfTables); // reserve space - int32_t contLen = sizeof(SCMSTableVgroupRspMsg) + 32 * sizeof(SCMVgroupInfo) + sizeof(SVgroupsMsg); + int32_t contLen = sizeof(SCMSTableVgroupRspMsg) + 32 * sizeof(SCMVgroupMsg) + sizeof(SVgroupsMsg); for (int32_t i = 0; i < numOfTable; ++i) { char *stableName = (char*)pInfo + sizeof(SCMSTableVgroupMsg) + (TSDB_TABLE_FNAME_LEN) * i; SSuperTableObj *pTable = mnodeGetSuperTable(stableName); @@ -1521,6 +1521,8 @@ static int32_t mnodeProcessSuperTableVgroupMsg(SMnodeMsg *pMsg) { if (pVgroup == NULL) continue; pVgroupMsg->vgroups[vgSize].vgId = htonl(pVgroup->vgId); + pVgroupMsg->vgroups[vgSize].numOfEps = 0; + for (int32_t vn = 0; vn < pVgroup->numOfVnodes; ++vn) { SDnodeObj *pDnode = pVgroup->vnodeGid[vn].pDnode; if (pDnode == NULL) break; From 67ffdd17bef41ef50157bd986aaaa216d6f96b4e Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 19 Oct 2020 23:32:04 +0800 Subject: [PATCH 18/51] [td-1747] --- src/mnode/src/mnodeTable.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index 143153be49..b72addaa97 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -1477,7 +1477,7 @@ static int32_t mnodeProcessSuperTableVgroupMsg(SMnodeMsg *pMsg) { char *stableName = (char*)pInfo + sizeof(SCMSTableVgroupMsg) + (TSDB_TABLE_FNAME_LEN) * i; SSuperTableObj *pTable = mnodeGetSuperTable(stableName); if (pTable != NULL && pTable->vgHash != NULL) { - contLen += (taosHashGetSize(pTable->vgHash) * sizeof(SCMVgroupInfo) + sizeof(SVgroupsMsg)); + contLen += (taosHashGetSize(pTable->vgHash) * sizeof(SCMVgroupMsg) + sizeof(SVgroupsMsg)); } mnodeDecTableRef(pTable); } From 05325532ecd3913d8c589a03440f9eddf2cddc44 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 20 Oct 2020 11:01:00 +0800 Subject: [PATCH 19/51] [td-1747] --- src/client/src/tscSQLParser.c | 20 ------------- src/client/src/tscSchemaUtil.c | 2 +- src/client/src/tscServer.c | 14 +++++---- src/client/src/tscSubquery.c | 9 ------ src/dnode/src/dnodeShell.c | 14 ++++----- src/inc/dnode.h | 2 +- src/inc/taosmsg.h | 8 ++--- src/mnode/inc/mnodeDef.h | 2 +- src/mnode/src/mnodeTable.c | 53 +++++++++++++++++----------------- src/mnode/src/mnodeVgroup.c | 10 +++---- src/tsdb/src/tsdbMeta.c | 2 +- src/vnode/src/vnodeWrite.c | 2 +- 12 files changed, 56 insertions(+), 82 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 055d44ff1e..667a3dbe5a 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -5259,26 +5259,6 @@ int32_t parseCreateDBOptions(SSqlCmd* pCmd, SCreateDBInfo* pCreateDbSql) { return TSDB_CODE_SUCCESS; } -//void tscAddTimestampColumn(SQueryInfo* pQueryInfo, int16_t functionId, int16_t tableIndex) { -// // the first column not timestamp column, add it -// SSqlExpr* pExpr = NULL; -// if (tscSqlExprNumOfExprs(pQueryInfo) > 0) { -// pExpr = tscSqlExprGet(pQueryInfo, 0); -// } -// -// if (pExpr == NULL || pExpr->colInfo.colId != PRIMARYKEY_TIMESTAMP_COL_INDEX || pExpr->functionId != functionId) { -// SColumnIndex index = {tableIndex, PRIMARYKEY_TIMESTAMP_COL_INDEX}; -// -// pExpr = tscSqlExprInsert(pQueryInfo, 0, functionId, &index, TSDB_DATA_TYPE_TIMESTAMP, TSDB_KEYSIZE, TSDB_KEYSIZE, false); -// pExpr->colInfo.flag = TSDB_COL_NORMAL; -// -// // NOTE: tag column does not add to source column list -// SColumnList ids = getColumnList(1, tableIndex, PRIMARYKEY_TIMESTAMP_COL_INDEX); -// -// insertResultField(pQueryInfo, 0, &ids, TSDB_KEYSIZE, TSDB_DATA_TYPE_TIMESTAMP, "ts", pExpr); -// } -//} - void addGroupInfoForSubquery(SSqlObj* pParentObj, SSqlObj* pSql, int32_t subClauseIndex, int32_t tableIndex) { SQueryInfo* pParentQueryInfo = tscGetQueryInfoDetail(&pParentObj->cmd, subClauseIndex); diff --git a/src/client/src/tscSchemaUtil.c b/src/client/src/tscSchemaUtil.c index fb9130b6c5..dfd707344c 100644 --- a/src/client/src/tscSchemaUtil.c +++ b/src/client/src/tscSchemaUtil.c @@ -163,7 +163,7 @@ STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg, size_t* size .numOfColumns = pTableMetaMsg->numOfColumns, }; - pTableMeta->id.tid = pTableMetaMsg->sid; + pTableMeta->id.tid = pTableMetaMsg->tid; pTableMeta->id.uid = pTableMetaMsg->uid; SCMVgroupInfo* pVgroupInfo = &pTableMeta->vgroupInfo; diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index ff798a9f99..8d63eb9775 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -1650,7 +1650,7 @@ int tscBuildHeartBeatMsg(SSqlObj *pSql, SSqlInfo *pInfo) { int tscProcessTableMetaRsp(SSqlObj *pSql) { STableMetaMsg *pMetaMsg = (STableMetaMsg *)pSql->res.pRsp; - pMetaMsg->sid = htonl(pMetaMsg->sid); + pMetaMsg->tid = htonl(pMetaMsg->tid); pMetaMsg->sversion = htons(pMetaMsg->sversion); pMetaMsg->tversion = htons(pMetaMsg->tversion); pMetaMsg->vgroup.vgId = htonl(pMetaMsg->vgroup.vgId); @@ -1660,9 +1660,9 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) { pMetaMsg->numOfColumns = htons(pMetaMsg->numOfColumns); if ((pMetaMsg->tableType != TSDB_SUPER_TABLE) && - (pMetaMsg->sid <= 0 || pMetaMsg->vgroup.vgId < 2 || pMetaMsg->vgroup.numOfEps <= 0)) { + (pMetaMsg->tid <= 0 || pMetaMsg->vgroup.vgId < 2 || pMetaMsg->vgroup.numOfEps <= 0)) { tscError("invalid value in table numOfEps:%d, vgId:%d tid:%d, name:%s", pMetaMsg->vgroup.numOfEps, pMetaMsg->vgroup.vgId, - pMetaMsg->sid, pMetaMsg->tableId); + pMetaMsg->tid, pMetaMsg->tableId); return TSDB_CODE_TSC_INVALID_VALUE; } @@ -1845,8 +1845,10 @@ int tscProcessSTableVgroupRsp(SSqlObj *pSql) { SVgroupsMsg * pVgroupMsg = (SVgroupsMsg *) pMsg; pVgroupMsg->numOfVgroups = htonl(pVgroupMsg->numOfVgroups); - size_t size = sizeof(SCMVgroupInfo) * pVgroupMsg->numOfVgroups + sizeof(SVgroupsInfo); - pInfo->vgroupList = calloc(1, size); + size_t size = sizeof(SCMVgroupMsg) * pVgroupMsg->numOfVgroups + sizeof(SVgroupsMsg); + + size_t vgroupsz = sizeof(SCMVgroupInfo) * pVgroupMsg->numOfVgroups + sizeof(SVgroupsInfo); + pInfo->vgroupList = calloc(1, vgroupsz); assert(pInfo->vgroupList != NULL); pInfo->vgroupList->numOfVgroups = pVgroupMsg->numOfVgroups; @@ -1898,7 +1900,7 @@ int tscProcessShowRsp(SSqlObj *pSql) { pMetaMsg->numOfColumns = ntohs(pMetaMsg->numOfColumns); pSchema = pMetaMsg->schema; - pMetaMsg->sid = ntohs(pMetaMsg->sid); + pMetaMsg->tid = ntohs(pMetaMsg->tid); for (int i = 0; i < pMetaMsg->numOfColumns; ++i) { pSchema->bytes = htons(pSchema->bytes); pSchema++; diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 7a2a242df5..4184d05ddc 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -1142,7 +1142,6 @@ static void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code); static SSqlObj *tscCreateSTableSubquery(SSqlObj *pSql, SRetrieveSupport *trsupport, SSqlObj *prevSqlObj); -// TODO int32_t tscCreateJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSupporter *pSupporter) { SSqlCmd * pCmd = &pSql->cmd; SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); @@ -1298,14 +1297,6 @@ void tscHandleMasterJoinQuery(SSqlObj* pSql) { assert((pQueryInfo->type & TSDB_QUERY_TYPE_SUBQUERY) == 0); int32_t code = TSDB_CODE_SUCCESS; - - // todo add test -// SSubqueryState *pState = calloc(1, sizeof(SSubqueryState)); -// if (pState == NULL) { -// code = TSDB_CODE_TSC_OUT_OF_MEMORY; -// goto _error; -// } - pSql->subState.numOfSub = pQueryInfo->numOfTables; bool hasEmptySub = false; diff --git a/src/dnode/src/dnodeShell.c b/src/dnode/src/dnodeShell.c index a5c5d4759b..4c6c2100e0 100644 --- a/src/dnode/src/dnodeShell.c +++ b/src/dnode/src/dnodeShell.c @@ -173,15 +173,15 @@ static int dnodeRetrieveUserAuthInfo(char *user, char *spi, char *encrypt, char return rpcRsp.code; } -void *dnodeSendCfgTableToRecv(int32_t vgId, int32_t sid) { - dDebug("vgId:%d, sid:%d send config table msg to mnode", vgId, sid); +void *dnodeSendCfgTableToRecv(int32_t vgId, int32_t tid) { + dDebug("vgId:%d, tid:%d send config table msg to mnode", vgId, tid); int32_t contLen = sizeof(SDMConfigTableMsg); SDMConfigTableMsg *pMsg = rpcMallocCont(contLen); pMsg->dnodeId = htonl(dnodeGetDnodeId()); pMsg->vgId = htonl(vgId); - pMsg->sid = htonl(sid); + pMsg->tid = htonl(tid); SRpcMsg rpcMsg = {0}; rpcMsg.pCont = pMsg; @@ -194,18 +194,18 @@ void *dnodeSendCfgTableToRecv(int32_t vgId, int32_t sid) { if (rpcRsp.code != 0) { rpcFreeCont(rpcRsp.pCont); - dError("vgId:%d, sid:%d failed to config table from mnode", vgId, sid); + dError("vgId:%d, tid:%d failed to config table from mnode", vgId, tid); return NULL; } else { - dInfo("vgId:%d, sid:%d config table msg is received", vgId, sid); + dInfo("vgId:%d, tid:%d config table msg is received", vgId, tid); // delete this after debug finished SMDCreateTableMsg *pTable = rpcRsp.pCont; int16_t numOfColumns = htons(pTable->numOfColumns); int16_t numOfTags = htons(pTable->numOfTags); - int32_t sid = htonl(pTable->sid); + int32_t tableId = htonl(pTable->tid); uint64_t uid = htobe64(pTable->uid); - dInfo("table:%s, numOfColumns:%d numOfTags:%d sid:%d uid:%" PRIu64, pTable->tableId, numOfColumns, numOfTags, sid, uid); + dInfo("table:%s, numOfColumns:%d numOfTags:%d tid:%d uid:%" PRIu64, pTable->tableId, numOfColumns, numOfTags, tableId, uid); return rpcRsp.pCont; } diff --git a/src/inc/dnode.h b/src/inc/dnode.h index 83d2a4ad9c..e84545be17 100644 --- a/src/inc/dnode.h +++ b/src/inc/dnode.h @@ -49,7 +49,7 @@ void dnodeAddClientRspHandle(uint8_t msgType, void (*fp)(SRpcMsg *rpcMsg)); void dnodeSendMsgToDnode(SRpcEpSet *epSet, SRpcMsg *rpcMsg); void dnodeSendMsgToMnodeRecv(SRpcMsg *rpcMsg, SRpcMsg *rpcRsp); void dnodeSendMsgToDnodeRecv(SRpcMsg *rpcMsg, SRpcMsg *rpcRsp, SRpcEpSet *epSet); -void *dnodeSendCfgTableToRecv(int32_t vgId, int32_t sid); +void *dnodeSendCfgTableToRecv(int32_t vgId, int32_t tid); void *dnodeAllocateVnodeWqueue(void *pVnode); void dnodeFreeVnodeWqueue(void *queue); diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index 635e2aba80..2c2d686099 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -251,7 +251,7 @@ typedef struct { int8_t tableType; int16_t numOfColumns; int16_t numOfTags; - int32_t sid; + int32_t tid; int32_t sversion; int32_t tversion; int32_t tagDataLen; @@ -360,7 +360,7 @@ typedef struct { typedef struct { int32_t contLen; int32_t vgId; - int32_t sid; + int32_t tid; uint64_t uid; char tableId[TSDB_TABLE_FNAME_LEN]; } SMDDropTableMsg; @@ -699,7 +699,7 @@ typedef struct STableMetaMsg { int16_t numOfColumns; int16_t sversion; int16_t tversion; - int32_t sid; + int32_t tid; uint64_t uid; SCMVgroupMsg vgroup; SSchema schema[]; @@ -747,7 +747,7 @@ typedef struct { typedef struct { int32_t dnodeId; int32_t vgId; - int32_t sid; + int32_t tid; } SDMConfigTableMsg; typedef struct { diff --git a/src/mnode/inc/mnodeDef.h b/src/mnode/inc/mnodeDef.h index 8a2947dd18..4bc840f026 100644 --- a/src/mnode/inc/mnodeDef.h +++ b/src/mnode/inc/mnodeDef.h @@ -115,7 +115,7 @@ typedef struct { uint64_t suid; int64_t createdTime; int32_t numOfColumns; //used by normal table - int32_t sid; + int32_t tid; int32_t vgId; int32_t sqlLen; int8_t updateEnd[4]; diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index b72addaa97..0a81b47c9b 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -303,7 +303,7 @@ static int32_t mnodeChildTableActionRestored() { SVgObj *pVgroup = mnodeGetVgroup(pTable->vgId); if (pVgroup == NULL) { - mError("ctable:%s, failed to get vgId:%d sid:%d, discard it", pTable->info.tableId, pTable->vgId, pTable->sid); + mError("ctable:%s, failed to get vgId:%d tid:%d, discard it", pTable->info.tableId, pTable->vgId, pTable->tid); pTable->vgId = 0; SSdbOper desc = {.type = SDB_OPER_LOCAL, .pObj = pTable, .table = tsChildTableSdb}; sdbDeleteRow(&desc); @@ -314,7 +314,7 @@ static int32_t mnodeChildTableActionRestored() { if (strcmp(pVgroup->dbName, pDb->name) != 0) { mError("ctable:%s, db:%s not match with vgId:%d db:%s sid:%d, discard it", - pTable->info.tableId, pDb->name, pTable->vgId, pVgroup->dbName, pTable->sid); + pTable->info.tableId, pDb->name, pTable->vgId, pVgroup->dbName, pTable->tid); pTable->vgId = 0; SSdbOper desc = {.type = SDB_OPER_LOCAL, .pObj = pTable, .table = tsChildTableSdb}; sdbDeleteRow(&desc); @@ -771,8 +771,8 @@ static int32_t mnodeProcessDropTableMsg(SMnodeMsg *pMsg) { return mnodeProcessDropSuperTableMsg(pMsg); } else { SChildTableObj *pCTable = (SChildTableObj *)pMsg->pTable; - mInfo("app:%p:%p, table:%s, start to drop ctable, vgId:%d sid:%d uid:%" PRIu64, pMsg->rpcMsg.ahandle, pMsg, - pDrop->tableId, pCTable->vgId, pCTable->sid, pCTable->uid); + mInfo("app:%p:%p, table:%s, start to drop ctable, vgId:%d tid:%d uid:%" PRIu64, pMsg->rpcMsg.ahandle, pMsg, + pDrop->tableId, pCTable->vgId, pCTable->tid, pCTable->uid); return mnodeProcessDropChildTableMsg(pMsg); } } @@ -1479,6 +1479,7 @@ static int32_t mnodeProcessSuperTableVgroupMsg(SMnodeMsg *pMsg) { if (pTable != NULL && pTable->vgHash != NULL) { contLen += (taosHashGetSize(pTable->vgHash) * sizeof(SCMVgroupMsg) + sizeof(SVgroupsMsg)); } + mnodeDecTableRef(pTable); } @@ -1593,7 +1594,7 @@ static void *mnodeBuildCreateChildTableMsg(SCMCreateTableMsg *pMsg, SChildTableO pCreate->vgId = htonl(pTable->vgId); pCreate->tableType = pTable->info.type; pCreate->createdTime = htobe64(pTable->createdTime); - pCreate->sid = htonl(pTable->sid); + pCreate->tid = htonl(pTable->tid); pCreate->sqlDataLen = htonl(pTable->sqlLen); pCreate->uid = htobe64(pTable->uid); @@ -1642,7 +1643,7 @@ static int32_t mnodeDoCreateChildTableFp(SMnodeMsg *pMsg) { assert(pTable); mDebug("app:%p:%p, table:%s, created in mnode, vgId:%d sid:%d, uid:%" PRIu64, pMsg->rpcMsg.ahandle, pMsg, - pTable->info.tableId, pTable->vgId, pTable->sid, pTable->uid); + pTable->info.tableId, pTable->vgId, pTable->tid, pTable->uid); SCMCreateTableMsg *pCreate = pMsg->rpcMsg.pCont; SMDCreateTableMsg *pMDCreate = mnodeBuildCreateChildTableMsg(pCreate, pTable); @@ -1684,7 +1685,7 @@ static int32_t mnodeDoCreateChildTableCb(SMnodeMsg *pMsg, int32_t code) { return TSDB_CODE_MND_ACTION_IN_PROGRESS; } else { mError("app:%p:%p, table:%s, failed to create table sid:%d, uid:%" PRIu64 ", reason:%s", pMsg->rpcMsg.ahandle, pMsg, - pTable->info.tableId, pTable->sid, pTable->uid, tstrerror(code)); + pTable->info.tableId, pTable->tid, pTable->uid, tstrerror(code)); SSdbOper desc = {.type = SDB_OPER_GLOBAL, .pObj = pTable, .table = tsChildTableSdb}; sdbDeleteRow(&desc); return code; @@ -1708,7 +1709,7 @@ static int32_t mnodeDoCreateChildTable(SMnodeMsg *pMsg, int32_t tid) { pTable->info.tableId = strdup(pCreate->tableId); pTable->createdTime = taosGetTimestampMs(); - pTable->sid = tid; + pTable->tid = tid; pTable->vgId = pVgroup->vgId; if (pTable->info.type == TSDB_CHILD_TABLE) { @@ -1722,7 +1723,7 @@ static int32_t mnodeDoCreateChildTable(SMnodeMsg *pMsg, int32_t tid) { } pTable->suid = pMsg->pSTable->uid; - pTable->uid = (((uint64_t)pTable->vgId) << 48) + ((((uint64_t)pTable->sid) & ((1ul << 24) - 1ul)) << 24) + + pTable->uid = (((uint64_t)pTable->vgId) << 48) + ((((uint64_t)pTable->tid) & ((1ul << 24) - 1ul)) << 24) + ((sdbGetVersion() & ((1ul << 16) - 1ul)) << 8) + (taosRand() & ((1ul << 8) - 1ul)); pTable->superTable = pMsg->pSTable; } else { @@ -1730,7 +1731,7 @@ static int32_t mnodeDoCreateChildTable(SMnodeMsg *pMsg, int32_t tid) { int64_t us = taosGetTimestampUs(); pTable->uid = (us << 24) + ((sdbGetVersion() & ((1ul << 16) - 1ul)) << 8) + (taosRand() & ((1ul << 8) - 1ul)); } else { - pTable->uid = (((uint64_t)pTable->vgId) << 48) + ((((uint64_t)pTable->sid) & ((1ul << 24) - 1ul)) << 24) + + pTable->uid = (((uint64_t)pTable->vgId) << 48) + ((((uint64_t)pTable->tid) & ((1ul << 24) - 1ul)) << 24) + ((sdbGetVersion() & ((1ul << 16) - 1ul)) << 8) + (taosRand() & ((1ul << 8) - 1ul)); } @@ -1787,7 +1788,7 @@ static int32_t mnodeDoCreateChildTable(SMnodeMsg *pMsg, int32_t tid) { tstrerror(code)); } else { mDebug("app:%p:%p, table:%s, allocated in vgroup, vgId:%d sid:%d uid:%" PRIu64, pMsg->rpcMsg.ahandle, pMsg, - pTable->info.tableId, pVgroup->vgId, pTable->sid, pTable->uid); + pTable->info.tableId, pVgroup->vgId, pTable->tid, pTable->uid); } return code; @@ -1805,8 +1806,8 @@ static int32_t mnodeProcessCreateChildTableMsg(SMnodeMsg *pMsg) { if (pMsg->retry == 0) { if (pMsg->pTable == NULL) { SVgObj *pVgroup = NULL; - int32_t sid = 0; - code = mnodeGetAvailableVgroup(pMsg, &pVgroup, &sid); + int32_t tid = 0; + code = mnodeGetAvailableVgroup(pMsg, &pVgroup, &tid); if (code != TSDB_CODE_SUCCESS) { mDebug("app:%p:%p, table:%s, failed to get available vgroup, reason:%s", pMsg->rpcMsg.ahandle, pMsg, pCreate->tableId, tstrerror(code)); @@ -1820,7 +1821,7 @@ static int32_t mnodeProcessCreateChildTableMsg(SMnodeMsg *pMsg) { pMsg->pVgroup = pVgroup; mnodeIncVgroupRef(pVgroup); - return mnodeDoCreateChildTable(pMsg, sid); + return mnodeDoCreateChildTable(pMsg, tid); } } else { if (pMsg->pTable == NULL) pMsg->pTable = mnodeGetTable(pCreate->tableId); @@ -1850,13 +1851,13 @@ static int32_t mnodeSendDropChildTableMsg(SMnodeMsg *pMsg, bool needReturn) { tstrncpy(pDrop->tableId, pTable->info.tableId, TSDB_TABLE_FNAME_LEN); pDrop->vgId = htonl(pTable->vgId); pDrop->contLen = htonl(sizeof(SMDDropTableMsg)); - pDrop->sid = htonl(pTable->sid); + pDrop->tid = htonl(pTable->tid); pDrop->uid = htobe64(pTable->uid); SRpcEpSet epSet = mnodeGetEpSetFromVgroup(pMsg->pVgroup); mInfo("app:%p:%p, ctable:%s, send drop ctable msg, vgId:%d sid:%d uid:%" PRIu64, pMsg->rpcMsg.ahandle, pMsg, - pDrop->tableId, pTable->vgId, pTable->sid, pTable->uid); + pDrop->tableId, pTable->vgId, pTable->tid, pTable->uid); SRpcMsg rpcMsg = { .ahandle = pMsg, @@ -2095,7 +2096,7 @@ static int32_t mnodeDoGetChildTableMeta(SMnodeMsg *pMsg, STableMetaMsg *pMeta) { SChildTableObj *pTable = (SChildTableObj *)pMsg->pTable; pMeta->uid = htobe64(pTable->uid); - pMeta->sid = htonl(pTable->sid); + pMeta->tid = htonl(pTable->tid); pMeta->precision = pDb->cfg.precision; pMeta->tableType = pTable->info.type; tstrncpy(pMeta->tableId, pTable->info.tableId, TSDB_TABLE_FNAME_LEN); @@ -2135,7 +2136,7 @@ static int32_t mnodeDoGetChildTableMeta(SMnodeMsg *pMsg, STableMetaMsg *pMeta) { pMeta->vgroup.vgId = htonl(pMsg->pVgroup->vgId); mDebug("app:%p:%p, table:%s, uid:%" PRIu64 " table meta is retrieved, vgId:%d sid:%d", pMsg->rpcMsg.ahandle, pMsg, - pTable->info.tableId, pTable->uid, pTable->vgId, pTable->sid); + pTable->info.tableId, pTable->uid, pTable->vgId, pTable->tid); return TSDB_CODE_SUCCESS; } @@ -2283,11 +2284,11 @@ static void mnodeDropAllChildTablesInStable(SSuperTableObj *pStable) { } #if 0 -static SChildTableObj* mnodeGetTableByPos(int32_t vnode, int32_t sid) { +static SChildTableObj* mnodeGetTableByPos(int32_t vnode, int32_t tid) { SVgObj *pVgroup = mnodeGetVgroup(vnode); if (pVgroup == NULL) return NULL; - SChildTableObj *pTable = pVgroup->tableList[sid - 1]; + SChildTableObj *pTable = pVgroup->tableList[tid - 1]; mnodeIncTableRef((STableObj *)pTable); mnodeDecVgroupRef(pVgroup); @@ -2335,12 +2336,12 @@ static void mnodeProcessDropChildTableRsp(SRpcMsg *rpcMsg) { assert(pTable); mInfo("app:%p:%p, table:%s, drop table rsp received, vgId:%d sid:%d uid:%" PRIu64 ", thandle:%p result:%s", - mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId, pTable->vgId, pTable->sid, pTable->uid, + mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId, pTable->vgId, pTable->tid, pTable->uid, mnodeMsg->rpcMsg.handle, tstrerror(rpcMsg->code)); if (rpcMsg->code != TSDB_CODE_SUCCESS) { mError("app:%p:%p, table:%s, failed to drop in dnode, vgId:%d sid:%d uid:%" PRIu64 ", reason:%s", - mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId, pTable->vgId, pTable->sid, pTable->uid, + mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId, pTable->vgId, pTable->tid, pTable->uid, tstrerror(rpcMsg->code)); dnodeSendRpcMnodeWriteRsp(mnodeMsg, rpcMsg->code); return; @@ -2378,7 +2379,7 @@ static void mnodeProcessCreateChildTableRsp(SRpcMsg *rpcMsg) { // If the table is deleted by another thread during creation, stop creating and send drop msg to vnode if (sdbCheckRowDeleted(tsChildTableSdb, pTable)) { mDebug("app:%p:%p, table:%s, create table rsp received, but a deleting opertion incoming, vgId:%d sid:%d uid:%" PRIu64, - mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId, pTable->vgId, pTable->sid, pTable->uid); + mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId, pTable->vgId, pTable->tid, pTable->uid); // if the vgroup is already dropped from hash, it can't be accquired by pTable->vgId // so the refCount of vgroup can not be decreased @@ -2413,13 +2414,13 @@ static void mnodeProcessCreateChildTableRsp(SRpcMsg *rpcMsg) { if (mnodeMsg->retry++ < 10) { mDebug("app:%p:%p, table:%s, create table rsp received, need retry, times:%d vgId:%d sid:%d uid:%" PRIu64 " result:%s thandle:%p", - mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId, mnodeMsg->retry, pTable->vgId, pTable->sid, + mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId, mnodeMsg->retry, pTable->vgId, pTable->tid, pTable->uid, tstrerror(rpcMsg->code), mnodeMsg->rpcMsg.handle); dnodeDelayReprocessMnodeWriteMsg(mnodeMsg); } else { mError("app:%p:%p, table:%s, failed to create in dnode, vgId:%d sid:%d uid:%" PRIu64 ", result:%s thandle:%p", - mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId, pTable->vgId, pTable->sid, pTable->uid, + mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId, pTable->vgId, pTable->tid, pTable->uid, tstrerror(rpcMsg->code), mnodeMsg->rpcMsg.handle); SSdbOper oper = {.type = SDB_OPER_GLOBAL, .table = tsChildTableSdb, .pObj = pTable}; @@ -2674,7 +2675,7 @@ static int32_t mnodeRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows // tid pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int32_t*) pWrite = pTable->sid; + *(int32_t*) pWrite = pTable->tid; cols++; //vgid diff --git a/src/mnode/src/mnodeVgroup.c b/src/mnode/src/mnodeVgroup.c index 28e4d17920..5084f1276a 100644 --- a/src/mnode/src/mnodeVgroup.c +++ b/src/mnode/src/mnodeVgroup.c @@ -793,12 +793,12 @@ static int32_t mnodeRetrieveVgroups(SShowObj *pShow, char *data, int32_t rows, v void mnodeAddTableIntoVgroup(SVgObj *pVgroup, SChildTableObj *pTable) { int32_t idPoolSize = taosIdPoolMaxSize(pVgroup->idPool); - if (pTable->sid > idPoolSize) { + if (pTable->tid > idPoolSize) { mnodeAllocVgroupIdPool(pVgroup); } - if (pTable->sid >= 1) { - taosIdPoolMarkStatus(pVgroup->idPool, pTable->sid); + if (pTable->tid >= 1) { + taosIdPoolMarkStatus(pVgroup->idPool, pTable->tid); pVgroup->numOfTables++; // The create vgroup message may be received later than the create table message // and the writing order in sdb is therefore uncertain @@ -808,8 +808,8 @@ void mnodeAddTableIntoVgroup(SVgObj *pVgroup, SChildTableObj *pTable) { } void mnodeRemoveTableFromVgroup(SVgObj *pVgroup, SChildTableObj *pTable) { - if (pTable->sid >= 1) { - taosFreeId(pVgroup->idPool, pTable->sid); + if (pTable->tid >= 1) { + taosFreeId(pVgroup->idPool, pTable->tid); pVgroup->numOfTables--; // The create vgroup message may be received later than the create table message // and the writing order in sdb is therefore uncertain diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index 564d7f5db5..f3bd91f038 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -239,7 +239,7 @@ STableCfg *tsdbCreateTableCfgFromMsg(SMDCreateTableMsg *pMsg) { return NULL; } - if (tsdbInitTableCfg(pCfg, pMsg->tableType, htobe64(pMsg->uid), htonl(pMsg->sid)) < 0) goto _err; + if (tsdbInitTableCfg(pCfg, pMsg->tableType, htobe64(pMsg->uid), htonl(pMsg->tid)) < 0) goto _err; if (tdInitTSchemaBuilder(&schemaBuilder, htonl(pMsg->sversion)) < 0) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; goto _err; diff --git a/src/vnode/src/vnodeWrite.c b/src/vnode/src/vnodeWrite.c index 70b08b9669..084e72a734 100644 --- a/src/vnode/src/vnodeWrite.c +++ b/src/vnode/src/vnodeWrite.c @@ -149,7 +149,7 @@ static int32_t vnodeProcessDropTableMsg(SVnodeObj *pVnode, void *pCont, SRspRet int32_t code = TSDB_CODE_SUCCESS; vDebug("vgId:%d, table:%s, start to drop", pVnode->vgId, pTable->tableId); - STableId tableId = {.uid = htobe64(pTable->uid), .tid = htonl(pTable->sid)}; + STableId tableId = {.uid = htobe64(pTable->uid), .tid = htonl(pTable->tid)}; if (tsdbDropTable(pVnode->tsdb, tableId) < 0) code = terrno; From 959c3405a05f74dfa43d026db66f9084eef6da4a Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 20 Oct 2020 11:41:35 +0800 Subject: [PATCH 20/51] [td-1756] add error check --- src/client/src/tscSQLParser.c | 2 +- src/query/inc/qSqlparser.h | 2 -- src/query/src/qParserImpl.c | 27 +++++---------------------- 3 files changed, 6 insertions(+), 25 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 667a3dbe5a..aaf0450da3 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -190,7 +190,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { SSqlRes* pRes = &pSql->res; int32_t code = TSDB_CODE_SUCCESS; - if (!pInfo->valid) { + if (!pInfo->valid || errno == TSDB_CODE_TSC_SQL_SYNTAX_ERROR) { return tscSQLSyntaxErrMsg(tscGetErrorMsgPayload(pCmd), NULL, pInfo->pzErrMsg); } diff --git a/src/query/inc/qSqlparser.h b/src/query/inc/qSqlparser.h index d6664577a3..bc8f9a5e23 100644 --- a/src/query/inc/qSqlparser.h +++ b/src/query/inc/qSqlparser.h @@ -276,8 +276,6 @@ tSQLExpr *tSQLExprNodeClone(tSQLExpr *pExpr); SAlterTableSQL *tAlterTableSQLElems(SStrToken *pMeterName, tFieldList *pCols, tVariantList *pVals, int32_t type); -tSQLExprListList *tSQLListListAppend(tSQLExprListList *pList, tSQLExprList *pExprList); - void destroyAllSelectClause(SSubclauseInfo *pSql); void doDestroyQuerySql(SQuerySQL *pSql); diff --git a/src/query/src/qParserImpl.c b/src/query/src/qParserImpl.c index 33237a58c2..c42a643ab5 100644 --- a/src/query/src/qParserImpl.c +++ b/src/query/src/qParserImpl.c @@ -130,13 +130,15 @@ tSQLExpr *tSQLExprIdValueCreate(SStrToken *pToken, int32_t optrType) { tVariantCreate(&pSQLExpr->val, pToken); pSQLExpr->nSQLOptr = optrType; } else if (optrType == TK_NOW) { - // default use microsecond + // use microsecond by default pSQLExpr->val.i64Key = taosGetTimestamp(TSDB_TIME_PRECISION_MICRO); pSQLExpr->val.nType = TSDB_DATA_TYPE_BIGINT; pSQLExpr->nSQLOptr = TK_TIMESTAMP; // TK_TIMESTAMP used to denote the time value is in microsecond } else if (optrType == TK_VARIABLE) { int32_t ret = parseAbsoluteDuration(pToken->z, pToken->n, &pSQLExpr->val.i64Key); - UNUSED(ret); + if (ret != TSDB_CODE_SUCCESS) { + errno = TSDB_CODE_TSC_SQL_SYNTAX_ERROR; + } pSQLExpr->val.nType = TSDB_DATA_TYPE_BIGINT; pSQLExpr->nSQLOptr = TK_TIMESTAMP; @@ -148,6 +150,7 @@ tSQLExpr *tSQLExprIdValueCreate(SStrToken *pToken, int32_t optrType) { pSQLExpr->nSQLOptr = optrType; } + return pSQLExpr; } @@ -532,26 +535,6 @@ SQuerySQL *tSetQuerySQLElems(SStrToken *pSelectToken, tSQLExprList *pSelection, return pQuery; } -tSQLExprListList *tSQLListListAppend(tSQLExprListList *pList, tSQLExprList *pExprList) { - if (pList == NULL) pList = calloc(1, sizeof(tSQLExprListList)); - - if (pList->nAlloc <= pList->nList) { // - pList->nAlloc = (pList->nAlloc << 1) + 4; - pList->a = realloc(pList->a, pList->nAlloc * sizeof(pList->a[0])); - if (pList->a == 0) { - pList->nList = pList->nAlloc = 0; - return pList; - } - } - assert(pList->a != 0); - - if (pExprList) { - pList->a[pList->nList++] = pExprList; - } - - return pList; -} - void doDestroyQuerySql(SQuerySQL *pQuerySql) { if (pQuerySql == NULL) { return; From d030cd2338db0a724c06fef1ae60781f18f19a4e Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 20 Oct 2020 11:50:29 +0800 Subject: [PATCH 21/51] [td-1756] add test cases. --- tests/script/general/parser/constCol.sim | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/script/general/parser/constCol.sim b/tests/script/general/parser/constCol.sim index 13b4455779..8ffb263ecc 100644 --- a/tests/script/general/parser/constCol.sim +++ b/tests/script/general/parser/constCol.sim @@ -352,5 +352,7 @@ print ======================udc with normal column group by sql_error select from t1 sql_error select abc from t1 sql_error select abc as tu from t1 +sql_error select * from t1 where ts>now-1y +sql_error select * from t1 where ts>now-1n system sh/exec.sh -n dnode1 -s stop -x SIGINT From 38de20a7ede4a1b009b67b3f5b82a2bdb8849b45 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 20 Oct 2020 13:42:51 +0800 Subject: [PATCH 22/51] [td-1756] add test cases. --- tests/script/general/parser/constCol.sim | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/script/general/parser/constCol.sim b/tests/script/general/parser/constCol.sim index 8ffb263ecc..7232851ae9 100644 --- a/tests/script/general/parser/constCol.sim +++ b/tests/script/general/parser/constCol.sim @@ -352,6 +352,8 @@ print ======================udc with normal column group by sql_error select from t1 sql_error select abc from t1 sql_error select abc as tu from t1 + +print ========================> td-1756 sql_error select * from t1 where ts>now-1y sql_error select * from t1 where ts>now-1n From 7c4276cb0502e6e8ec7c5137094826321f97a646 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 20 Oct 2020 13:44:05 +0800 Subject: [PATCH 23/51] [td-1752] --- src/query/src/qAst.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/query/src/qAst.c b/src/query/src/qAst.c index 63411aaf3f..893105e44a 100644 --- a/src/query/src/qAst.c +++ b/src/query/src/qAst.c @@ -646,9 +646,7 @@ static bool filterItem(tExprNode *pExpr, const void *pItem, SExprTraverseSupp *p } // handle the leaf node - assert(pLeft->nodeType == TSQL_NODE_COL && pRight->nodeType == TSQL_NODE_VALUE); param->setupInfoFn(pExpr, param->pExtInfo); - return param->nodeFilterFn(pItem, pExpr->_node.info); } @@ -769,6 +767,7 @@ void tExprTreeTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, S assert(taosArrayGetSize(result) == 0); tSQLBinaryTraverseOnSkipList(pExpr, result, pSkipList, param); } + return; } From 2ed28bf703dd3d620e8c986a9f3ec26d8ac8c5ab Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 20 Oct 2020 13:49:35 +0800 Subject: [PATCH 24/51] [td-1752] add test cases. --- tests/script/general/parser/constCol.sim | 28 ++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tests/script/general/parser/constCol.sim b/tests/script/general/parser/constCol.sim index 7232851ae9..69f4111700 100644 --- a/tests/script/general/parser/constCol.sim +++ b/tests/script/general/parser/constCol.sim @@ -357,4 +357,32 @@ print ========================> td-1756 sql_error select * from t1 where ts>now-1y sql_error select * from t1 where ts>now-1n +print ========================> td-1752 +sql select * from st2 where t2 < 200 and t2 is not null; +if $rows != 1 then + return -1 +endi + +if $data00 != @19-12-09 16:27:35.000@ then + return -1 +endi + +if $data01 != 2 then + return -1 +endi + +if $data02 != 1 then + return -1 +endi + +sql select * from st2 where t2 > 200 or t2 is null; +if $rows != 0 then + return -1 +endi + +sql select * from st2 where t2 < 200 and t2 is null; +if $rows != 0 then + return -1 +endi + system sh/exec.sh -n dnode1 -s stop -x SIGINT From e910cae1ec5d61914f58ba0fb8fd2935a4d079f8 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 20 Oct 2020 14:06:03 +0800 Subject: [PATCH 25/51] [td-1756] --- src/query/src/qParserImpl.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/query/src/qParserImpl.c b/src/query/src/qParserImpl.c index c42a643ab5..7e8128f200 100644 --- a/src/query/src/qParserImpl.c +++ b/src/query/src/qParserImpl.c @@ -137,7 +137,7 @@ tSQLExpr *tSQLExprIdValueCreate(SStrToken *pToken, int32_t optrType) { } else if (optrType == TK_VARIABLE) { int32_t ret = parseAbsoluteDuration(pToken->z, pToken->n, &pSQLExpr->val.i64Key); if (ret != TSDB_CODE_SUCCESS) { - errno = TSDB_CODE_TSC_SQL_SYNTAX_ERROR; + terrno = TSDB_CODE_TSC_SQL_SYNTAX_ERROR; } pSQLExpr->val.nType = TSDB_DATA_TYPE_BIGINT; From beb6599256dd42a6dc5e285e05785f551f340771 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 20 Oct 2020 14:07:28 +0800 Subject: [PATCH 26/51] [td-1756] --- src/client/src/tscSQLParser.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index aaf0450da3..7a8be085eb 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -190,7 +190,8 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { SSqlRes* pRes = &pSql->res; int32_t code = TSDB_CODE_SUCCESS; - if (!pInfo->valid || errno == TSDB_CODE_TSC_SQL_SYNTAX_ERROR) { + if (!pInfo->valid || terrno == TSDB_CODE_TSC_SQL_SYNTAX_ERROR) { + terrno = TSDB_CODE_SUCCESS; // clear the error number return tscSQLSyntaxErrMsg(tscGetErrorMsgPayload(pCmd), NULL, pInfo->pzErrMsg); } From 9f653edacd1a0a374133876cdd79a070fbc3ad14 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 20 Oct 2020 14:08:53 +0800 Subject: [PATCH 27/51] [td-1752] add test cases. --- tests/script/general/parser/constCol.sim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/script/general/parser/constCol.sim b/tests/script/general/parser/constCol.sim index 69f4111700..7ae496f1ac 100644 --- a/tests/script/general/parser/constCol.sim +++ b/tests/script/general/parser/constCol.sim @@ -358,7 +358,7 @@ sql_error select * from t1 where ts>now-1y sql_error select * from t1 where ts>now-1n print ========================> td-1752 -sql select * from st2 where t2 < 200 and t2 is not null; +sql select * from db.st2 where t2 < 200 and t2 is not null; if $rows != 1 then return -1 endi @@ -375,7 +375,7 @@ if $data02 != 1 then return -1 endi -sql select * from st2 where t2 > 200 or t2 is null; +sql select * from db.st2 where t2 > 200 or t2 is null; if $rows != 0 then return -1 endi From e491a8908c4d7ed99076eea5f67d7f257427e4d0 Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Tue, 20 Oct 2020 15:00:12 +0800 Subject: [PATCH 28/51] fix TD-1471 --- src/client/src/tscSql.c | 7 +- .../python/linux/python2/taos/cursor.py | 6 +- .../python/linux/python3/taos/cursor.py | 6 +- .../python/windows/python2/taos/cursor.py | 3 + .../python/windows/python3/taos/cursor.py | 3 + tests/pytest/fulltest.sh | 1 + tests/pytest/query/bug1471.py | 73 +++++++++++++++++++ 7 files changed, 93 insertions(+), 6 deletions(-) create mode 100644 tests/pytest/query/bug1471.py diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index be91255b9c..02a2c1e052 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -479,8 +479,11 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) { SSqlCmd *pCmd = &pSql->cmd; SSqlRes *pRes = &pSql->res; - if (pRes->qhandle == 0 || - pRes->code == TSDB_CODE_TSC_QUERY_CANCELLED || + if (pRes->qhandle == 0) { + pRes->code = TSDB_CODE_TSC_INVALID_QHANDLE; + return NULL; + } + if (pRes->code == TSDB_CODE_TSC_QUERY_CANCELLED || pCmd->command == TSDB_SQL_RETRIEVE_EMPTY_RESULT || pCmd->command == TSDB_SQL_INSERT) { return NULL; diff --git a/src/connector/python/linux/python2/taos/cursor.py b/src/connector/python/linux/python2/taos/cursor.py index 8c268d8afb..37c02d330e 100644 --- a/src/connector/python/linux/python2/taos/cursor.py +++ b/src/connector/python/linux/python2/taos/cursor.py @@ -192,8 +192,10 @@ class TDengineCursor(object): buffer = [[] for i in range(len(self._fields))] self._rowcount = 0 while True: - block, num_of_fields = CTaosInterface.fetchBlock( - self._result, self._fields) + block, num_of_fields = CTaosInterface.fetchBlock(self._result, self._fields) + errno = CTaosInterface.libtaos.taos_errno(self._result) + if errno != 0: + raise ProgrammingError(CTaosInterface.errStr(self._result), errno) if num_of_fields == 0: break self._rowcount += num_of_fields diff --git a/src/connector/python/linux/python3/taos/cursor.py b/src/connector/python/linux/python3/taos/cursor.py index 3f0f315d33..ec7a85ee1a 100644 --- a/src/connector/python/linux/python3/taos/cursor.py +++ b/src/connector/python/linux/python3/taos/cursor.py @@ -207,8 +207,10 @@ class TDengineCursor(object): buffer = [[] for i in range(len(self._fields))] self._rowcount = 0 while True: - block, num_of_fields = CTaosInterface.fetchBlock( - self._result, self._fields) + block, num_of_fields = CTaosInterface.fetchBlock(self._result, self._fields) + errno = CTaosInterface.libtaos.taos_errno(self._result) + if errno != 0: + raise ProgrammingError(CTaosInterface.errStr(self._result), errno) if num_of_fields == 0: break self._rowcount += num_of_fields diff --git a/src/connector/python/windows/python2/taos/cursor.py b/src/connector/python/windows/python2/taos/cursor.py index 7eee3bfc8f..8714fe77cb 100644 --- a/src/connector/python/windows/python2/taos/cursor.py +++ b/src/connector/python/windows/python2/taos/cursor.py @@ -142,6 +142,9 @@ class TDengineCursor(object): self._rowcount = 0 while True: block, num_of_fields = CTaosInterface.fetchBlock(self._result, self._fields) + errno = CTaosInterface.libtaos.taos_errno(self._result) + if errno != 0: + raise ProgrammingError(CTaosInterface.errStr(self._result), errno) if num_of_fields == 0: break self._rowcount += num_of_fields for i in range(len(self._fields)): diff --git a/src/connector/python/windows/python3/taos/cursor.py b/src/connector/python/windows/python3/taos/cursor.py index 5f5aa4e1d7..c2c442b06e 100644 --- a/src/connector/python/windows/python3/taos/cursor.py +++ b/src/connector/python/windows/python3/taos/cursor.py @@ -142,6 +142,9 @@ class TDengineCursor(object): self._rowcount = 0 while True: block, num_of_fields = CTaosInterface.fetchBlock(self._result, self._fields) + errno = CTaosInterface.libtaos.taos_errno(self._result) + if errno != 0: + raise ProgrammingError(CTaosInterface.errStr(self._result), errno) if num_of_fields == 0: break self._rowcount += num_of_fields for i in range(len(self._fields)): diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index fd973b8b76..4b27c42d38 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -149,6 +149,7 @@ python3 ./test.py -f query/queryNullValueTest.py python3 ./test.py -f query/queryInsertValue.py python3 ./test.py -f query/queryConnection.py python3 ./test.py -f query/natualInterval.py +python3 ./test.py -f query/bug1471.py #stream python3 ./test.py -f stream/metric_1.py diff --git a/tests/pytest/query/bug1471.py b/tests/pytest/query/bug1471.py new file mode 100644 index 0000000000..f1cb0bdcdf --- /dev/null +++ b/tests/pytest/query/bug1471.py @@ -0,0 +1,73 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +import time +import threading + + +class myThread(threading.Thread): + def __init__(self, conn): + threading.Thread.__init__(self) + self.event = threading.Event() + self.conn = taos.connect(conn._host, port=conn._port, config=conn._config) + + def run(self): + cur = self.conn.cursor() + self.event.wait() + cur.execute("drop database db") + cur.close() + self.conn.close() + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + for i in range(50): + print("round", i) + thread = myThread(tdSql.cursor._connection) + thread.start() + + tdSql.execute('reset query cache') + tdSql.execute('drop database if exists db') + tdSql.execute('create database db') + tdSql.execute('use db') + tdSql.execute("create table car (ts timestamp, s int)") + tdSql.execute("insert into car values('2020-10-19 17:00:00', 123)") + + thread.event.set() + try: + tdSql.query("select s from car where ts = '2020-10-19 17:00:00'") + except Exception as e: + pass + else: + tdSql.checkData(0, 0, 123) + + thread.join() + time.sleep(0.2) + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From 9fdd251873e87e106c23d74b0295db8ef5edfffd Mon Sep 17 00:00:00 2001 From: songtianyi Date: Tue, 20 Oct 2020 18:59:46 +0800 Subject: [PATCH 29/51] fix: docs and tests typos, taos_fetch_subfields --> taos_fetch_fields, taos_subfields_count --> taos_field_count --- .../tdenginedocs-cn/advanced-features/index.html | 4 ++-- documentation/tdenginedocs-cn/connector/index.html | 6 +++--- documentation/tdenginedocs-en/connector/index.html | 6 +++--- tests/examples/rust/src/bindings.rs | 6 ------ tests/examples/rust/src/subscriber.rs | 10 +++++----- 5 files changed, 13 insertions(+), 19 deletions(-) diff --git a/documentation/tdenginedocs-cn/advanced-features/index.html b/documentation/tdenginedocs-cn/advanced-features/index.html index 9a1b908cd5..b4953b4dd4 100644 --- a/documentation/tdenginedocs-cn/advanced-features/index.html +++ b/documentation/tdenginedocs-cn/advanced-features/index.html @@ -51,8 +51,8 @@ INTERVAL(1M)
  • mseconds:查询数据库更新的时间间隔,单位为毫秒。一般设置为1000毫秒。返回值为指向TDengine_SUB 结构的指针,如果返回为空,表示失败。

  • TAOS_ROW taos_consume(TAOS_SUB *tsub)

    该函数用来获取订阅的结果,用户应用程序将其置于一个无限循环语句。如果数据库有新记录到达,该API将返回该最新的记录。如果没有新的记录,该API将阻塞。如果返回值为空,说明系统出错。参数说明:

    • tsub:taos_subscribe的结构体指针。

  • void taos_unsubscribe(TAOS_SUB *tsub)

    取消订阅。应用程序退出时,务必调用该函数以避免资源泄露。

  • -
  • int taos_num_subfields(TAOS_SUB *tsub)

    获取返回的一行记录中数据包含多少列。

  • -
  • TAOS_FIELD *taos_fetch_subfields(TAOS_SUB *tsub)

    获取每列数据的属性(数据类型、名字、长度),与taos_num_subfileds配合使用,可解析返回的每行数据。

  • +
  • int taos_num_fields(TAOS_SUB *tsub)

    获取返回的一行记录中数据包含多少列。

  • +
  • TAOS_FIELD *taos_fetch_fields(TAOS_SUB *tsub)

    获取每列数据的属性(数据类型、名字、长度),与taos_num_subfileds配合使用,可解析返回的每行数据。

  • 示例代码:请看安装包中的的示范程序

    缓存 (Cache)

    TDengine采用时间驱动缓存管理策略(First-In-First-Out,FIFO),又称为写驱动的缓存管理机制。这种策略有别于读驱动的数据缓存模式(Least-Recent-Use,LRU),直接将最近写入的数据保存在系统的缓存中。当缓存达到临界值的时候,将最早的数据批量写入磁盘。一般意义上来说,对于物联网数据的使用,用户最为关心最近产生的数据,即当前状态。TDengine充分利用了这一特性,将最近到达的(当前状态)数据保存在缓存中。

    diff --git a/documentation/tdenginedocs-cn/connector/index.html b/documentation/tdenginedocs-cn/connector/index.html index 3167c1521f..34ea19813f 100644 --- a/documentation/tdenginedocs-cn/connector/index.html +++ b/documentation/tdenginedocs-cn/connector/index.html @@ -64,9 +64,9 @@

    该API用来获取最新消息,应用程序一般会将其置于一个无限循环语句中。其中参数tsub是taos_subscribe的返回值。如果数据库有新的记录,该API将返回,返回参数是一行记录。如果没有新的记录,该API将阻塞。如果返回值为空,说明系统出错,需要检查系统是否还在正常运行。

  • void taos_unsubscribe(TAOS_SUB *tsub)

    该API用于取消订阅,参数tsub是taos_subscribe的返回值。应用程序退出时,需要调用该API,否则有资源泄露。

  • -
  • int taos_num_subfields(TAOS_SUB *tsub)

    +
  • int taos_num_fields(TAOS_SUB *tsub)

    该API用来获取返回的一排数据中数据的列数

  • -
  • TAOS_FIELD *taos_fetch_subfields(TAOS_RES *res)

    +
  • TAOS_FIELD *taos_fetch_fields(TAOS_RES *res)

    该API用来获取每列数据的属性(数据类型、名字、字节数),与taos_num_subfileds配合使用,可用来解析返回的一排数据。

  • Java Connector

    @@ -259,4 +259,4 @@ conn.close() _ "taosSql" )

    taosSql驱动包内采用cgo模式,调用了TDengine的C/C++同步接口,与TDengine进行交互,因此,在数据库操作执行完成之前,客户端应用将处于阻塞状态。单个数据库连接,在同一时刻只能有一个线程调用API。客户应用可以建立多个连接,进行多线程的数据写入或查询处理。

    -

    更多使用的细节,请参考下载目录中的示例源码。

    回去 \ No newline at end of file +

    更多使用的细节,请参考下载目录中的示例源码。

    回去 diff --git a/documentation/tdenginedocs-en/connector/index.html b/documentation/tdenginedocs-en/connector/index.html index 0f9e6b4717..ea1f75ae00 100644 --- a/documentation/tdenginedocs-en/connector/index.html +++ b/documentation/tdenginedocs-en/connector/index.html @@ -72,9 +72,9 @@ The API is used to start a subscription session by given a handle. The parameter The API used to get the new data from a TDengine server. It should be put in an infinite loop. The parameter tsub is the handle returned by taos_subscribe. If new data are updated, the API will return a row of the result. Otherwise, the API is blocked until new data arrives. If NULL pointer is returned, it means an error occurs.

  • void taos_unsubscribe(TAOS_SUB *tsub) Stop a subscription session by the handle returned by taos_subscribe.

  • -
  • int taos_num_subfields(TAOS_SUB *tsub) +

  • int taos_num_fields(TAOS_SUB *tsub) The API used to get the number of fields in a row.

  • -
  • TAOS_FIELD *taos_fetch_subfields(TAOS_RES *res) +

  • TAOS_FIELD *taos_fetch_fields(TAOS_RES *res) The API used to get the description of each column.

  • Java Connector

    @@ -351,4 +351,4 @@ promise2.then(function(result) { })

    Example

    An example of using the NodeJS connector to create a table with weather data and create and execute queries can be found here (The preferred method for using the connector)

    -

    An example of using the NodeJS connector to achieve the same things but without all the object wrappers that wrap around the data returned to achieve higher functionality can be found here

    Back \ No newline at end of file +

    An example of using the NodeJS connector to achieve the same things but without all the object wrappers that wrap around the data returned to achieve higher functionality can be found here

    Back diff --git a/tests/examples/rust/src/bindings.rs b/tests/examples/rust/src/bindings.rs index b93e833ba0..fc13647130 100644 --- a/tests/examples/rust/src/bindings.rs +++ b/tests/examples/rust/src/bindings.rs @@ -308,12 +308,6 @@ extern "C" { extern "C" { pub fn taos_unsubscribe(tsub: *mut ::std::os::raw::c_void); } -extern "C" { - pub fn taos_subfields_count(tsub: *mut ::std::os::raw::c_void) -> ::std::os::raw::c_int; -} -extern "C" { - pub fn taos_fetch_subfields(tsub: *mut ::std::os::raw::c_void) -> *mut TAOS_FIELD; -} extern "C" { pub fn taos_open_stream( taos: *mut ::std::os::raw::c_void, diff --git a/tests/examples/rust/src/subscriber.rs b/tests/examples/rust/src/subscriber.rs index b6812d7b6e..78c6f5cd8d 100644 --- a/tests/examples/rust/src/subscriber.rs +++ b/tests/examples/rust/src/subscriber.rs @@ -37,16 +37,16 @@ impl Subscriber { println!("subscribed to {} user:{}, db:{}, tb:{}, time:{}, mseconds:{}", host, username, db, table, time, mseconds); - let mut fields = taos_fetch_subfields(tsub); + let mut fields = taos_fetch_fields(tsub); if fields.is_null() { taos_unsubscribe(tsub); - return Err("fetch subfields error") + return Err("fetch fields error") } - let fcount = taos_subfields_count(tsub); + let fcount = taos_field_count(tsub); if fcount == 0 { taos_unsubscribe(tsub); - return Err("subfields count is 0") + return Err("fields count is 0") } Ok(Subscriber{tsub, fields, fcount}) @@ -74,4 +74,4 @@ impl Drop for Subscriber { fn drop(&mut self) { unsafe {taos_unsubscribe(self.tsub);} } -} \ No newline at end of file +} From 74c1842e389ef33b28f48bf8da1452b43fbc0d6f Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Wed, 21 Oct 2020 08:05:48 +0800 Subject: [PATCH 30/51] Update cluster-ch.md --- documentation20/webdocs/markdowndocs/cluster-ch.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/documentation20/webdocs/markdowndocs/cluster-ch.md b/documentation20/webdocs/markdowndocs/cluster-ch.md index 10579df79a..10c28c284c 100644 --- a/documentation20/webdocs/markdowndocs/cluster-ch.md +++ b/documentation20/webdocs/markdowndocs/cluster-ch.md @@ -213,8 +213,6 @@ SHOW MNODES; ## Arbitrator的使用 -如果副本数为偶数,当一个vnode group里一半或超过一半的vnode不工作时,是无法从中选出master的。同理,一半或超过一半的mnode不工作时,是无法选出mnode的master的,因为存在“split brain”问题。为解决这个问题,TDengine引入了arbitrator的概念。Arbitrator模拟一个vnode或mnode在工作,但只简单的负责网络连接,不处理任何数据插入或访问。只要包含arbitrator在内,超过半数的vnode或mnode工作,那么该vnode group或mnode组就可以正常的提供数据插入或查询服务。比如对于副本数为2的情形,如果一个节点A离线,但另外一个节点B正常,而且能连接到arbitrator, 那么节点B就能正常工作。 +如果副本数为偶数,当一个vnode group里一半vnode不工作时,是无法从中选出master的。同理,一半mnode不工作时,是无法选出mnode的master的,因为存在“split brain”问题。为解决这个问题,TDengine引入了arbitrator的概念。Arbitrator模拟一个vnode或mnode在工作,但只简单的负责网络连接,不处理任何数据插入或访问。只要包含arbitrator在内,超过半数的vnode或mnode工作,那么该vnode group或mnode组就可以正常的提供数据插入或查询服务。比如对于副本数为2的情形,如果一个节点A离线,但另外一个节点B正常,而且能连接到arbitrator, 那么节点B就能正常工作。 -下载最新arbitrator及之前版本的安装包,请点击[安装包下载](https://www.taosdata.com/cn/all-downloads/),在TDengine Arbitrator Linux一节中,选择适合的版本下载并安装。 - -TDengine Arbitrator安装包里带有一个执行程序tarbitrator, 找任何一台Linux服务器运行它即可。该程序对系统资源几乎没有要求,只需要保证有网络连接即可。该应用的命令行参数`-p`可以指定其对外服务的端口号,缺省是6042。配置每个taosd实例时,可以在配置文件taos.cfg里将参数arbitrator设置为arbitrator的End Point。如果该参数配置了,当副本数为偶数数,系统将自动连接配置的arbitrator。 +TDengine提供一个执行程序tarbitrator, 找任何一台Linux服务器运行它即可。请点击[安装包下载](https://www.taosdata.com/cn/all-downloads/),在TDengine Arbitrator Linux一节中,选择适合的版本下载并安装。该程序对系统资源几乎没有要求,只需要保证有网络连接即可。该应用的命令行参数`-p`可以指定其对外服务的端口号,缺省是6042。配置每个taosd实例时,可以在配置文件taos.cfg里将参数arbitrator设置为arbitrator的End Point。如果该参数配置了,当副本数为偶数数,系统将自动连接配置的arbitrator。如果副本数为奇数,即使配置了arbitrator, 系统也不会去建立连接。 From eea4bf2129511663177296f50ff7352d1ec6639e Mon Sep 17 00:00:00 2001 From: Xiaowei Su <46439638+Shawshank-Smile@users.noreply.github.com> Date: Wed, 21 Oct 2020 09:23:28 +0800 Subject: [PATCH 31/51] Update taosd-ch.md --- documentation20/webdocs/markdowndocs/taosd-ch.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation20/webdocs/markdowndocs/taosd-ch.md b/documentation20/webdocs/markdowndocs/taosd-ch.md index e90bc2233f..08be0c163e 100644 --- a/documentation20/webdocs/markdowndocs/taosd-ch.md +++ b/documentation20/webdocs/markdowndocs/taosd-ch.md @@ -61,7 +61,7 @@ vnode与其子模块是通过API直接调用,而不是通过消息队列传递 mnode是整个系统的大脑,负责整个系统的资源调度,负责meta data的管理与存储。 -一个运行的系统里,只有一个mnode,但它有多个副本(由系统配置参数numOfMpeers控制)。这些副本分布在不同的dnode里,目的是保证系统的高可靠运行。副本之间的数据复制是采用同步而非异步的方式,以确保数据的一致性,确保数据不会丢失。这些副本会自动选举一个Master,其他副本是slave。所有数据更新类的操作,都只能在master上进行,而查询类的可以在slave节点上进行。代码实现上,同步模块与vnode共享,但mnode被分配一个特殊的vgroup ID: 1,而且quorum大于1。整个集群系统是由多个dnode组成的,运行的mnode的副本数不可能超过dnode的个数,但不会超过配置的副本数。如果某个mnode副本宕机一段时间,只要超过半数的mnode副本仍在运行,运行的mnode会自动根据整个系统的资源情况,在其他dnode里再启动一个mnode, 以保证运行的副本数。 +一个运行的系统里,只有一个mnode,但它有多个副本(由系统配置参数numOfMnodes控制)。这些副本分布在不同的dnode里,目的是保证系统的高可靠运行。副本之间的数据复制是采用同步而非异步的方式,以确保数据的一致性,确保数据不会丢失。这些副本会自动选举一个Master,其他副本是slave。所有数据更新类的操作,都只能在master上进行,而查询类的可以在slave节点上进行。代码实现上,同步模块与vnode共享,但mnode被分配一个特殊的vgroup ID: 1,而且quorum大于1。整个集群系统是由多个dnode组成的,运行的mnode的副本数不可能超过dnode的个数,但不会超过配置的副本数。如果某个mnode副本宕机一段时间,只要超过半数的mnode副本仍在运行,运行的mnode会自动根据整个系统的资源情况,在其他dnode里再启动一个mnode, 以保证运行的副本数。 各个dnode通过信息交换,保存有mnode各个副本的End Point列表,并向其中的master节点定时(间隔由系统配置参数statusInterval控制)发送status消息,消息体里包含该dnode的CPU、内存、剩余存储空间、vnode个数,以及各个vnode的状态(存储空间、原始数据大小、记录条数、角色等)。这样mnode就了解整个系统的资源情况,如果用户创建新的表,就可以决定需要在哪个dnode创建;如果增加或删除dnode, 或者监测到某dnode数据过热、或离线太长,就可以决定需要挪动那些vnode,以实现负载均衡。 From 015f1b82c5b5f78be0b69d3f81543f5cb31473fc Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 21 Oct 2020 10:07:35 +0800 Subject: [PATCH 32/51] [td-1752] add test cases. --- .../parser/col_arithmetic_operation.sim | 18 ++++++++++++++++++ tests/script/general/parser/union.sim | 4 ++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/tests/script/general/parser/col_arithmetic_operation.sim b/tests/script/general/parser/col_arithmetic_operation.sim index 132ecf4342..e1557227d4 100644 --- a/tests/script/general/parser/col_arithmetic_operation.sim +++ b/tests/script/general/parser/col_arithmetic_operation.sim @@ -91,4 +91,22 @@ endi sql_error select max(c2*2) from $tb sql_error select max(c1-c2) from $tb +print =====================> td-1764 +sql select sum(c1)/count(*), sum(c1) as b, count(*) as b from $stb interval(1y) +if $rows != 1 then + return -1 +endi + +if $data00 != @18-01-01 00:00:00.000@ then + return -1 +endi + +if $data01 != 2.250000000 then + return -1 +endi + +if $data02 != 225000 then + return -1 +endi + system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/parser/union.sim b/tests/script/general/parser/union.sim index 9e178537a2..4af482bde0 100644 --- a/tests/script/general/parser/union.sim +++ b/tests/script/general/parser/union.sim @@ -251,7 +251,7 @@ if $rows != 15 then endi # first subclause are empty -sql select count(*) as c from union_tb0 where ts>now+10y union all select sum(c1) as c from union_tb1; +sql select count(*) as c from union_tb0 where ts > now + 3650d union all select sum(c1) as c from union_tb1; if $rows != 1 then return -1 endi @@ -346,7 +346,7 @@ if $data91 != 99 then return -1 endi -#1111111111111111111111111111111111111111111111111 +#================================================================================================= # two aggregated functions for normal tables sql select sum(c1) as a from union_tb0 limit 1 union all select sum(c3) as a from union_tb1 limit 2; if $rows != 2 then From 8e188274f085dc22984a0fda49e0c0f674a13aa4 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 21 Oct 2020 10:08:37 +0800 Subject: [PATCH 33/51] [td-1764] --- src/client/inc/tscUtil.h | 7 +-- src/client/inc/tsclient.h | 13 +++-- src/client/src/tscAsync.c | 4 +- src/client/src/tscLocal.c | 12 ++-- src/client/src/tscSQLParser.c | 37 ++++++------ src/client/src/tscServer.c | 2 +- src/client/src/tscSql.c | 22 ++++++- src/client/src/tscSubquery.c | 6 +- src/client/src/tscUtil.c | 104 +++++++++------------------------- src/inc/taosmsg.h | 1 + 10 files changed, 88 insertions(+), 120 deletions(-) diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index 1bd86714c5..3df493349e 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -149,14 +149,13 @@ int tscAllocPayload(SSqlCmd* pCmd, int size); TAOS_FIELD tscCreateField(int8_t type, const char* name, int16_t bytes); -SFieldSupInfo* tscFieldInfoAppend(SFieldInfo* pFieldInfo, TAOS_FIELD* pField); -SFieldSupInfo* tscFieldInfoInsert(SFieldInfo* pFieldInfo, int32_t index, TAOS_FIELD* field); +SInternalField* tscFieldInfoAppend(SFieldInfo* pFieldInfo, TAOS_FIELD* pField); +SInternalField* tscFieldInfoInsert(SFieldInfo* pFieldInfo, int32_t index, TAOS_FIELD* field); -SFieldSupInfo* tscFieldInfoGetSupp(SFieldInfo* pFieldInfo, int32_t index); +SInternalField* tscFieldInfoGetInternalField(SFieldInfo* pFieldInfo, int32_t index); TAOS_FIELD* tscFieldInfoGetField(SFieldInfo* pFieldInfo, int32_t index); void tscFieldInfoUpdateOffset(SQueryInfo* pQueryInfo); -void tscFieldInfoCopy(SFieldInfo* dst, const SFieldInfo* src); void tscFieldInfoUpdateOffsetForInterResult(SQueryInfo* pQueryInfo); int16_t tscFieldInfoGetOffset(SQueryInfo* pQueryInfo, int32_t index); diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index 5d9abf852e..a3a58a4e0f 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -142,16 +142,17 @@ typedef struct SColumnIndex { int16_t columnIndex; } SColumnIndex; -typedef struct SFieldSupInfo { +typedef struct SInternalField { + TAOS_FIELD field; bool visible; SExprInfo *pArithExprInfo; SSqlExpr *pSqlExpr; -} SFieldSupInfo; +} SInternalField; typedef struct SFieldInfo { - int16_t numOfOutput; // number of column in result - SArray *pFields; // SArray - SArray *pSupportInfo; // SArray + int16_t numOfOutput; // number of column in result + TAOS_FIELD* final; + SArray *internalField; // SArray } SFieldInfo; typedef struct SColumn { @@ -469,7 +470,7 @@ int32_t tscSQLSyntaxErrMsg(char* msg, const char* additionalInfo, const char* s int32_t tscToSQLCmd(SSqlObj *pSql, struct SSqlInfo *pInfo); static FORCE_INLINE void tscGetResultColumnChr(SSqlRes* pRes, SFieldInfo* pFieldInfo, int32_t columnIndex) { - SFieldSupInfo* pInfo = (SFieldSupInfo*) TARRAY_GET_ELEM(pFieldInfo->pSupportInfo, columnIndex); + SInternalField* pInfo = (SInternalField*) TARRAY_GET_ELEM(pFieldInfo->internalField, columnIndex); assert(pInfo->pSqlExpr != NULL); int32_t type = pInfo->pSqlExpr->resType; diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c index 003eddc8d2..c996bb2a76 100644 --- a/src/client/src/tscAsync.c +++ b/src/client/src/tscAsync.c @@ -327,7 +327,7 @@ void tscAsyncFetchSingleRowProxy(void *param, TAOS_RES *tres, int numOfRows) { } for (int i = 0; i < pCmd->numOfCols; ++i){ - SFieldSupInfo* pSup = taosArrayGet(pQueryInfo->fieldsInfo.pSupportInfo, i); + SInternalField* pSup = taosArrayGet(pQueryInfo->fieldsInfo.internalField, i); if (pSup->pSqlExpr != NULL) { // pRes->tsrow[i] = TSC_GET_RESPTR_BASE(pRes, pQueryInfo, i) + pSup->pSqlExpr->resBytes * pRes->row; } else { @@ -348,7 +348,7 @@ void tscProcessFetchRow(SSchedMsg *pMsg) { SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); for (int i = 0; i < pCmd->numOfCols; ++i) { - SFieldSupInfo* pSup = taosArrayGet(pQueryInfo->fieldsInfo.pSupportInfo, i); + SInternalField* pSup = taosArrayGet(pQueryInfo->fieldsInfo.internalField, i); if (pSup->pSqlExpr != NULL) { tscGetResultColumnChr(pRes, &pQueryInfo->fieldsInfo, i); diff --git a/src/client/src/tscLocal.c b/src/client/src/tscLocal.c index a15673bdb1..ddab61fb45 100644 --- a/src/client/src/tscLocal.c +++ b/src/client/src/tscLocal.c @@ -239,7 +239,7 @@ static int32_t tscBuildTableSchemaResultFields(SSqlObj *pSql, int32_t numOfCols, TAOS_FIELD f = {.type = TSDB_DATA_TYPE_BINARY, .bytes = (TSDB_COL_NAME_LEN - 1) + VARSTR_HEADER_SIZE}; tstrncpy(f.name, "Field", sizeof(f.name)); - SFieldSupInfo* pInfo = tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); + SInternalField* pInfo = tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); pInfo->pSqlExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY, (TSDB_COL_NAME_LEN - 1) + VARSTR_HEADER_SIZE, (TSDB_COL_NAME_LEN - 1), false); @@ -485,7 +485,7 @@ static int32_t tscSCreateBuildResultFields(SSqlObj *pSql, BuildType type, const tstrncpy(f.name, "Database", sizeof(f.name)); } - SFieldSupInfo* pInfo = tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); + SInternalField* pInfo = tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); pInfo->pSqlExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, TSDB_DATA_TYPE_BINARY, f.bytes, f.bytes - VARSTR_HEADER_SIZE, false); @@ -922,19 +922,17 @@ void tscSetLocalQueryResult(SSqlObj *pSql, const char *val, const char *columnNa pQueryInfo->order.order = TSDB_ORDER_ASC; tscFieldInfoClear(&pQueryInfo->fieldsInfo); - pQueryInfo->fieldsInfo.pFields = taosArrayInit(1, sizeof(TAOS_FIELD)); - pQueryInfo->fieldsInfo.pSupportInfo = taosArrayInit(1, sizeof(SFieldSupInfo)); + pQueryInfo->fieldsInfo.internalField = taosArrayInit(1, sizeof(SInternalField)); TAOS_FIELD f = tscCreateField((int8_t)type, columnName, (int16_t)valueLength); tscFieldInfoAppend(&pQueryInfo->fieldsInfo, &f); tscInitResObjForLocalQuery(pSql, 1, (int32_t)valueLength); - TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 0); - SFieldSupInfo* pInfo = tscFieldInfoGetSupp(&pQueryInfo->fieldsInfo, 0); + SInternalField* pInfo = tscFieldInfoGetInternalField(&pQueryInfo->fieldsInfo, 0); pInfo->pSqlExpr = taosArrayGetP(pQueryInfo->exprList, 0); - memcpy(pRes->data, val, pField->bytes); + memcpy(pRes->data, val, pInfo->field.bytes); } int tscProcessLocalCmd(SSqlObj *pSql) { diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 7a8be085eb..c8e80790ef 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -122,7 +122,7 @@ static int32_t doCheckForCreateTable(SSqlObj* pSql, int32_t subClauseIndex, SSql static int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo); static int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo); static int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index); -static int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSQLExpr* pSqlExpr, SArray* pExprInfo, SQueryInfo* pQueryInfo, SArray* pCols); +static int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSQLExpr* pSqlExpr, SQueryInfo* pQueryInfo, SArray* pCols, int64_t *uid); /* * Used during parsing query sql. Since the query sql usually small in length, error position @@ -1250,7 +1250,7 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t tExprNode* pNode = NULL; SArray* colList = taosArrayInit(10, sizeof(SColIndex)); - int32_t ret = exprTreeFromSqlExpr(pCmd, &pNode, pItem->pNode, pQueryInfo->exprList, pQueryInfo, colList); + int32_t ret = exprTreeFromSqlExpr(pCmd, &pNode, pItem->pNode, pQueryInfo, colList, NULL); if (ret != TSDB_CODE_SUCCESS) { taosArrayDestroy(colList); tExprTreeDestroy(&pNode, NULL); @@ -1306,17 +1306,17 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t insertResultField(pQueryInfo, exprIndex, &columnList, sizeof(double), TSDB_DATA_TYPE_DOUBLE, aliasName, NULL); int32_t slot = tscNumOfFields(pQueryInfo) - 1; - SFieldSupInfo* pInfo = tscFieldInfoGetSupp(&pQueryInfo->fieldsInfo, slot); + SInternalField* pInfo = tscFieldInfoGetInternalField(&pQueryInfo->fieldsInfo, slot); if (pInfo->pSqlExpr == NULL) { SExprInfo* pArithExprInfo = calloc(1, sizeof(SExprInfo)); // arithmetic expression always return result in the format of double float - pArithExprInfo->bytes = sizeof(double); + pArithExprInfo->bytes = sizeof(double); pArithExprInfo->interBytes = sizeof(double); - pArithExprInfo->type = TSDB_DATA_TYPE_DOUBLE; + pArithExprInfo->type = TSDB_DATA_TYPE_DOUBLE; - int32_t ret = exprTreeFromSqlExpr(pCmd, &pArithExprInfo->pExpr, pItem->pNode, pQueryInfo->exprList, pQueryInfo, NULL); + int32_t ret = exprTreeFromSqlExpr(pCmd, &pArithExprInfo->pExpr, pItem->pNode, pQueryInfo, NULL, &pArithExprInfo->uid); if (ret != TSDB_CODE_SUCCESS) { tExprTreeDestroy(&pArithExprInfo->pExpr, NULL); return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), "invalid expression in select clause"); @@ -1373,7 +1373,7 @@ static void addPrimaryTsColIntoResult(SQueryInfo* pQueryInfo) { int32_t numOfCols = (int32_t)tscSqlExprNumOfExprs(pQueryInfo); tscAddSpecialColumnForSelect(pQueryInfo, numOfCols, TSDB_FUNC_PRJ, &index, pSchema, TSDB_COL_NORMAL); - SFieldSupInfo* pSupInfo = tscFieldInfoGetSupp(&pQueryInfo->fieldsInfo, numOfCols); + SInternalField* pSupInfo = tscFieldInfoGetInternalField(&pQueryInfo->fieldsInfo, numOfCols); pSupInfo->visible = false; pQueryInfo->type |= TSDB_QUERY_TYPE_PROJECTION_QUERY; @@ -1470,7 +1470,7 @@ int32_t insertResultField(SQueryInfo* pQueryInfo, int32_t outputIndex, SColumnLi } TAOS_FIELD f = tscCreateField(type, fieldName, bytes); - SFieldSupInfo* pInfo = tscFieldInfoInsert(&pQueryInfo->fieldsInfo, outputIndex, &f); + SInternalField* pInfo = tscFieldInfoInsert(&pQueryInfo->fieldsInfo, outputIndex, &f); pInfo->pSqlExpr = pSqlExpr; return TSDB_CODE_SUCCESS; @@ -4063,7 +4063,7 @@ static int32_t getTagQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SCondE tExprNode* p = NULL; SArray* colList = taosArrayInit(10, sizeof(SColIndex)); - ret = exprTreeFromSqlExpr(pCmd, &p, p1, NULL, pQueryInfo, colList); + ret = exprTreeFromSqlExpr(pCmd, &p, p1, pQueryInfo, colList, NULL); SBufferWriter bw = tbufInitWriter(NULL, false); TRY(0) { @@ -5316,7 +5316,7 @@ void doAddGroupColumnForSubquery(SQueryInfo* pQueryInfo, int32_t tagIndex) { tscAddSpecialColumnForSelect(pQueryInfo, (int32_t)size, TSDB_FUNC_PRJ, &colIndex, pSchema, TSDB_COL_NORMAL); - SFieldSupInfo* pInfo = tscFieldInfoGetSupp(&pQueryInfo->fieldsInfo, (int32_t)size); + SInternalField* pInfo = tscFieldInfoGetInternalField(&pQueryInfo->fieldsInfo, (int32_t)size); doLimitOutputNormalColOfGroupby(pInfo->pSqlExpr); pInfo->visible = false; } @@ -6361,19 +6361,19 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { return TSDB_CODE_SUCCESS; // Does not build query message here } -int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSQLExpr* pSqlExpr, SArray* pExprInfo, SQueryInfo* pQueryInfo, SArray* pCols) { +int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSQLExpr* pSqlExpr, SQueryInfo* pQueryInfo, SArray* pCols, int64_t *uid) { tExprNode* pLeft = NULL; tExprNode* pRight= NULL; if (pSqlExpr->pLeft != NULL) { - int32_t ret = exprTreeFromSqlExpr(pCmd, &pLeft, pSqlExpr->pLeft, pExprInfo, pQueryInfo, pCols); + int32_t ret = exprTreeFromSqlExpr(pCmd, &pLeft, pSqlExpr->pLeft, pQueryInfo, pCols, uid); if (ret != TSDB_CODE_SUCCESS) { return ret; } } if (pSqlExpr->pRight != NULL) { - int32_t ret = exprTreeFromSqlExpr(pCmd, &pRight, pSqlExpr->pRight, pExprInfo, pQueryInfo, pCols); + int32_t ret = exprTreeFromSqlExpr(pCmd, &pRight, pSqlExpr->pRight, pQueryInfo, pCols, uid); if (ret != TSDB_CODE_SUCCESS) { return ret; } @@ -6400,14 +6400,19 @@ int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSQLExpr* pS strncpy((*pExpr)->pSchema->name, pSqlExpr->operand.z, pSqlExpr->operand.n); // set the input column data byte and type. - size_t size = taosArrayGetSize(pExprInfo); + size_t size = taosArrayGetSize(pQueryInfo->exprList); for (int32_t i = 0; i < size; ++i) { - SSqlExpr* p1 = taosArrayGetP(pExprInfo, i); + SSqlExpr* p1 = taosArrayGetP(pQueryInfo->exprList, i); if (strcmp((*pExpr)->pSchema->name, p1->aliasName) == 0) { - (*pExpr)->pSchema->type = (uint8_t)p1->resType; + (*pExpr)->pSchema->type = (uint8_t)p1->resType; (*pExpr)->pSchema->bytes = p1->resBytes; + + if (uid != NULL) { + *uid = p1->uid; + } + break; } } diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 8d63eb9775..966eee6b01 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -1934,7 +1934,7 @@ int tscProcessShowRsp(SSqlObj *pSql) { tscColumnListInsert(pQueryInfo->colList, &index); TAOS_FIELD f = tscCreateField(pSchema->type, pSchema->name, pSchema->bytes); - SFieldSupInfo* pInfo = tscFieldInfoAppend(pFieldInfo, &f); + SInternalField* pInfo = tscFieldInfoAppend(pFieldInfo, &f); pInfo->pSqlExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_TS_DUMMY, &index, pTableSchema[i].type, pTableSchema[i].bytes, pTableSchema[i].bytes, false); diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index be91255b9c..e99297ef16 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -370,7 +370,7 @@ int taos_num_fields(TAOS_RES *res) { size_t numOfCols = tscNumOfFields(pQueryInfo); for(int32_t i = 0; i < numOfCols; ++i) { - SFieldSupInfo* pInfo = taosArrayGet(pQueryInfo->fieldsInfo.pSupportInfo, i); + SInternalField* pInfo = taosArrayGet(pQueryInfo->fieldsInfo.internalField, i); if (pInfo->visible) { num++; } @@ -406,8 +406,24 @@ TAOS_FIELD *taos_fetch_fields(TAOS_RES *res) { if (numOfCols == 0) { return NULL; } - - return pQueryInfo->fieldsInfo.pFields->pData; + + SFieldInfo *pFieldInfo = &pQueryInfo->fieldsInfo; + + if (pFieldInfo->final == NULL) { + TAOS_FIELD* f = calloc(pFieldInfo->numOfOutput, sizeof(TAOS_FIELD)); + + int32_t j = 0; + for(int32_t i = 0; i < pFieldInfo->numOfOutput; ++i) { + SInternalField* pField = tscFieldInfoGetInternalField(pFieldInfo, i); + if (pField->visible) { + f[j++] = pField->field; + } + } + + pFieldInfo->final = f; + } + + return pFieldInfo->final; } int taos_retrieve(TAOS_RES *res) { diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 4184d05ddc..90490f07d8 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -2013,7 +2013,7 @@ int32_t tscHandleMultivnodeInsert(SSqlObj *pSql) { static char* getResultBlockPosition(SSqlCmd* pCmd, SSqlRes* pRes, int32_t columnIndex, int16_t* bytes) { SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); - SFieldSupInfo* pInfo = (SFieldSupInfo*) TARRAY_GET_ELEM(pQueryInfo->fieldsInfo.pSupportInfo, columnIndex); + SInternalField* pInfo = (SInternalField*) TARRAY_GET_ELEM(pQueryInfo->fieldsInfo.internalField, columnIndex); assert(pInfo->pSqlExpr != NULL); *bytes = pInfo->pSqlExpr->resBytes; @@ -2163,7 +2163,7 @@ void **doSetResultRowData(SSqlObj *pSql, bool finalResult) { size_t size = tscNumOfFields(pQueryInfo); for (int i = 0; i < size; ++i) { - SFieldSupInfo* pSup = TARRAY_GET_ELEM(pQueryInfo->fieldsInfo.pSupportInfo, i); + SInternalField* pSup = TARRAY_GET_ELEM(pQueryInfo->fieldsInfo.internalField, i); if (pSup->pSqlExpr != NULL) { tscGetResultColumnChr(pRes, &pQueryInfo->fieldsInfo, i); } @@ -2173,7 +2173,7 @@ void **doSetResultRowData(SSqlObj *pSql, bool finalResult) { continue; } - TAOS_FIELD *pField = TARRAY_GET_ELEM(pQueryInfo->fieldsInfo.pFields, i); + TAOS_FIELD *pField = TARRAY_GET_ELEM(pQueryInfo->fieldsInfo.internalField, i); if (pRes->tsrow[i] != NULL && pField->type == TSDB_DATA_TYPE_NCHAR) { transferNcharData(pSql, i, pField); } diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 9e861d3f28..d35fb21c57 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -847,35 +847,30 @@ TAOS_FIELD tscCreateField(int8_t type, const char* name, int16_t bytes) { return f; } -SFieldSupInfo* tscFieldInfoAppend(SFieldInfo* pFieldInfo, TAOS_FIELD* pField) { +SInternalField* tscFieldInfoAppend(SFieldInfo* pFieldInfo, TAOS_FIELD* pField) { assert(pFieldInfo != NULL); - taosArrayPush(pFieldInfo->pFields, pField); pFieldInfo->numOfOutput++; - struct SFieldSupInfo info = { + struct SInternalField info = { .pSqlExpr = NULL, .pArithExprInfo = NULL, .visible = true, }; - - return taosArrayPush(pFieldInfo->pSupportInfo, &info); + + info.field = *pField; + return taosArrayPush(pFieldInfo->internalField, &info); } -SFieldSupInfo* tscFieldInfoGetSupp(SFieldInfo* pFieldInfo, int32_t index) { - return TARRAY_GET_ELEM(pFieldInfo->pSupportInfo, index); -} - -SFieldSupInfo* tscFieldInfoInsert(SFieldInfo* pFieldInfo, int32_t index, TAOS_FIELD* field) { - taosArrayInsert(pFieldInfo->pFields, index, field); +SInternalField* tscFieldInfoInsert(SFieldInfo* pFieldInfo, int32_t index, TAOS_FIELD* field) { pFieldInfo->numOfOutput++; - - struct SFieldSupInfo info = { + struct SInternalField info = { .pSqlExpr = NULL, .pArithExprInfo = NULL, .visible = true, }; - - return taosArrayInsert(pFieldInfo->pSupportInfo, index, &info); + + info.field = *field; + return taosArrayInsert(pFieldInfo->internalField, index, &info); } void tscFieldInfoUpdateOffset(SQueryInfo* pQueryInfo) { @@ -909,29 +904,18 @@ void tscFieldInfoUpdateOffsetForInterResult(SQueryInfo* pQueryInfo) { } } -void tscFieldInfoCopy(SFieldInfo* dst, const SFieldInfo* src) { - dst->numOfOutput = src->numOfOutput; - - if (dst->pFields == NULL) { - dst->pFields = taosArrayClone(src->pFields); - } else { - taosArrayCopy(dst->pFields, src->pFields); - } - - if (dst->pSupportInfo == NULL) { - dst->pSupportInfo = taosArrayClone(src->pSupportInfo); - } else { - taosArrayCopy(dst->pSupportInfo, src->pSupportInfo); - } +SInternalField* tscFieldInfoGetInternalField(SFieldInfo* pFieldInfo, int32_t index) { + assert(index < pFieldInfo->numOfOutput); + return TARRAY_GET_ELEM(pFieldInfo->internalField, index); } TAOS_FIELD* tscFieldInfoGetField(SFieldInfo* pFieldInfo, int32_t index) { assert(index < pFieldInfo->numOfOutput); - return TARRAY_GET_ELEM(pFieldInfo->pFields, index); + return &((SInternalField*)TARRAY_GET_ELEM(pFieldInfo->internalField, index))->field; } int16_t tscFieldInfoGetOffset(SQueryInfo* pQueryInfo, int32_t index) { - SFieldSupInfo* pInfo = tscFieldInfoGetSupp(&pQueryInfo->fieldsInfo, index); + SInternalField* pInfo = tscFieldInfoGetInternalField(&pQueryInfo->fieldsInfo, index); assert(pInfo != NULL && pInfo->pSqlExpr != NULL); return pInfo->pSqlExpr->offset; @@ -978,10 +962,8 @@ void tscFieldInfoClear(SFieldInfo* pFieldInfo) { return; } - taosArrayDestroy(pFieldInfo->pFields); - for(int32_t i = 0; i < pFieldInfo->numOfOutput; ++i) { - SFieldSupInfo* pInfo = taosArrayGet(pFieldInfo->pSupportInfo, i); + SInternalField* pInfo = taosArrayGet(pFieldInfo->internalField, i); if (pInfo->pArithExprInfo != NULL) { tExprTreeDestroy(&pInfo->pArithExprInfo->pExpr, NULL); @@ -989,7 +971,7 @@ void tscFieldInfoClear(SFieldInfo* pFieldInfo) { } } - taosArrayDestroy(pFieldInfo->pSupportInfo); + taosArrayDestroy(pFieldInfo->internalField); memset(pFieldInfo, 0, sizeof(SFieldInfo)); } @@ -1633,11 +1615,8 @@ STableMetaInfo* tscGetTableMetaInfoByUid(SQueryInfo* pQueryInfo, uint64_t uid, i } void tscInitQueryInfo(SQueryInfo* pQueryInfo) { - assert(pQueryInfo->fieldsInfo.pFields == NULL); - pQueryInfo->fieldsInfo.pFields = taosArrayInit(4, sizeof(TAOS_FIELD)); - - assert(pQueryInfo->fieldsInfo.pSupportInfo == NULL); - pQueryInfo->fieldsInfo.pSupportInfo = taosArrayInit(4, sizeof(SFieldSupInfo)); + assert(pQueryInfo->fieldsInfo.internalField == NULL); + pQueryInfo->fieldsInfo.internalField = taosArrayInit(4, sizeof(SInternalField)); assert(pQueryInfo->exprList == NULL); pQueryInfo->exprList = taosArrayInit(4, POINTER_BYTES); @@ -1849,54 +1828,23 @@ SSqlObj* createSimpleSubObj(SSqlObj* pSql, void (*fp)(), void* param, int32_t cm return pNew; } -// current sql function is not direct output result, so create a dummy output field -static void doSetNewFieldInfo(SQueryInfo* pNewQueryInfo, SSqlExpr* pExpr) { - TAOS_FIELD f = {.type = (uint8_t)pExpr->resType, .bytes = pExpr->resBytes}; - tstrncpy(f.name, pExpr->aliasName, sizeof(f.name)); - - SFieldSupInfo* pInfo1 = tscFieldInfoAppend(&pNewQueryInfo->fieldsInfo, &f); - - pInfo1->pSqlExpr = pExpr; - pInfo1->visible = false; -} - static void doSetSqlExprAndResultFieldInfo(SQueryInfo* pQueryInfo, SQueryInfo* pNewQueryInfo, int64_t uid) { int32_t numOfOutput = (int32_t)tscSqlExprNumOfExprs(pNewQueryInfo); if (numOfOutput == 0) { return; } - size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo); - SFieldInfo* pFieldInfo = &pQueryInfo->fieldsInfo; - // set the field info in pNewQueryInfo object + size_t numOfExprs = tscSqlExprNumOfExprs(pNewQueryInfo); for (int32_t i = 0; i < numOfExprs; ++i) { - SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); + SSqlExpr* pExpr = tscSqlExprGet(pNewQueryInfo, i); - if (pExpr->uid == uid) { - if (i < pFieldInfo->numOfOutput) { - SFieldSupInfo* pInfo = tscFieldInfoGetSupp(pFieldInfo, i); - - if (pInfo->pSqlExpr != NULL) { - TAOS_FIELD* p = tscFieldInfoGetField(pFieldInfo, i); - assert(strcmp(p->name, pExpr->aliasName) == 0); - - SFieldSupInfo* pInfo1 = tscFieldInfoAppend(&pNewQueryInfo->fieldsInfo, p); - *pInfo1 = *pInfo; - } else { - assert(pInfo->pArithExprInfo != NULL); - doSetNewFieldInfo(pNewQueryInfo, pExpr); - } - } else { // it is a arithmetic column, does not have actual field for sqlExpr, so build it - doSetNewFieldInfo(pNewQueryInfo, pExpr); - } - } + TAOS_FIELD f = tscCreateField(pExpr->resType, pExpr->aliasName, pExpr->resBytes); + SInternalField* pInfo1 = tscFieldInfoAppend(&pNewQueryInfo->fieldsInfo, &f); + pInfo1->pSqlExpr = pExpr; } - // make sure the the sqlExpr for each fields is correct - numOfExprs = tscSqlExprNumOfExprs(pNewQueryInfo); - - // update the pSqlExpr pointer in SFieldSupInfo according the field name + // update the pSqlExpr pointer in SInternalField according the field name // make sure the pSqlExpr point to the correct SqlExpr in pNewQueryInfo, not SqlExpr in pQueryInfo for (int32_t f = 0; f < pNewQueryInfo->fieldsInfo.numOfOutput; ++f) { TAOS_FIELD* field = tscFieldInfoGetField(&pNewQueryInfo->fieldsInfo, f); @@ -1906,7 +1854,7 @@ static void doSetSqlExprAndResultFieldInfo(SQueryInfo* pQueryInfo, SQueryInfo* p SSqlExpr* pExpr1 = tscSqlExprGet(pNewQueryInfo, k1); if (strcmp(field->name, pExpr1->aliasName) == 0) { // establish link according to the result field name - SFieldSupInfo* pInfo = tscFieldInfoGetSupp(&pNewQueryInfo->fieldsInfo, f); + SInternalField* pInfo = tscFieldInfoGetInternalField(&pNewQueryInfo->fieldsInfo, f); pInfo->pSqlExpr = pExpr1; matched = true; diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index 2c2d686099..600347c44f 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -407,6 +407,7 @@ typedef struct SExprInfo { int16_t bytes; int16_t type; int32_t interBytes; + int64_t uid; } SExprInfo; typedef struct SColumnFilterInfo { From 820bf89a2e9eaf2f5cdc088fe3a9c683b322d071 Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Wed, 21 Oct 2020 10:42:09 +0800 Subject: [PATCH 34/51] TD-1471: use new error code when qhandle is null --- src/client/src/tscSql.c | 7 ++----- src/query/src/qExecutor.c | 3 +++ src/vnode/src/vnodeRead.c | 4 +++- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index 02a2c1e052..be91255b9c 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -479,11 +479,8 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) { SSqlCmd *pCmd = &pSql->cmd; SSqlRes *pRes = &pSql->res; - if (pRes->qhandle == 0) { - pRes->code = TSDB_CODE_TSC_INVALID_QHANDLE; - return NULL; - } - if (pRes->code == TSDB_CODE_TSC_QUERY_CANCELLED || + if (pRes->qhandle == 0 || + pRes->code == TSDB_CODE_TSC_QUERY_CANCELLED || pCmd->command == TSDB_SQL_RETRIEVE_EMPTY_RESULT || pCmd->command == TSDB_SQL_INSERT) { return NULL; diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index f742616b05..869e2dd7ed 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -7101,6 +7101,7 @@ void qCleanupQueryMgmt(void* pQMgmt) { void** qRegisterQInfo(void* pMgmt, uint64_t qInfo) { if (pMgmt == NULL) { + terrno = TSDB_CODE_VND_INVALID_VGROUP_ID; return NULL; } @@ -7109,6 +7110,7 @@ void** qRegisterQInfo(void* pMgmt, uint64_t qInfo) { SQueryMgmt *pQueryMgmt = pMgmt; if (pQueryMgmt->qinfoPool == NULL) { qError("QInfo:%p failed to add qhandle into qMgmt, since qMgmt is closed", (void *)qInfo); + terrno = TSDB_CODE_VND_INVALID_VGROUP_ID; return NULL; } @@ -7116,6 +7118,7 @@ void** qRegisterQInfo(void* pMgmt, uint64_t qInfo) { if (pQueryMgmt->closed) { // pthread_mutex_unlock(&pQueryMgmt->lock); qError("QInfo:%p failed to add qhandle into cache, since qMgmt is colsing", (void *)qInfo); + terrno = TSDB_CODE_VND_INVALID_VGROUP_ID; return NULL; } else { TSDB_CACHE_PTR_TYPE handleVal = (TSDB_CACHE_PTR_TYPE) qInfo; diff --git a/src/vnode/src/vnodeRead.c b/src/vnode/src/vnodeRead.c index baaeae2a81..969a29359f 100644 --- a/src/vnode/src/vnodeRead.c +++ b/src/vnode/src/vnodeRead.c @@ -166,10 +166,12 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { if (code == TSDB_CODE_SUCCESS) { handle = qRegisterQInfo(pVnode->qMgmt, (uint64_t)pQInfo); if (handle == NULL) { // failed to register qhandle, todo add error test case + pRsp->code = terrno; + terrno = 0; vError("vgId:%d QInfo:%p register qhandle failed, return to app, code:%s", pVnode->vgId, (void *)pQInfo, tstrerror(pRsp->code)); - pRsp->code = TSDB_CODE_QRY_INVALID_QHANDLE; qDestroyQueryInfo(pQInfo); // destroy it directly + return pRsp->code; } else { assert(*handle == pQInfo); pRsp->qhandle = htobe64((uint64_t)pQInfo); From 6b985a6e8677070ccd9bf7d3c0c10d9cd7b756d2 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 21 Oct 2020 10:51:46 +0800 Subject: [PATCH 35/51] [td-1764] fix memory leak. --- src/client/src/tscLocalMerge.c | 3 ++- src/client/src/tscUtil.c | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/client/src/tscLocalMerge.c b/src/client/src/tscLocalMerge.c index d800d62c8a..44ccb2471a 100644 --- a/src/client/src/tscLocalMerge.c +++ b/src/client/src/tscLocalMerge.c @@ -510,7 +510,8 @@ void tscDestroyLocalReducer(SSqlObj *pSql) { taosTFree(pLocalReducer->pResultBuf); if (pLocalReducer->pResInfo != NULL) { - for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { + size_t num = tscSqlExprNumOfExprs(pQueryInfo); + for (int32_t i = 0; i < num; ++i) { taosTFree(pLocalReducer->pResInfo[i].interResultBuf); } diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index d35fb21c57..1a8ed2e725 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -972,6 +972,8 @@ void tscFieldInfoClear(SFieldInfo* pFieldInfo) { } taosArrayDestroy(pFieldInfo->internalField); + taosTFree(pFieldInfo->final); + memset(pFieldInfo, 0, sizeof(SFieldInfo)); } @@ -1834,7 +1836,7 @@ static void doSetSqlExprAndResultFieldInfo(SQueryInfo* pQueryInfo, SQueryInfo* p return; } - // set the field info in pNewQueryInfo object + // set the field info in pNewQueryInfo object according to sqlExpr information size_t numOfExprs = tscSqlExprNumOfExprs(pNewQueryInfo); for (int32_t i = 0; i < numOfExprs; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pNewQueryInfo, i); From 6137a2b377f8c027aeb946bacee3f1eb9d4b4ac9 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 21 Oct 2020 03:07:16 +0000 Subject: [PATCH 36/51] TD-1746 --- src/balance/src/balance.c | 4 +- src/client/src/tscSql.c | 41 ++++++++++++++- src/client/src/tscUtil.c | 18 +++++-- src/dnode/src/dnodeVRead.c | 1 - src/inc/vnode.h | 2 + src/mnode/src/mnodeProfile.c | 5 +- src/vnode/src/vnodeMain.c | 96 ++++++++++++++++++++++-------------- src/vnode/src/vnodeWrite.c | 4 +- 8 files changed, 123 insertions(+), 48 deletions(-) diff --git a/src/balance/src/balance.c b/src/balance/src/balance.c index 0fa4b3f346..0e9bb85b25 100644 --- a/src/balance/src/balance.c +++ b/src/balance/src/balance.c @@ -216,8 +216,8 @@ static bool balanceCheckVgroupReady(SVgObj *pVgroup, SVnodeGid *pRmVnode) { SVnodeGid *pVnode = pVgroup->vnodeGid + i; if (pVnode == pRmVnode) continue; - mTrace("vgId:%d, change vgroup status, dnode:%d status:%d", pVgroup->vgId, pVnode->pDnode->dnodeId, - pVnode->pDnode->status); + mTrace("vgId:%d, check vgroup status, dnode:%d status:%d, vnode role:%s", pVgroup->vgId, pVnode->pDnode->dnodeId, + pVnode->pDnode->status, syncRole[pVnode->role]); if (pVnode->pDnode->status == TAOS_DN_STATUS_DROPPING) continue; if (pVnode->pDnode->status == TAOS_DN_STATUS_OFFLINE) continue; diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index be91255b9c..1ccf9c5ec4 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -588,7 +588,20 @@ static UNUSED_FUNC bool tscKillQueryInDnode(SSqlObj* pSql) { return true; } - SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); + SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); + + if (pQueryInfo != 0) { + STableMetaInfo *pTableMetaInfo1 = tscGetMetaInfo(pQueryInfo, 0); + if (pTableMetaInfo1 != NULL) { + // for select query super table, the super table vgroup list can not be null in any cases. + if (pQueryInfo->command == TSDB_SQL_SELECT && UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo1)) { + if (pTableMetaInfo1->pVgroupTables == 0) { + tscError("error !!!%p, vgroupTable is null", pSql); + } + } + } + } + if ((pQueryInfo == NULL) || tscIsTwoStageSTableQuery(pQueryInfo, 0)) { return true; } @@ -702,6 +715,19 @@ static void tscKillSTableQuery(SSqlObj *pSql) { SSqlCmd* pCmd = &pSql->cmd; SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); + + if (pQueryInfo != 0) { + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + if (pTableMetaInfo != NULL) { + // for select query super table, the super table vgroup list can not be null in any cases. + if (pQueryInfo->command == TSDB_SQL_SELECT && UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { + if (pTableMetaInfo->pVgroupTables == 0) { + tscError("error !!!%p, vgroupTable is null", pSql); + } + } + } + } + if (!tscIsTwoStageSTableQuery(pQueryInfo, 0)) { return; } @@ -750,6 +776,19 @@ void taos_stop_query(TAOS_RES *res) { pSql->res.code = TSDB_CODE_TSC_QUERY_CANCELLED; SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); + + if (pQueryInfo != 0) { + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + if (pTableMetaInfo != NULL) { + // for select query super table, the super table vgroup list can not be null in any cases. + if (pQueryInfo->command == TSDB_SQL_SELECT && UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { + if (pTableMetaInfo->pVgroupTables == 0) { + tscError("error !!!%p, vgroupTable is null", pSql); + } + } + } + } + if (tscIsTwoStageSTableQuery(pQueryInfo, 0)) { assert(pSql->pRpcCtx == NULL); tscKillSTableQuery(pSql); diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 0235f037bd..baa733ef49 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -114,9 +114,9 @@ bool tscIsTwoStageSTableQuery(SQueryInfo* pQueryInfo, int32_t tableIndex) { } // for select query super table, the super table vgroup list can not be null in any cases. - if (pQueryInfo->command == TSDB_SQL_SELECT && UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { - assert(pTableMetaInfo->vgroupList != NULL); - } + // if (pQueryInfo->command == TSDB_SQL_SELECT && UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { + // assert(pTableMetaInfo->vgroupList != NULL); + // } if ((pQueryInfo->type & TSDB_QUERY_TYPE_FREE_RESOURCE) == TSDB_QUERY_TYPE_FREE_RESOURCE) { return false; @@ -2098,6 +2098,18 @@ void tscDoQuery(SSqlObj* pSql) { } else { SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); uint16_t type = pQueryInfo->type; + + if (pQueryInfo != 0) { + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + if (pTableMetaInfo != NULL) { + // for select query super table, the super table vgroup list can not be null in any cases. + if (pQueryInfo->command == TSDB_SQL_SELECT && UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { + if (pTableMetaInfo->pVgroupTables == 0) { + tscError("error !!!%p, vgroupTable is null", pSql); + } + } + } + } if (pSql->fp == (void(*)())tscHandleMultivnodeInsert) { // multi-vnodes insertion tscHandleMultivnodeInsert(pSql); diff --git a/src/dnode/src/dnodeVRead.c b/src/dnode/src/dnodeVRead.c index 1a3d0ebc27..654e35cad5 100644 --- a/src/dnode/src/dnodeVRead.c +++ b/src/dnode/src/dnodeVRead.c @@ -189,7 +189,6 @@ void dnodeSendRpcReadRsp(void *pVnode, SReadMsg *pRead, int32_t code) { void dnodeDispatchNonRspMsg(void *pVnode, SReadMsg *pRead, int32_t code) { rpcFreeCont(pRead->rpcMsg.pCont); vnodeRelease(pVnode); - return; } static void *dnodeProcessReadQueue(void *param) { diff --git a/src/inc/vnode.h b/src/inc/vnode.h index 15ddb6afee..ed6c232fdb 100644 --- a/src/inc/vnode.h +++ b/src/inc/vnode.h @@ -41,6 +41,8 @@ typedef struct { SRpcMsg rpcMsg; } SReadMsg; +extern char *vnodeStatus[]; + int32_t vnodeCreate(SMDCreateVnodeMsg *pVnodeCfg); int32_t vnodeDrop(int32_t vgId); int32_t vnodeOpen(int32_t vgId, char *rootDir); diff --git a/src/mnode/src/mnodeProfile.c b/src/mnode/src/mnodeProfile.c index fc76e3dce3..f8f99e22c6 100644 --- a/src/mnode/src/mnodeProfile.c +++ b/src/mnode/src/mnodeProfile.c @@ -111,7 +111,6 @@ void mnodeReleaseConn(SConnObj *pConn) { } SConnObj *mnodeAccquireConn(int32_t connId, char *user, uint32_t ip, uint16_t port) { - uint64_t expireTime = CONN_KEEP_TIME * 1000 + (uint64_t)taosGetTimestampMs(); SConnObj *pConn = taosCacheAcquireByKey(tsMnodeConnCache, &connId, sizeof(int32_t)); if (pConn == NULL) { mDebug("connId:%d, is already destroyed, user:%s ip:%s:%u", connId, user, taosIpStr(ip), port); @@ -126,7 +125,7 @@ SConnObj *mnodeAccquireConn(int32_t connId, char *user, uint32_t ip, uint16_t po } // mDebug("connId:%d, is incoming, user:%s ip:%s:%u", connId, pConn->user, taosIpStr(pConn->ip), pConn->port); - pConn->lastAccess = expireTime; + pConn->lastAccess = CONN_KEEP_TIME * 1000 + (uint64_t)taosGetTimestampMs(); return pConn; } @@ -626,7 +625,7 @@ static int32_t mnodeProcessKillConnectionMsg(SMnodeMsg *pMsg) { SCMKillConnMsg *pKill = pMsg->rpcMsg.pCont; int32_t connId = atoi(pKill->queryId); - SConnObj * pConn = taosCacheAcquireByKey(tsMnodeConnCache, &connId, sizeof(int32_t)); + SConnObj *pConn = taosCacheAcquireByKey(tsMnodeConnCache, &connId, sizeof(int32_t)); if (pConn == NULL) { mError("connId:%s, failed to kill, conn not exist", pKill->queryId); return TSDB_CODE_MND_INVALID_CONN_ID; diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index a9bcf948b6..e4e5399a86 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -56,6 +56,14 @@ int syncGetNodesRole(tsync_h shandle, SNodesRole * cfg) { return 0; } void syncConfirmForward(tsync_h shandle, uint64_t version, int32_t code) {} #endif +char* vnodeStatus[] = { + "init", + "ready", + "closing", + "updating", + "reset" +}; + int32_t vnodeInitResources() { int code = syncInit(); if (code != 0) return code; @@ -74,6 +82,7 @@ int32_t vnodeInitResources() { void vnodeCleanupResources() { if (tsDnodeVnodesHash != NULL) { + vDebug("vnode list is cleanup"); taosHashCleanup(tsDnodeVnodesHash); tsDnodeVnodesHash = NULL; } @@ -84,9 +93,10 @@ void vnodeCleanupResources() { int32_t vnodeCreate(SMDCreateVnodeMsg *pVnodeCfg) { int32_t code; - SVnodeObj *pTemp = (SVnodeObj *)taosHashGet(tsDnodeVnodesHash, (const char *)&pVnodeCfg->cfg.vgId, sizeof(int32_t)); - if (pTemp != NULL) { - vInfo("vgId:%d, vnode already exist, pVnode:%p", pVnodeCfg->cfg.vgId, pTemp); + SVnodeObj *pVnode = vnodeAcquire(pVnodeCfg->cfg.vgId); + if (pVnode != NULL) { + vDebug("vgId:%d, vnode already exist, refCount:%d pVnode:%p", pVnodeCfg->cfg.vgId, pVnode->refCount, pVnode); + vnodeRelease(pVnode); return TSDB_CODE_SUCCESS; } @@ -143,22 +153,24 @@ int32_t vnodeCreate(SMDCreateVnodeMsg *pVnodeCfg) { return TSDB_CODE_VND_INIT_FAILED; } - vInfo("vgId:%d, vnode is created, walLevel:%d fsyncPeriod:%d", pVnodeCfg->cfg.vgId, pVnodeCfg->cfg.walLevel, pVnodeCfg->cfg.fsyncPeriod); + vInfo("vgId:%d, vnode dir is created, walLevel:%d fsyncPeriod:%d", pVnodeCfg->cfg.vgId, pVnodeCfg->cfg.walLevel, + pVnodeCfg->cfg.fsyncPeriod); code = vnodeOpen(pVnodeCfg->cfg.vgId, rootDir); return code; } int32_t vnodeDrop(int32_t vgId) { - SVnodeObj **ppVnode = (SVnodeObj **)taosHashGet(tsDnodeVnodesHash, (const char *)&vgId, sizeof(int32_t)); - if (ppVnode == NULL || *ppVnode == NULL) { - vDebug("vgId:%d, failed to drop, vgId not find", vgId); + SVnodeObj *pVnode = vnodeAcquire(vgId); + if (pVnode == NULL) { + vDebug("vgId:%d, failed to drop, vnode not find", vgId); return TSDB_CODE_VND_INVALID_VGROUP_ID; } - SVnodeObj *pVnode = *ppVnode; - vTrace("vgId:%d, vnode will be dropped, refCount:%d", pVnode->vgId, pVnode->refCount); + vInfo("vgId:%d, vnode will be dropped, refCount:%d pVnode:%p", pVnode->vgId, pVnode->refCount, pVnode); pVnode->dropped = 1; + + vnodeRelease(pVnode); vnodeCleanUp(pVnode); return TSDB_CODE_SUCCESS; @@ -340,11 +352,11 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) { } int32_t vnodeClose(int32_t vgId) { - SVnodeObj **ppVnode = (SVnodeObj **)taosHashGet(tsDnodeVnodesHash, (const char *)&vgId, sizeof(int32_t)); - if (ppVnode == NULL || *ppVnode == NULL) return 0; + SVnodeObj *pVnode = vnodeAcquire(vgId); + if (pVnode == NULL) return 0; - SVnodeObj *pVnode = *ppVnode; - vDebug("vgId:%d, vnode will be closed", pVnode->vgId); + vDebug("vgId:%d, vnode will be closed, pVnode:%p", pVnode->vgId, pVnode); + vnodeRelease(pVnode); vnodeCleanUp(pVnode); return 0; @@ -355,21 +367,27 @@ void vnodeRelease(void *pVnodeRaw) { int32_t vgId = pVnode->vgId; int32_t refCount = atomic_sub_fetch_32(&pVnode->refCount, 1); + vTrace("vgId:%d, release vnode, refCount:%d pVnode:%p", vgId, refCount, pVnode); assert(refCount >= 0); if (refCount > 0) { - vDebug("vgId:%d, release vnode, refCount:%d", vgId, refCount); - if (pVnode->status == TAOS_VN_STATUS_RESET && refCount == 2) + if (pVnode->status == TAOS_VN_STATUS_RESET && refCount == 2) { tsem_post(&pVnode->sem); + } return; } - qCleanupQueryMgmt(pVnode->qMgmt); - pVnode->qMgmt = NULL; + vDebug("vgId:%d, vnode will be destroyed, refCount:%d pVnode:%p", vgId, refCount, pVnode); - if (pVnode->tsdb) + if (pVnode->qMgmt) { + qCleanupQueryMgmt(pVnode->qMgmt); + pVnode->qMgmt = NULL; + } + + if (pVnode->tsdb) { tsdbCloseRepo(pVnode->tsdb, 1); - pVnode->tsdb = NULL; + pVnode->tsdb = NULL; + } // stop continuous query if (pVnode->cq) { @@ -378,18 +396,21 @@ void vnodeRelease(void *pVnodeRaw) { cqClose(cq); } - if (pVnode->wal) + if (pVnode->wal) { walClose(pVnode->wal); - pVnode->wal = NULL; + pVnode->wal = NULL; + } - if (pVnode->wqueue) + if (pVnode->wqueue) { dnodeFreeVnodeWqueue(pVnode->wqueue); - pVnode->wqueue = NULL; + pVnode->wqueue = NULL; + } - if (pVnode->rqueue) + if (pVnode->rqueue) { dnodeFreeVnodeRqueue(pVnode->rqueue); - pVnode->rqueue = NULL; - + pVnode->rqueue = NULL; + } + taosTFree(pVnode->rootDir); if (pVnode->dropped) { @@ -413,20 +434,20 @@ void vnodeRelease(void *pVnodeRaw) { free(pVnode); int32_t count = taosHashGetSize(tsDnodeVnodesHash); - vDebug("vgId:%d, vnode is released, vnodes:%d", vgId, count); + vDebug("vgId:%d, vnode is destroyed, vnodes:%d", vgId, count); } void *vnodeAcquire(int32_t vgId) { SVnodeObj **ppVnode = (SVnodeObj **)taosHashGet(tsDnodeVnodesHash, (const char *)&vgId, sizeof(int32_t)); if (ppVnode == NULL || *ppVnode == NULL) { terrno = TSDB_CODE_VND_INVALID_VGROUP_ID; - vInfo("vgId:%d, not exist", vgId); + vDebug("vgId:%d, not exist", vgId); return NULL; } SVnodeObj *pVnode = *ppVnode; atomic_add_fetch_32(&pVnode->refCount, 1); - vDebug("vgId:%d, get vnode, refCount:%d", pVnode->vgId, pVnode->refCount); + vTrace("vgId:%d, get vnode, refCount:%d pVnode:%p", pVnode->vgId, pVnode->refCount, pVnode); return pVnode; } @@ -528,7 +549,7 @@ void vnodeSetAccess(SDMVgroupAccess *pAccess, int32_t numOfVnodes) { if (pVnode != NULL) { pVnode->accessState = pAccess[i].accessState; if (pVnode->accessState != TSDB_VN_ALL_ACCCESS) { - vDebug("vgId:%d, access state is set to %d", pAccess[i].vgId, pVnode->accessState) + vDebug("vgId:%d, access state is set to %d", pAccess[i].vgId, pVnode->accessState); } vnodeRelease(pVnode); } @@ -538,11 +559,12 @@ void vnodeSetAccess(SDMVgroupAccess *pAccess, int32_t numOfVnodes) { static void vnodeCleanUp(SVnodeObj *pVnode) { // remove from hash, so new messages wont be consumed taosHashRemove(tsDnodeVnodesHash, (const char *)&pVnode->vgId, sizeof(int32_t)); - int i = 0; if (pVnode->status != TAOS_VN_STATUS_INIT) { // it may be in updateing or reset state, then it shall wait - while (atomic_val_compare_exchange_8(&pVnode->status, TAOS_VN_STATUS_READY, TAOS_VN_STATUS_CLOSING) != TAOS_VN_STATUS_READY) { + int i = 0; + while (atomic_val_compare_exchange_8(&pVnode->status, TAOS_VN_STATUS_READY, TAOS_VN_STATUS_CLOSING) != + TAOS_VN_STATUS_READY) { if (++i % 1000 == 0) { sched_yield(); } @@ -556,7 +578,7 @@ static void vnodeCleanUp(SVnodeObj *pVnode) { syncStop(sync); } - vTrace("vgId:%d, vnode will cleanup, refCount:%d", pVnode->vgId, pVnode->refCount); + vDebug("vgId:%d, vnode will cleanup, refCount:%d pVnode:%p", pVnode->vgId, pVnode->refCount, pVnode); // release local resources only after cutting off outside connections qQueryMgmtNotifyClosed(pVnode->qMgmt); @@ -613,17 +635,19 @@ static int vnodeResetTsdb(SVnodeObj *pVnode) char rootDir[128] = "\0"; sprintf(rootDir, "%s/tsdb", pVnode->rootDir); - if (atomic_val_compare_exchange_8(&pVnode->status, TAOS_VN_STATUS_READY, TAOS_VN_STATUS_RESET) != TAOS_VN_STATUS_READY) + if (atomic_val_compare_exchange_8(&pVnode->status, TAOS_VN_STATUS_READY, TAOS_VN_STATUS_RESET) != TAOS_VN_STATUS_READY) { return -1; + } void *tsdb = pVnode->tsdb; pVnode->tsdb = NULL; // acquire vnode - int32_t refCount = atomic_add_fetch_32(&pVnode->refCount, 1); + int32_t refCount = atomic_add_fetch_32(&pVnode->refCount, 1); - if (refCount > 2) + if (refCount > 2) { tsem_wait(&pVnode->sem); + } // close tsdb, then open tsdb tsdbCloseRepo(tsdb, 0); diff --git a/src/vnode/src/vnodeWrite.c b/src/vnode/src/vnodeWrite.c index c4924f312f..0966192c94 100644 --- a/src/vnode/src/vnodeWrite.c +++ b/src/vnode/src/vnodeWrite.c @@ -202,7 +202,7 @@ int vnodeWriteCqMsgToQueue(void *param, void *data, int type) { memcpy(pWal, pHead, size); atomic_add_fetch_32(&pVnode->refCount, 1); - vDebug("CQ: vgId:%d, get vnode wqueue, refCount:%d", pVnode->vgId, pVnode->refCount); + vTrace("CQ: vgId:%d, get vnode wqueue, refCount:%d pVnode:%p", pVnode->vgId, pVnode->refCount, pVnode); taosWriteQitem(pVnode->wqueue, type, pSync); @@ -219,7 +219,7 @@ int vnodeWriteToQueue(void *param, void *data, int type) { memcpy(pWal, pHead, size); atomic_add_fetch_32(&pVnode->refCount, 1); - vDebug("vgId:%d, get vnode wqueue, refCount:%d", pVnode->vgId, pVnode->refCount); + vTrace("vgId:%d, get vnode wqueue, refCount:%d pVnode:%p", pVnode->vgId, pVnode->refCount, pVnode); taosWriteQitem(pVnode->wqueue, type, pWal); From 438aef99f7c080ffcd4665b5f6e62c8932579a60 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 21 Oct 2020 03:07:22 +0000 Subject: [PATCH 37/51] TD-1661 --- src/vnode/src/vnodeRead.c | 39 ++++++++++++++++++++++++++++++++------- 1 file changed, 32 insertions(+), 7 deletions(-) diff --git a/src/vnode/src/vnodeRead.c b/src/vnode/src/vnodeRead.c index baaeae2a81..2a39e2e76f 100644 --- a/src/vnode/src/vnodeRead.c +++ b/src/vnode/src/vnodeRead.c @@ -38,8 +38,7 @@ void vnodeInitReadFp(void) { vnodeProcessReadMsgFp[TSDB_MSG_TYPE_FETCH] = vnodeProcessFetchMsg; } -int32_t vnodeProcessRead(void *param, SReadMsg *pReadMsg) { - SVnodeObj *pVnode = (SVnodeObj *)param; +static int32_t vnodeProcessReadImp(SVnodeObj *pVnode, SReadMsg *pReadMsg) { int msgType = pReadMsg->rpcMsg.msgType; if (vnodeProcessReadMsgFp[msgType] == NULL) { @@ -48,16 +47,23 @@ int32_t vnodeProcessRead(void *param, SReadMsg *pReadMsg) { } if (pVnode->status != TAOS_VN_STATUS_READY) { - vDebug("vgId:%d, msgType:%s not processed, vnode status is %d", pVnode->vgId, taosMsg[msgType], pVnode->status); + vDebug("vgId:%d, msgType:%s not processed, vnode status is %s", pVnode->vgId, taosMsg[msgType], + vnodeStatus[pVnode->status]); return TSDB_CODE_APP_NOT_READY; } // tsdb may be in reset state - if (pVnode->tsdb == NULL) return TSDB_CODE_APP_NOT_READY; - if (pVnode->status == TAOS_VN_STATUS_CLOSING) return TSDB_CODE_APP_NOT_READY; + if (pVnode->tsdb == NULL) { + vDebug("vgId:%d, msgType:%s not processed, tsdb is null", pVnode->vgId, taosMsg[msgType]); + return TSDB_CODE_APP_NOT_READY; + } + + if (pVnode->status == TAOS_VN_STATUS_CLOSING) { + vDebug("vgId:%d, msgType:%s not processed, vstatus is %s", pVnode->vgId, taosMsg[msgType], + vnodeStatus[pVnode->status]); + return TSDB_CODE_APP_NOT_READY; + } - // TODO: Later, let slave to support query - // if (pVnode->syncCfg.replica > 1 && pVnode->role != TAOS_SYNC_ROLE_MASTER) { if (pVnode->role != TAOS_SYNC_ROLE_SLAVE && pVnode->role != TAOS_SYNC_ROLE_MASTER) { vDebug("vgId:%d, msgType:%s not processed, replica:%d role:%s", pVnode->vgId, taosMsg[msgType], pVnode->syncCfg.replica, syncRole[pVnode->role]); @@ -67,6 +73,25 @@ int32_t vnodeProcessRead(void *param, SReadMsg *pReadMsg) { return (*vnodeProcessReadMsgFp[msgType])(pVnode, pReadMsg); } +int32_t vnodeProcessRead(void *param, SReadMsg *pRead) { + SVnodeObj *pVnode = (SVnodeObj *)param; + int32_t code = vnodeProcessReadImp(pVnode, pRead); + + if (code == TSDB_CODE_APP_NOT_READY && pRead->rpcMsg.msgType == TSDB_MSG_TYPE_QUERY) { + // After the fetch request enters the vnode queue + // If the vnode cannot provide services, the following operations are still required + // Or, there will be a deadlock + void **qhandle = (void **)pRead->pCont; + vError("QInfo:%p msg:%p will be killed for vstatus is %s", *qhandle, pRead, vnodeStatus[pVnode->status]); + + // qKillQuery(*qhandle); + // qReleaseQInfo(pVnode->qMgmt, (void **)&qhandle, true); + return TSDB_CODE_APP_NOT_READY; + } else { + return code; + } +} + static void vnodePutItemIntoReadQueue(SVnodeObj *pVnode, void **qhandle) { SReadMsg *pRead = (SReadMsg *)taosAllocateQitem(sizeof(SReadMsg)); pRead->rpcMsg.msgType = TSDB_MSG_TYPE_QUERY; From 374987e039c1c914c43fbeb5b7163d7e5144c9d0 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 21 Oct 2020 03:29:14 +0000 Subject: [PATCH 38/51] TD-1746 --- src/vnode/src/vnodeMain.c | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index e4e5399a86..bc728e86d9 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -437,19 +437,28 @@ void vnodeRelease(void *pVnodeRaw) { vDebug("vgId:%d, vnode is destroyed, vnodes:%d", vgId, count); } +static void vnodeIncRef(void *ptNode) { + assert(ptNode != NULL); + + SVnodeObj **ppVnode = (SVnodeObj **)ptNode; + assert(ppVnode); + assert(*ppVnode); + + SVnodeObj *pVnode = *ppVnode; + atomic_add_fetch_32(&pVnode->refCount, 1); + vTrace("vgId:%d, get vnode, refCount:%d pVnode:%p", pVnode->vgId, pVnode->refCount, pVnode); +} + void *vnodeAcquire(int32_t vgId) { - SVnodeObj **ppVnode = (SVnodeObj **)taosHashGet(tsDnodeVnodesHash, (const char *)&vgId, sizeof(int32_t)); + SVnodeObj **ppVnode = taosHashGetCB(tsDnodeVnodesHash, &vgId, sizeof(int32_t), vnodeIncRef, NULL, sizeof(void *)); + if (ppVnode == NULL || *ppVnode == NULL) { terrno = TSDB_CODE_VND_INVALID_VGROUP_ID; vDebug("vgId:%d, not exist", vgId); return NULL; } - SVnodeObj *pVnode = *ppVnode; - atomic_add_fetch_32(&pVnode->refCount, 1); - vTrace("vgId:%d, get vnode, refCount:%d pVnode:%p", pVnode->vgId, pVnode->refCount, pVnode); - - return pVnode; + return *ppVnode; } void *vnodeAcquireRqueue(int32_t vgId) { From 196b03f1bdf595f323affd87128ad0ad2e9eaad2 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 21 Oct 2020 03:37:24 +0000 Subject: [PATCH 39/51] remove debug codes add by hjliao --- src/client/src/tscSql.c | 36 ------------------------------------ src/client/src/tscUtil.c | 12 ------------ 2 files changed, 48 deletions(-) diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index 1ccf9c5ec4..7594a73c96 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -590,18 +590,6 @@ static UNUSED_FUNC bool tscKillQueryInDnode(SSqlObj* pSql) { SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); - if (pQueryInfo != 0) { - STableMetaInfo *pTableMetaInfo1 = tscGetMetaInfo(pQueryInfo, 0); - if (pTableMetaInfo1 != NULL) { - // for select query super table, the super table vgroup list can not be null in any cases. - if (pQueryInfo->command == TSDB_SQL_SELECT && UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo1)) { - if (pTableMetaInfo1->pVgroupTables == 0) { - tscError("error !!!%p, vgroupTable is null", pSql); - } - } - } - } - if ((pQueryInfo == NULL) || tscIsTwoStageSTableQuery(pQueryInfo, 0)) { return true; } @@ -716,18 +704,6 @@ static void tscKillSTableQuery(SSqlObj *pSql) { SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); - if (pQueryInfo != 0) { - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - if (pTableMetaInfo != NULL) { - // for select query super table, the super table vgroup list can not be null in any cases. - if (pQueryInfo->command == TSDB_SQL_SELECT && UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { - if (pTableMetaInfo->pVgroupTables == 0) { - tscError("error !!!%p, vgroupTable is null", pSql); - } - } - } - } - if (!tscIsTwoStageSTableQuery(pQueryInfo, 0)) { return; } @@ -777,18 +753,6 @@ void taos_stop_query(TAOS_RES *res) { SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); - if (pQueryInfo != 0) { - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - if (pTableMetaInfo != NULL) { - // for select query super table, the super table vgroup list can not be null in any cases. - if (pQueryInfo->command == TSDB_SQL_SELECT && UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { - if (pTableMetaInfo->pVgroupTables == 0) { - tscError("error !!!%p, vgroupTable is null", pSql); - } - } - } - } - if (tscIsTwoStageSTableQuery(pQueryInfo, 0)) { assert(pSql->pRpcCtx == NULL); tscKillSTableQuery(pSql); diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index baa733ef49..a38be04ae0 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -2098,18 +2098,6 @@ void tscDoQuery(SSqlObj* pSql) { } else { SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); uint16_t type = pQueryInfo->type; - - if (pQueryInfo != 0) { - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - if (pTableMetaInfo != NULL) { - // for select query super table, the super table vgroup list can not be null in any cases. - if (pQueryInfo->command == TSDB_SQL_SELECT && UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { - if (pTableMetaInfo->pVgroupTables == 0) { - tscError("error !!!%p, vgroupTable is null", pSql); - } - } - } - } if (pSql->fp == (void(*)())tscHandleMultivnodeInsert) { // multi-vnodes insertion tscHandleMultivnodeInsert(pSql); From 2c7a9137b548e7424d94e63fadb2e639383717ce Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Wed, 21 Oct 2020 13:45:44 +0800 Subject: [PATCH 40/51] modify test case --- Jenkinsfile | 16 +++ tests/pytest/concurrent_inquiry.py | 154 +++++++++++++++-------------- 2 files changed, 94 insertions(+), 76 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 6d563d132b..08b6121f12 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -118,6 +118,22 @@ pipeline { date''' } } + stage('connector'){ + agent{label "release"} + steps{ + sh''' + cd ${WORKSPACE} + git checkout develop + cd tests/gotest + bash batchtest.sh + cd ${WORKSPACE}/tests/examples/JDBC/JDBCDemo/ + mvn clean package assembly:single >/dev/null + java -jar target/jdbcChecker-SNAPSHOT-jar-with-dependencies.jar -host 127.0.0.1 + cd ${WORKSPACE}/tests/examples/python/PYTHONConnectorChecker + python3 PythonChecker.py + ''' + } + } } } diff --git a/tests/pytest/concurrent_inquiry.py b/tests/pytest/concurrent_inquiry.py index 2460900891..39a4cb48fd 100644 --- a/tests/pytest/concurrent_inquiry.py +++ b/tests/pytest/concurrent_inquiry.py @@ -12,108 +12,105 @@ # -*- coding: utf-8 -*- import threading import taos - +import sys import json import time import random # query sql query_sql = [ # first supertable -"select count(*) from test.meters where c1 > 50;", -"select count(*) from test.meters where c2 >= 50 and c2 < 100;", -"select count(*) from test.meters where c3 != 5;", +"select count(*) from test.meters ;", "select count(*) from test.meters where t3 > 2;", "select count(*) from test.meters where ts <> '2020-05-13 10:00:00.002';", -"select count(*) from test.meters where t7 like 'fi%';", -"select count(*) from test.meters where t7 like '_econd';", +"select count(*) from test.meters where t7 like 'taos_1%';", +"select count(*) from test.meters where t7 like '_____2';", +"select count(*) from test.meters where t8 like '%思%';", "select count(*) from test.meters interval(1n) order by ts desc;", -"select max(c0) from test.meters group by tbname", -"select first(*) from test.meters;", -"select last(*) from test.meters;", +#"select max(c0) from test.meters group by tbname", +"select first(ts) from test.meters where t5 >5000 and t5<5100;", +"select last(ts) from test.meters where t5 >5000 and t5<5100;", "select last_row(*) from test.meters;", "select twa(c1) from test.t1 where ts > 1500000001000 and ts < 1500000101000" , -"select avg(c1) from test.meters;", +"select avg(c1) from test.meters where t5 >5000 and t5<5100;", "select bottom(c1, 2) from test.t1;", "select diff(c1) from test.t1;", "select leastsquares(c1, 1, 1) from test.t1 ;", -"select max(c1) from test.meters;", -"select min(c1) from test.meters;", -"select c1 + c2 * c3 + c1 / c5 + c4 + c2 from test.t1;", +"select max(c1) from test.meters where t5 >5000 and t5<5100;", +"select min(c1) from test.meters where t5 >5000 and t5<5100;", +"select c1 + c2 + c1 / c5 + c4 + c2 from test.t1;", "select percentile(c1, 50) from test.t1;", "select spread(c1) from test.t1 ;", "select stddev(c1) from test.t1;", -"select sum(c1) from test.meters;", -"select top(c1, 2) from test.meters;" -"select twa(c6) from test.t1 where ts > 1500000001000 and ts < 1500000101000" , -"select avg(c6) from test.meters;", -"select bottom(c6, 2) from test.t1;", -"select diff(c6) from test.t1;", -"select leastsquares(c6, 1, 1) from test.t1 ;", -"select max(c6) from test.meters;", -"select min(c6) from test.meters;", -"select c6 + c2 * c3 + c6 / c5 + c4 + c2 from test.t1;", -"select percentile(c6, 50) from test.t1;", -"select spread(c6) from test.t1 ;", -"select stddev(c6) from test.t1;", -"select sum(c6) from test.meters;", -"select top(c6, 2) from test.meters;", +"select sum(c1) from test.meters where t5 >5000 and t5<5100;", +"select top(c1, 2) from test.meters where t5 >5000 and t5<5100;" +"select twa(c4) from test.t1 where ts > 1500000001000 and ts < 1500000101000" , +"select avg(c4) from test.meters where t5 >5000 and t5<5100;", +"select bottom(c4, 2) from test.t1 where t5 >5000 and t5<5100;", +"select diff(c4) from test.t1 where t5 >5000 and t5<5100;", +"select leastsquares(c4, 1, 1) from test.t1 ;", +"select max(c4) from test.meters where t5 >5000 and t5<5100;", +"select min(c4) from test.meters where t5 >5000 and t5<5100;", +"select c5 + c2 + c4 / c5 + c4 + c2 from test.t1 ;", +"select percentile(c5, 50) from test.t1;", +"select spread(c5) from test.t1 ;", +"select stddev(c5) from test.t1 where t5 >5000 and t5<5100;", +"select sum(c5) from test.meters where t5 >5000 and t5<5100;", +"select top(c5, 2) from test.meters where t5 >5000 and t5<5100;", #all vnode -"select count(*) from test.meters where t5 >2500 and t5<7500", -"select max(c0),avg(c1) from test.meters where t5 >2500 and t5<7500", -"select sum(c5),avg(c1) from test.meters where t5 >2500 and t5<7500", -"select max(c0),min(c6) from test.meters where t5 >2500 and t5<7500", -"select min(c0),avg(c6) from test.meters where t5 >2500 and t5<7500", +"select count(*) from test.meters where t5 >5000 and t5<5100", +"select max(c0),avg(c1) from test.meters where t5 >5000 and t5<5100", +"select sum(c5),avg(c1) from test.meters where t5 >5000 and t5<5100", +"select max(c0),min(c5) from test.meters where t5 >5000 and t5<5100", +"select min(c0),avg(c5) from test.meters where t5 >5000 and t5<5100", # second supertable -"select count(*) from test.meters1 where c1 > 50;", -"select count(*) from test.meters1 where c2 >= 50 and c2 < 100;", -"select count(*) from test.meters1 where c3 != 5;", "select count(*) from test.meters1 where t3 > 2;", "select count(*) from test.meters1 where ts <> '2020-05-13 10:00:00.002';", -"select count(*) from test.meters1 where t7 like 'fi%';", -"select count(*) from test.meters1 where t7 like '_econd';", +"select count(*) from test.meters where t7 like 'taos_1%';", +"select count(*) from test.meters where t7 like '_____2';", +"select count(*) from test.meters where t8 like '%思%';", "select count(*) from test.meters1 interval(1n) order by ts desc;", -"select max(c0) from test.meters1 group by tbname", -"select first(*) from test.meters1;", -"select last(*) from test.meters1;", -"select last_row(*) from test.meters1;", +#"select max(c0) from test.meters1 group by tbname", +"select first(ts) from test.meters1 where t5 >5000 and t5<5100;", +"select last(ts) from test.meters1 where t5 >5000 and t5<5100;", +"select last_row(*) from test.meters1 ;", "select twa(c1) from test.m1 where ts > 1500000001000 and ts < 1500000101000" , -"select avg(c1) from test.meters1;", -"select bottom(c1, 2) from test.m1;", -"select diff(c1) from test.m1;", +"select avg(c1) from test.meters1 where t5 >5000 and t5<5100;", +"select bottom(c1, 2) from test.m1 where t5 >5000 and t5<5100;", +"select diff(c1) from test.m1 ;", "select leastsquares(c1, 1, 1) from test.m1 ;", -"select max(c1) from test.meters1;", -"select min(c1) from test.meters1;", -"select c1 + c2 * c3 + c1 / c5 + c3 + c2 from test.m1;", +"select max(c1) from test.meters1 where t5 >5000 and t5<5100;", +"select min(c1) from test.meters1 where t5 >5000 and t5<5100;", +"select c1 + c2 + c1 / c0 + c2 from test.m1 ;", "select percentile(c1, 50) from test.m1;", "select spread(c1) from test.m1 ;", "select stddev(c1) from test.m1;", -"select sum(c1) from test.meters1;", -"select top(c1, 2) from test.meters1;", -"select twa(c6) from test.m1 where ts > 1500000001000 and ts < 1500000101000" , -"select avg(c6) from test.meters1;", -"select bottom(c6, 2) from test.m1;", -"select diff(c6) from test.m1;", -"select leastsquares(c6, 1, 1) from test.m1 ;", -"select max(c6) from test.meters1;", -"select min(c6) from test.meters1;", -"select c6 + c2 * c3 + c6 / c5 + c3 + c2 from test.m1;", -"select percentile(c6, 50) from test.m1;", -"select spread(c6) from test.m1 ;", -"select stddev(c6) from test.m1;", -"select sum(c6) from test.meters1;", -"select top(c6, 2) from test.meters1;", -"select count(*) from test.meters1 where t5 >2500 and t5<7500", +"select sum(c1) from test.meters1 where t5 >5000 and t5<5100;", +"select top(c1, 2) from test.meters1 where t5 >5000 and t5<5100;", +"select twa(c5) from test.m1 where ts > 1500000001000 and ts < 1500000101000" , +"select avg(c5) from test.meters1 where t5 >5000 and t5<5100;", +"select bottom(c5, 2) from test.m1;", +"select diff(c5) from test.m1;", +"select leastsquares(c5, 1, 1) from test.m1 ;", +"select max(c5) from test.meters1 where t5 >5000 and t5<5100;", +"select min(c5) from test.meters1 where t5 >5000 and t5<5100;", +"select c5 + c2 + c4 / c5 + c0 from test.m1;", +"select percentile(c4, 50) from test.m1;", +"select spread(c4) from test.m1 ;", +"select stddev(c4) from test.m1;", +"select sum(c4) from test.meters1 where t5 >5100 and t5<5300;", +"select top(c4, 2) from test.meters1 where t5 >5100 and t5<5300;", +"select count(*) from test.meters1 where t5 >5100 and t5<5300", #all vnode -"select count(*) from test.meters1 where t5 >2500 and t5<7500", -"select max(c0),avg(c1) from test.meters1 where t5 >2500 and t5<7500", -"select sum(c5),avg(c1) from test.meters1 where t5 >2500 and t5<7500", -"select max(c0),min(c6) from test.meters1 where t5 >2500 and t5<7500", -"select min(c0),avg(c6) from test.meters1 where t5 >2500 and t5<7500", +"select count(*) from test.meters1 where t5 >5100 and t5<5300", +"select max(c0),avg(c1) from test.meters1 where t5 >5000 and t5<5100", +"select sum(c5),avg(c1) from test.meters1 where t5 >5000 and t5<5100", +"select max(c0),min(c5) from test.meters1 where t5 >5000 and t5<5100", +"select min(c0),avg(c5) from test.meters1 where t5 >5000 and t5<5100", #join -"select * from meters,meters1 where meters.ts = meters1.ts and meters.t5 = meters1.t5", -"select * from meters,meters1 where meters.ts = meters1.ts and meters.t7 = meters1.t7", -"select * from meters,meters1 where meters.ts = meters1.ts and meters.t8 = meters1.t8", -"select meters.ts,meters1.c2 from meters,meters1 where meters.ts = meters1.ts and meters.t8 = meters1.t8" +# "select * from meters,meters1 where meters.ts = meters1.ts and meters.t5 = meters1.t5", +# "select * from meters,meters1 where meters.ts = meters1.ts and meters.t7 = meters1.t7", +# "select * from meters,meters1 where meters.ts = meters1.ts and meters.t8 = meters1.t8", +# "select meters.ts,meters1.c2 from meters,meters1 where meters.ts = meters1.ts and meters.t8 = meters1.t8" ] class ConcurrentInquiry: @@ -121,7 +118,8 @@ class ConcurrentInquiry: self.numOfTherads = 50 self.ts=1500000001000 - + def SetThreadsNum(self,num): + self.numOfTherads=num def query_thread(self,threadID): host = "10.211.55.14" user = "root" @@ -142,12 +140,16 @@ class ConcurrentInquiry: for i in ran_query_sql: print("Thread %d : %s"% (threadID,i)) try: + start = time.time() cl.execute(i) cl.fetchall + end = time.time() + print("time cost :",end-start) except Exception as e: print( "Failure thread%d, sql: %s,exception: %s" % (threadID, str(i),str(e))) + exit(-1) print("Thread %d: finishing" % threadID) @@ -155,9 +157,9 @@ class ConcurrentInquiry: def run(self): - + threads = [] - for i in range(50): + for i in range(self.numOfTherads): thread = threading.Thread(target=self.query_thread, args=(i,)) threads.append(thread) thread.start() From a170da720130b053c806b4ebd412d4dc99ca55ff Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 21 Oct 2020 13:49:31 +0800 Subject: [PATCH 41/51] [td-1664] --- src/client/src/tscSQLParser.c | 18 +++++++++- src/query/src/qSyntaxtreefunction.c | 3 ++ .../parser/col_arithmetic_operation.sim | 35 +++++++++++++++++++ 3 files changed, 55 insertions(+), 1 deletion(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index c8e80790ef..b28dd9cf1c 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -3385,10 +3385,26 @@ static int32_t validateSQLExpr(SSqlCmd* pCmd, tSQLExpr* pExpr, SQueryInfo* pQuer tSQLExprItem item = {.pNode = pExpr, .aliasName = NULL}; - // sql function in selection clause, append sql function info in pSqlCmd structure sequentially + // sql function list in selection clause. + // Append the sqlExpr into exprList of pQueryInfo structure sequentially if (addExprAndResultField(pCmd, pQueryInfo, outputIndex, &item, false) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; } + + // It is invalid in case of more than one sqlExpr, such as first(ts, k) - last(ts, k) + int32_t inc = (int32_t) tscSqlExprNumOfExprs(pQueryInfo) - outputIndex; + if (inc > 1) { + return TSDB_CODE_TSC_INVALID_SQL; + } + + // Not supported data type in arithmetic expression + for(int32_t i = 0; i < inc; ++i) { + SSqlExpr* p1 = tscSqlExprGet(pQueryInfo, i + outputIndex); + int16_t t = p1->resType; + if (t == TSDB_DATA_TYPE_BINARY || t == TSDB_DATA_TYPE_NCHAR || t == TSDB_DATA_TYPE_BOOL || t == TSDB_DATA_TYPE_TIMESTAMP) { + return TSDB_CODE_TSC_INVALID_SQL; + } + } } return TSDB_CODE_SUCCESS; diff --git a/src/query/src/qSyntaxtreefunction.c b/src/query/src/qSyntaxtreefunction.c index 2104edfd91..7f7fca2c1e 100644 --- a/src/query/src/qSyntaxtreefunction.c +++ b/src/query/src/qSyntaxtreefunction.c @@ -1247,7 +1247,10 @@ _bi_consumer_fn_t tGetBiConsumerFn(int32_t leftType, int32_t rightType, int32_t case TSDB_BINARY_OP_REMAINDER: return rem_function_arraylist[leftType][rightType]; default: + assert(0); return NULL; } + + assert(0); return NULL; } diff --git a/tests/script/general/parser/col_arithmetic_operation.sim b/tests/script/general/parser/col_arithmetic_operation.sim index e1557227d4..4977e906d7 100644 --- a/tests/script/general/parser/col_arithmetic_operation.sim +++ b/tests/script/general/parser/col_arithmetic_operation.sim @@ -109,4 +109,39 @@ if $data02 != 225000 then return -1 endi +sql select first(speed) - last(speed), first(speed) as b, last(speed) as b, min(speed) - max(speed), spread(speed) from stb interval(1y)", NULL); +if $rows != 1 then + return -1 +endi + +if $data00 != @20-01-01 00:00:00.000@ then + return -1 +endi + +if $data01 != -20.000000000 then + return -1 +endi + +if $data02 != 100.000000000 then + return -1 +endi + +if $data03 != 120.000000000 then + return -1 +endi + +if $data04 != -20.000000000 then + return -1 +endi + +if $data05 != 20.000000000 then + return -1 +endi + +sql_error select first(speed, mileage) - last(speed, mileage) from stb interval(1y) +sql_error select first(ts) - last(ts) from stb interval(1y) +sql_error select top(speed, 2) - last(speed) from stb; +sql_error select stddev(speed) - last(speed) from stb; +sql_error select diff(speed) - last(speed) from stb; + system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file From 84d24478e62f952b8a26e5c7e6a66680b1e09d50 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 21 Oct 2020 14:21:11 +0800 Subject: [PATCH 42/51] [td-1664] --- .../parser/col_arithmetic_operation.sim | 25 +++++++++++-------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/tests/script/general/parser/col_arithmetic_operation.sim b/tests/script/general/parser/col_arithmetic_operation.sim index 4977e906d7..3208df95e4 100644 --- a/tests/script/general/parser/col_arithmetic_operation.sim +++ b/tests/script/general/parser/col_arithmetic_operation.sim @@ -109,39 +109,42 @@ if $data02 != 225000 then return -1 endi -sql select first(speed) - last(speed), first(speed) as b, last(speed) as b, min(speed) - max(speed), spread(speed) from stb interval(1y)", NULL); +sql select first(c1) - last(c1), first(c1) as b, last(c1) as b, min(c1) - max(c1), spread(c1) from ca_stb0 interval(1y) if $rows != 1 then return -1 endi -if $data00 != @20-01-01 00:00:00.000@ then +if $data00 != @18-01-01 00:00:00.000@ then return -1 endi -if $data01 != -20.000000000 then +if $data01 != -9.000000000 then return -1 endi -if $data02 != 100.000000000 then +if $data02 != 0 then return -1 endi -if $data03 != 120.000000000 then +if $data03 != 9 then return -1 endi -if $data04 != -20.000000000 then +if $data04 != -9.000000000 then return -1 endi -if $data05 != 20.000000000 then +if $data05 != 9.000000000 then return -1 endi -sql_error select first(speed, mileage) - last(speed, mileage) from stb interval(1y) +sql_error select first(c1, c2) - last(c1, c2) from stb interval(1y) sql_error select first(ts) - last(ts) from stb interval(1y) -sql_error select top(speed, 2) - last(speed) from stb; -sql_error select stddev(speed) - last(speed) from stb; -sql_error select diff(speed) - last(speed) from stb; +sql_error select top(c1, 2) - last(c1) from stb; +sql_error select stddev(c1) - last(c1) from stb; +sql_error select diff(c1) - last(c1) from stb; +sql_error select first(c7) - last(c7) from stb; +sql_error select first(c8) - last(c8) from stb; +sql_error select first(c9) - last(c9) from stb; system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file From 478fd09f288c15e883edc9e2325a148acd9ba0db Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Wed, 21 Oct 2020 14:59:49 +0800 Subject: [PATCH 43/51] fix TD-1758 --- src/client/src/tscUtil.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index a38be04ae0..e129657acc 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -686,16 +686,14 @@ static int32_t getRowExpandSize(STableMeta* pTableMeta) { int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SArray* pTableDataBlockList) { SSqlCmd* pCmd = &pSql->cmd; - // the maximum expanded size in byte when a row-wise data is converted to SDataRow format - STableDataBlocks* pOneTableBlock = taosArrayGetP(pTableDataBlockList, 0); - int32_t expandSize = getRowExpandSize(pOneTableBlock->pTableMeta); - void* pVnodeDataBlockHashList = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); SArray* pVnodeDataBlockList = taosArrayInit(8, POINTER_BYTES); size_t total = taosArrayGetSize(pTableDataBlockList); for (int32_t i = 0; i < total; ++i) { - pOneTableBlock = taosArrayGetP(pTableDataBlockList, i); + // the maximum expanded size in byte when a row-wise data is converted to SDataRow format + STableDataBlocks* pOneTableBlock = taosArrayGetP(pTableDataBlockList, i); + int32_t expandSize = getRowExpandSize(pOneTableBlock->pTableMeta); STableDataBlocks* dataBuf = NULL; int32_t ret = From c3a31c681d158964363139c33b5e61936aab071f Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 21 Oct 2020 15:11:57 +0800 Subject: [PATCH 44/51] [td-1765] --- src/client/src/tscSQLParser.c | 2 +- tests/script/general/parser/tags_dynamically_specifiy.sim | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index b28dd9cf1c..e2573f7e19 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -4750,7 +4750,7 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { // validate the length of binary if ((pTagsSchema->type == TSDB_DATA_TYPE_BINARY || pTagsSchema->type == TSDB_DATA_TYPE_NCHAR) && - (pVarList->a[1].pVar.nLen + VARSTR_HEADER_SIZE) > pTagsSchema->bytes) { + varDataTLen(pAlterSQL->tagData.data) > pTagsSchema->bytes) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg14); } diff --git a/tests/script/general/parser/tags_dynamically_specifiy.sim b/tests/script/general/parser/tags_dynamically_specifiy.sim index 0a5d5c9716..07bf4d8dd1 100644 --- a/tests/script/general/parser/tags_dynamically_specifiy.sim +++ b/tests/script/general/parser/tags_dynamically_specifiy.sim @@ -96,5 +96,10 @@ if $rows != 14 then return -1 endi +print ===============================> td-1765 +sql create table m1(ts timestamp, k int) tags(a binary(4), b nchar(4)); +sql create table tm0 using m1 tags('abcd', 'abcd'); +sql_error alter table tm0 set tag b = 'abcd1'; +sql_error alter table tm0 set tag a = 'abcd1'; system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file From 19cced6f0ec45ae8e31c368ceb3f40f58859f865 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 21 Oct 2020 15:29:44 +0800 Subject: [PATCH 45/51] TD-1768 --- src/inc/vnode.h | 2 ++ src/vnode/src/vnodeMain.c | 16 +++++---- src/vnode/src/vnodeRead.c | 71 +++++++++++++++++++------------------- src/vnode/src/vnodeWrite.c | 31 ++++++++++++----- 4 files changed, 69 insertions(+), 51 deletions(-) diff --git a/src/inc/vnode.h b/src/inc/vnode.h index ed6c232fdb..fdce4d6279 100644 --- a/src/inc/vnode.h +++ b/src/inc/vnode.h @@ -56,6 +56,7 @@ void vnodeRelease(void *pVnode); // dec refCount void* vnodeGetWal(void *pVnode); int32_t vnodeProcessWrite(void *pVnode, int qtype, void *pHead, void *item); +int32_t vnodeCheckWrite(void *pVnode); int32_t vnodeGetVnodeList(int32_t vnodeList[], int32_t *numOfVnodes); void vnodeBuildStatusMsg(void *param); void vnodeConfirmForward(void *param, uint64_t version, int32_t code); @@ -65,6 +66,7 @@ int32_t vnodeInitResources(); void vnodeCleanupResources(); int32_t vnodeProcessRead(void *pVnode, SReadMsg *pReadMsg); +int32_t vnodeCheckRead(void *pVnode); #ifdef __cplusplus } diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index bc728e86d9..e529f27f55 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -465,9 +465,10 @@ void *vnodeAcquireRqueue(int32_t vgId) { SVnodeObj *pVnode = vnodeAcquire(vgId); if (pVnode == NULL) return NULL; - if (pVnode->status == TAOS_VN_STATUS_RESET) { - terrno = TSDB_CODE_APP_NOT_READY; - vInfo("vgId:%d, status is in reset", vgId); + int32_t code = vnodeCheckRead(pVnode); + if (code != TSDB_CODE_SUCCESS) { + terrno = code; + vInfo("vgId:%d, can not provide read service, status is %s", vgId, vnodeStatus[pVnode->status]); vnodeRelease(pVnode); return NULL; } @@ -479,13 +480,14 @@ void *vnodeAcquireWqueue(int32_t vgId) { SVnodeObj *pVnode = vnodeAcquire(vgId); if (pVnode == NULL) return NULL; - if (pVnode->status == TAOS_VN_STATUS_RESET) { - terrno = TSDB_CODE_APP_NOT_READY; - vInfo("vgId:%d, status is in reset", vgId); + int32_t code = vnodeCheckWrite(pVnode); + if (code != TSDB_CODE_SUCCESS) { + terrno = code; + vInfo("vgId:%d, can not provide write service, status is %s", vgId, vnodeStatus[pVnode->status]); vnodeRelease(pVnode); return NULL; } - + return pVnode->wqueue; } diff --git a/src/vnode/src/vnodeRead.c b/src/vnode/src/vnodeRead.c index 2a39e2e76f..0ad85bbf91 100644 --- a/src/vnode/src/vnodeRead.c +++ b/src/vnode/src/vnodeRead.c @@ -38,7 +38,13 @@ void vnodeInitReadFp(void) { vnodeProcessReadMsgFp[TSDB_MSG_TYPE_FETCH] = vnodeProcessFetchMsg; } -static int32_t vnodeProcessReadImp(SVnodeObj *pVnode, SReadMsg *pReadMsg) { +// +// After the fetch request enters the vnode queue, if the vnode cannot provide services, the process function are +// still required, or there will be a deadlock, so we don’t do any check here, but put the check codes before the +// request enters the queue +// +int32_t vnodeProcessRead(void *param, SReadMsg *pReadMsg) { + SVnodeObj *pVnode = (SVnodeObj *)param; int msgType = pReadMsg->rpcMsg.msgType; if (vnodeProcessReadMsgFp[msgType] == NULL) { @@ -46,53 +52,36 @@ static int32_t vnodeProcessReadImp(SVnodeObj *pVnode, SReadMsg *pReadMsg) { return TSDB_CODE_VND_MSG_NOT_PROCESSED; } + return (*vnodeProcessReadMsgFp[msgType])(pVnode, pReadMsg); +} + +int32_t vnodeCheckRead(void *param) { + SVnodeObj *pVnode = param; if (pVnode->status != TAOS_VN_STATUS_READY) { - vDebug("vgId:%d, msgType:%s not processed, vnode status is %s", pVnode->vgId, taosMsg[msgType], - vnodeStatus[pVnode->status]); + vDebug("vgId:%d, vnode status is %s, recCount:%d pVnode:%p", pVnode->vgId, vnodeStatus[pVnode->status], + pVnode->refCount, pVnode); return TSDB_CODE_APP_NOT_READY; } // tsdb may be in reset state if (pVnode->tsdb == NULL) { - vDebug("vgId:%d, msgType:%s not processed, tsdb is null", pVnode->vgId, taosMsg[msgType]); - return TSDB_CODE_APP_NOT_READY; - } - - if (pVnode->status == TAOS_VN_STATUS_CLOSING) { - vDebug("vgId:%d, msgType:%s not processed, vstatus is %s", pVnode->vgId, taosMsg[msgType], - vnodeStatus[pVnode->status]); + vDebug("vgId:%d, tsdb is null, recCount:%d pVnode:%p", pVnode->vgId, pVnode->refCount, pVnode); return TSDB_CODE_APP_NOT_READY; } if (pVnode->role != TAOS_SYNC_ROLE_SLAVE && pVnode->role != TAOS_SYNC_ROLE_MASTER) { - vDebug("vgId:%d, msgType:%s not processed, replica:%d role:%s", pVnode->vgId, taosMsg[msgType], - pVnode->syncCfg.replica, syncRole[pVnode->role]); + vDebug("vgId:%d, replica:%d role:%s, recCount:%d pVnode:%p", pVnode->vgId, pVnode->syncCfg.replica, + syncRole[pVnode->role], pVnode->refCount, pVnode); return TSDB_CODE_APP_NOT_READY; } - return (*vnodeProcessReadMsgFp[msgType])(pVnode, pReadMsg); + return TSDB_CODE_SUCCESS; } -int32_t vnodeProcessRead(void *param, SReadMsg *pRead) { - SVnodeObj *pVnode = (SVnodeObj *)param; - int32_t code = vnodeProcessReadImp(pVnode, pRead); +static int32_t vnodePutItemIntoReadQueue(SVnodeObj *pVnode, void **qhandle) { + int32_t code = vnodeCheckRead(pVnode); + if (code != TSDB_CODE_SUCCESS) return code; - if (code == TSDB_CODE_APP_NOT_READY && pRead->rpcMsg.msgType == TSDB_MSG_TYPE_QUERY) { - // After the fetch request enters the vnode queue - // If the vnode cannot provide services, the following operations are still required - // Or, there will be a deadlock - void **qhandle = (void **)pRead->pCont; - vError("QInfo:%p msg:%p will be killed for vstatus is %s", *qhandle, pRead, vnodeStatus[pVnode->status]); - - // qKillQuery(*qhandle); - // qReleaseQInfo(pVnode->qMgmt, (void **)&qhandle, true); - return TSDB_CODE_APP_NOT_READY; - } else { - return code; - } -} - -static void vnodePutItemIntoReadQueue(SVnodeObj *pVnode, void **qhandle) { SReadMsg *pRead = (SReadMsg *)taosAllocateQitem(sizeof(SReadMsg)); pRead->rpcMsg.msgType = TSDB_MSG_TYPE_QUERY; pRead->pCont = qhandle; @@ -103,6 +92,8 @@ static void vnodePutItemIntoReadQueue(SVnodeObj *pVnode, void **qhandle) { vDebug("QInfo:%p add to vread queue for exec query, msg:%p", *qhandle, pRead); taosWriteQitem(pVnode->rqueue, TAOS_QTYPE_QUERY, pRead); + + return TSDB_CODE_SUCCESS; } static int32_t vnodeDumpQueryResult(SRspRet *pRet, void *pVnode, void **handle, bool *freeHandle) { @@ -112,8 +103,13 @@ static int32_t vnodeDumpQueryResult(SRspRet *pRet, void *pVnode, void **handle, if ((code = qDumpRetrieveResult(*handle, (SRetrieveTableRsp **)&pRet->rsp, &pRet->len, &continueExec)) == TSDB_CODE_SUCCESS) { if (continueExec) { *freeHandle = false; - vnodePutItemIntoReadQueue(pVnode, handle); - pRet->qhandle = *handle; + code = vnodePutItemIntoReadQueue(pVnode, handle); + if (code != TSDB_CODE_SUCCESS) { + *freeHandle = true; + return code; + } else { + pRet->qhandle = *handle; + } } else { *freeHandle = true; vDebug("QInfo:%p exec completed, free handle:%d", *handle, *freeHandle); @@ -214,7 +210,12 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { if (handle != NULL) { vDebug("vgId:%d, QInfo:%p, dnode query msg disposed, create qhandle and returns to app", vgId, *handle); - vnodePutItemIntoReadQueue(pVnode, handle); + code = vnodePutItemIntoReadQueue(pVnode, handle); + if (code != TSDB_CODE_SUCCESS) { + pRsp->code = code; + qReleaseQInfo(pVnode->qMgmt, (void **)&handle, true); + return pRsp->code; + } } } else { assert(pCont != NULL); diff --git a/src/vnode/src/vnodeWrite.c b/src/vnode/src/vnodeWrite.c index 0966192c94..f541c7717a 100644 --- a/src/vnode/src/vnodeWrite.c +++ b/src/vnode/src/vnodeWrite.c @@ -56,15 +56,6 @@ int32_t vnodeProcessWrite(void *param1, int qtype, void *param2, void *item) { return TSDB_CODE_VND_MSG_NOT_PROCESSED; } - if (!(pVnode->accessState & TSDB_VN_WRITE_ACCCESS)) { - vDebug("vgId:%d, msgType:%s not processed, no write auth", pVnode->vgId, taosMsg[pHead->msgType]); - return TSDB_CODE_VND_NO_WRITE_AUTH; - } - - // tsdb may be in reset state - if (pVnode->tsdb == NULL) return TSDB_CODE_APP_NOT_READY; - if (pVnode->status == TAOS_VN_STATUS_CLOSING) return TSDB_CODE_APP_NOT_READY; - if (pHead->version == 0) { // from client or CQ if (pVnode->status != TAOS_VN_STATUS_READY) { vDebug("vgId:%d, msgType:%s not processed, vnode status is %d", pVnode->vgId, taosMsg[pHead->msgType], @@ -105,6 +96,28 @@ int32_t vnodeProcessWrite(void *param1, int qtype, void *param2, void *item) { return syncCode; } +int32_t vnodeCheckWrite(void *param) { + SVnodeObj *pVnode = param; + if (!(pVnode->accessState & TSDB_VN_WRITE_ACCCESS)) { + vDebug("vgId:%d, no write auth, recCount:%d pVnode:%p", pVnode->vgId, pVnode->refCount, pVnode); + return TSDB_CODE_VND_NO_WRITE_AUTH; + } + + // tsdb may be in reset state + if (pVnode->tsdb == NULL) { + vDebug("vgId:%d, tsdb is null, recCount:%d pVnode:%p", pVnode->vgId, pVnode->refCount, pVnode); + return TSDB_CODE_APP_NOT_READY; + } + + if (pVnode->status == TAOS_VN_STATUS_CLOSING) { + vDebug("vgId:%d, vnode status is %s, recCount:%d pVnode:%p", pVnode->vgId, vnodeStatus[pVnode->status], + pVnode->refCount, pVnode); + return TSDB_CODE_APP_NOT_READY; + } + + return TSDB_CODE_SUCCESS; +} + void vnodeConfirmForward(void *param, uint64_t version, int32_t code) { SVnodeObj *pVnode = (SVnodeObj *)param; syncConfirmForward(pVnode->sync, version, code); From 9072e302e1e20715b464bb2c134b7a63d6ff399f Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Wed, 21 Oct 2020 15:46:34 +0800 Subject: [PATCH 46/51] [TD-1758] add test case --- tests/pytest/fulltest.sh | 1 + tests/pytest/insert/insertIntoTwoTables.py | 53 ++++++++++++++++++++++ 2 files changed, 54 insertions(+) create mode 100644 tests/pytest/insert/insertIntoTwoTables.py diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index fd973b8b76..86147a4b2b 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -18,6 +18,7 @@ python3 ./test.py -f insert/multi.py python3 ./test.py -f insert/randomNullCommit.py python3 insert/retentionpolicy.py python3 ./test.py -f insert/alterTableAndInsert.py +python3 ./test.py -f insert/insertIntoTwoTables.py python3 ./test.py -f table/column_name.py python3 ./test.py -f table/column_num.py diff --git a/tests/pytest/insert/insertIntoTwoTables.py b/tests/pytest/insert/insertIntoTwoTables.py new file mode 100644 index 0000000000..dc985fbe12 --- /dev/null +++ b/tests/pytest/insert/insertIntoTwoTables.py @@ -0,0 +1,53 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + tdSql.prepare() + + ### test case for TD-1758 ### + print("==============step1") + tdSql.execute( + "create table t0(ts timestamp, c int)") + tdSql.execute( + 'create table t1(ts timestamp, c binary(1))') + tdSql.execute( + "insert into t0 values(now,1) t1 values(now,'0')(now+1a,'1')(now+2a,'2')(now+3a,'3')(now+4a,'4')") + + print("==============step2") + + tdSql.query("select * from t0") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 1) + + tdSql.query("select * from t1") + tdSql.checkRows(5) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From fccb4851dee72cc0a70a41d0c837c098a88dfc35 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 21 Oct 2020 15:48:16 +0800 Subject: [PATCH 47/51] [td-225] --- src/query/src/qPercentile.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/query/src/qPercentile.c b/src/query/src/qPercentile.c index 3a8be781d5..ab9ffb7bcb 100644 --- a/src/query/src/qPercentile.c +++ b/src/query/src/qPercentile.c @@ -365,7 +365,7 @@ void tMemBucketDestroy(tMemBucket *pBucket) { taosTFree(pBucket); } -void tMemBucketUpdateBoundingBox(MinMaxEntry *r, char *data, int32_t dataType) { +void tMemBucketUpdateBoundingBox(MinMaxEntry *r, const char *data, int32_t dataType) { switch (dataType) { case TSDB_DATA_TYPE_INT: { int32_t val = *(int32_t *)data; From b8e1f3d3b4b2ce1f2f359128c683684b0efb02d6 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 21 Oct 2020 16:07:10 +0800 Subject: [PATCH 48/51] [td-225] fix compiler error. --- src/client/src/tscUtil.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index e53b58daa2..7e1c378584 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -1841,7 +1841,7 @@ static void doSetSqlExprAndResultFieldInfo(SQueryInfo* pQueryInfo, SQueryInfo* p for (int32_t i = 0; i < numOfExprs; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pNewQueryInfo, i); - TAOS_FIELD f = tscCreateField(pExpr->resType, pExpr->aliasName, pExpr->resBytes); + TAOS_FIELD f = tscCreateField((int8_t) pExpr->resType, pExpr->aliasName, pExpr->resBytes); SInternalField* pInfo1 = tscFieldInfoAppend(&pNewQueryInfo->fieldsInfo, &f); pInfo1->pSqlExpr = pExpr; } From c7d95710888cd71b8ff43003df4e2c84d87b1dea Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 21 Oct 2020 08:29:03 +0000 Subject: [PATCH 49/51] #3922 --- src/sync/CMakeLists.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/sync/CMakeLists.txt b/src/sync/CMakeLists.txt index efdf2bd185..60271c771c 100644 --- a/src/sync/CMakeLists.txt +++ b/src/sync/CMakeLists.txt @@ -5,14 +5,14 @@ INCLUDE_DIRECTORIES(inc) AUX_SOURCE_DIRECTORY(src SRC) IF (TD_LINUX) - LIST(REMOVE_ITEM SRC ./src/tarbitrator.c) + LIST(REMOVE_ITEM SRC src/tarbitrator.c) ADD_LIBRARY(sync ${SRC}) TARGET_LINK_LIBRARIES(sync tutil pthread common) - LIST(APPEND BIN_SRC ./src/tarbitrator.c) - LIST(APPEND BIN_SRC ./src/taosTcpPool.c) + LIST(APPEND BIN_SRC src/tarbitrator.c) + LIST(APPEND BIN_SRC src/taosTcpPool.c) ADD_EXECUTABLE(tarbitrator ${BIN_SRC}) TARGET_LINK_LIBRARIES(tarbitrator sync common osdetail tutil) - ADD_SUBDIRECTORY(test) + #ADD_SUBDIRECTORY(test) ENDIF () From 9101e5059153a63f5ee14c25a54d0486c4504c34 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 21 Oct 2020 09:02:25 +0000 Subject: [PATCH 50/51] fix compile errors --- src/client/src/tscSql.c | 6 +++--- src/inc/taos.h | 13 ++++++------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index 9799c9129b..7c856e6cc3 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -174,7 +174,7 @@ static void syncConnCallback(void *param, TAOS_RES *tres, int code) { TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, const char *auth, const char *db, uint16_t port) { STscObj *pObj = NULL; - SSqlObj *pSql = taosConnectImpl(ip, user, pass, auth, db, port, syncConnCallback, NULL, &pObj); + SSqlObj *pSql = taosConnectImpl(ip, user, pass, auth, db, port, syncConnCallback, NULL, (void **)&pObj); if (pSql != NULL) { pSql->fp = syncConnCallback; pSql->param = pSql; @@ -245,11 +245,11 @@ static void asyncConnCallback(void *param, TAOS_RES *tres, int code) { TAOS *taos_connect_a(char *ip, char *user, char *pass, char *db, uint16_t port, void (*fp)(void *, TAOS_RES *, int), void *param, TAOS **taos) { STscObj *pObj = NULL; - SSqlObj* pSql = taosConnectImpl(ip, user, pass, NULL, db, port, asyncConnCallback, param, &pObj); + SSqlObj *pSql = taosConnectImpl(ip, user, pass, NULL, db, port, asyncConnCallback, param, (void **)&pObj); if (pSql == NULL) { return NULL; } - + if (taos) *taos = pObj; pSql->fetchFp = fp; diff --git a/src/inc/taos.h b/src/inc/taos.h index 66b5f930bc..3153137347 100644 --- a/src/inc/taos.h +++ b/src/inc/taos.h @@ -22,13 +22,12 @@ extern "C" { #endif -typedef struct STscObj TAOS; -typedef struct STscStmt TAOS_STMT; -typedef struct SSqlObj TAOS_RES; -typedef struct SSqlStream TAOS_STREAM; -typedef struct SSub TAOS_SUB; -typedef unsigned char** TAOS_ROW; - +typedef void TAOS; +typedef void TAOS_STMT; +typedef void TAOS_RES; +typedef void TAOS_STREAM; +typedef void TAOS_SUB; +typedef void **TAOS_ROW; // Data type definition #define TSDB_DATA_TYPE_NULL 0 // 1 bytes From 2b343e33b3e0bdeb7269baba413e35b2f26f4c70 Mon Sep 17 00:00:00 2001 From: wangyazhou1313 Date: Thu, 22 Oct 2020 09:29:38 +0800 Subject: [PATCH 51/51] edit fulltest.sh add testcase table/alter_wal0.py & edit architectur-ch.md --- documentation20/webdocs/markdowndocs/architecture-ch.md | 2 +- tests/pytest/fulltest.sh | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/documentation20/webdocs/markdowndocs/architecture-ch.md b/documentation20/webdocs/markdowndocs/architecture-ch.md index ba45bc4796..d4705ccb05 100644 --- a/documentation20/webdocs/markdowndocs/architecture-ch.md +++ b/documentation20/webdocs/markdowndocs/architecture-ch.md @@ -228,7 +228,7 @@ TDengine采用数据驱动的方式让缓存中的数据写入硬盘进行持久 为充分利用时序数据特点,TDengine将一个vnode保存在持久化存储的数据切分成多个文件,每个文件只保存固定天数的数据,这个天数由系统配置参数days决定。切分成多个文件后,给定查询的起止日期,无需任何索引,就可以立即定位需要打开哪些数据文件,大大加快读取速度。 -对于采集的数据,一般有保留时长,这个时长由系统配置参数keep决定。超过这个设置天数的数据文件,将被系统将自动删除,释放存储空间。 +对于采集的数据,一般有保留时长,这个时长由系统配置参数keep决定。超过这个设置天数的数据文件,将被系统自动删除,释放存储空间。 给定days与keep两个参数,一个vnode总的数据文件数为:keep/days。总的数据文件个数不宜过大,也不宜过小。10到100以内合适。基于这个原则,可以设置合理的days。 目前的版本,参数keep可以修改,但对于参数days,一但设置后,不可修改。 diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index fd973b8b76..2a5909f271 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -19,6 +19,7 @@ python3 ./test.py -f insert/randomNullCommit.py python3 insert/retentionpolicy.py python3 ./test.py -f insert/alterTableAndInsert.py +python3 ./test.py -f table/alter_wal0.py python3 ./test.py -f table/column_name.py python3 ./test.py -f table/column_num.py python3 ./test.py -f table/db_table.py