From ec8cd0c6b8783dacf39e384e52fabf3e9cee6de8 Mon Sep 17 00:00:00 2001 From: kailixu Date: Sun, 9 Apr 2023 18:44:46 +0800 Subject: [PATCH 01/61] enh: support passwd version --- include/client/taos.h | 3 + include/common/tmsg.h | 16 +++ include/libs/catalog/catalog.h | 7 ++ source/client/inc/clientInt.h | 6 ++ source/client/src/clientHb.c | 89 ++++++++++++++++ source/client/src/clientMain.c | 17 ++++ source/common/src/tmsg.c | 58 ++++++++++- source/dnode/mnode/impl/inc/mndDef.h | 1 + source/dnode/mnode/impl/inc/mndUser.h | 2 + source/dnode/mnode/impl/src/mndDump.c | 1 + source/dnode/mnode/impl/src/mndProfile.c | 10 ++ source/dnode/mnode/impl/src/mndUser.c | 123 ++++++++++++++++++++++- 12 files changed, 330 insertions(+), 3 deletions(-) diff --git a/include/client/taos.h b/include/client/taos.h index cf410a42da..bfb29456be 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -101,6 +101,7 @@ typedef struct TAOS_FIELD_E { #endif typedef void (*__taos_async_fn_t)(void *param, TAOS_RES *res, int code); +typedef void (*__taos_notify_fn_t)(void *param); typedef struct TAOS_MULTI_BIND { int buffer_type; @@ -225,6 +226,8 @@ DLL_EXPORT int taos_get_tables_vgId(TAOS *taos, const char *db, const char *tabl DLL_EXPORT int taos_load_table_info(TAOS *taos, const char *tableNameList); +DLL_EXPORT int taos_set_notify_cb(TAOS *taos, __taos_notify_fn_t *fp, void *param); + /* --------------------------schemaless INTERFACE------------------------------- */ DLL_EXPORT TAOS_RES *taos_schemaless_insert(TAOS *taos, char *lines[], int numLines, int protocol, int precision); diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 999cc17767..9e2ef087e0 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -106,6 +106,7 @@ enum { HEARTBEAT_KEY_DBINFO, HEARTBEAT_KEY_STBINFO, HEARTBEAT_KEY_TMQ, + HEARTBEAT_KEY_USER_PASSINFO, }; typedef enum _mgmt_table { @@ -704,6 +705,13 @@ int32_t tSerializeSGetUserAuthRsp(void* buf, int32_t bufLen, SGetUserAuthRsp* pR int32_t tDeserializeSGetUserAuthRsp(void* buf, int32_t bufLen, SGetUserAuthRsp* pRsp); void tFreeSGetUserAuthRsp(SGetUserAuthRsp* pRsp); +typedef SGetUserAuthReq SGetUserPassReq; + +typedef struct { + char user[TSDB_USER_LEN]; + int32_t version; +} SGetUserPassRsp; + /* * for client side struct, only column id, type, bytes are necessary * But for data in vnode side, we need all the following information. @@ -1034,6 +1042,14 @@ int32_t tSerializeSUserAuthBatchRsp(void* buf, int32_t bufLen, SUserAuthBatchRsp int32_t tDeserializeSUserAuthBatchRsp(void* buf, int32_t bufLen, SUserAuthBatchRsp* pRsp); void tFreeSUserAuthBatchRsp(SUserAuthBatchRsp* pRsp); +typedef struct { + SArray* pArray; // Array of SGetUserPassRsp +} SUserPassBatchRsp; + +int32_t tSerializeSUserPassBatchRsp(void* buf, int32_t bufLen, SUserPassBatchRsp* pRsp); +int32_t tDeserializeSUserPassBatchRsp(void* buf, int32_t bufLen, SUserPassBatchRsp* pRsp); +void tFreeSUserPassBatchRsp(SUserPassBatchRsp* pRsp); + typedef struct { char db[TSDB_DB_FNAME_LEN]; STimeWindow timeRange; diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index fbb24d2862..498a6e4c7d 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -140,6 +140,11 @@ typedef struct SUserAuthVersion { int32_t version; } SUserAuthVersion; +typedef struct SUserPassVersion { + char user[TSDB_USER_LEN]; + int32_t version; +} SUserPassVersion; + typedef SDbCfgRsp SDbCfgInfo; typedef SUserIndexRsp SIndexInfo; @@ -320,6 +325,8 @@ int32_t catalogChkAuthFromCache(SCatalog* pCtg, const char* user, const char* db int32_t catalogUpdateUserAuthInfo(SCatalog* pCtg, SGetUserAuthRsp* pAuth); +// int32_t catalogUpdateUserPassInfo(SCatalog* pCtg, SGetUserPassRsp* pPass); + int32_t catalogUpdateVgEpSet(SCatalog* pCtg, const char* dbFName, int32_t vgId, SEpSet* epSet); int32_t catalogGetServerVersion(SCatalog* pCtg, SRequestConnInfo* pConn, char** pVersion); diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 86db35b412..8878275d68 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -133,6 +133,11 @@ typedef struct SAppInfo { TdThreadMutex mutex; } SAppInfo; +typedef struct { + int32_t ver; + __taos_notify_fn_t* fp; +} SPassInfo; + typedef struct STscObj { char user[TSDB_USER_LEN]; char pass[TSDB_PASSWORD_LEN]; @@ -148,6 +153,7 @@ typedef struct STscObj { int32_t numOfReqs; // number of sqlObj bound to this connection SAppInstInfo* pAppInfo; SHashObj* pRequests; + SPassInfo passInfo; } STscObj; typedef struct STscDbg { diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index c9c2e7a5f8..f547f9123c 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -49,6 +49,42 @@ static int32_t hbProcessUserAuthInfoRsp(void *value, int32_t valueLen, struct SC return TSDB_CODE_SUCCESS; } +static int32_t hbProcessUserPassInfoRsp(void *value, int32_t valueLen, SClientHbKey *connKey) { + int32_t code = 0; + STscObj *pTscObj = (STscObj *)acquireTscObj(connKey->tscRid); + if (NULL == pTscObj) { + tscWarn("tscObj rid %" PRIx64 " not exist", connKey->tscRid); + return TSDB_CODE_SUCCESS; + } + + SUserPassBatchRsp batchRsp = {0}; + if (tDeserializeSUserPassBatchRsp(value, valueLen, &batchRsp) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + releaseTscObj(connKey->tscRid); + assert(0); + return -1; + } + + SPassInfo *passInfo = &pTscObj->passInfo; + int32_t numOfBatchs = taosArrayGetSize(batchRsp.pArray); + for (int32_t i = 0; i < numOfBatchs; ++i) { + SGetUserPassRsp *rsp = taosArrayGet(batchRsp.pArray, i); + if (0 == strncmp(rsp->user, pTscObj->user, TSDB_USER_LEN)) { + tscError("update user:%s passVer from %d to %d", rsp->user, passInfo->ver, rsp->version); + if (atomic_load_32(&passInfo->ver) < rsp->version) { + atomic_store_32(&passInfo->ver, rsp->version); + if (passInfo->fp) { + (*passInfo->fp)(NULL); + } + } + } + } + + taosArrayDestroy(batchRsp.pArray); + releaseTscObj(connKey->tscRid); + return TSDB_CODE_SUCCESS; +} + static int32_t hbGenerateVgInfoFromRsp(SDBVgInfo **pInfo, SUseDbRsp *rsp) { int32_t code = 0; SDBVgInfo *vgInfo = taosMemoryCalloc(1, sizeof(SDBVgInfo)); @@ -291,6 +327,15 @@ static int32_t hbQueryHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) { hbProcessStbInfoRsp(kv->value, kv->valueLen, pCatalog); break; } + case HEARTBEAT_KEY_USER_PASSINFO: { + if (kv->valueLen <= 0 || NULL == kv->value) { + tscError("invalid hb user pass info, len:%d, value:%p", kv->valueLen, kv->value); + break; + } + + hbProcessUserPassInfoRsp(kv->value, kv->valueLen, &pRsp->connKey); + break; + } default: tscError("invalid hb key type:%d", kv->key); break; @@ -472,6 +517,48 @@ int32_t hbGetQueryBasicInfo(SClientHbKey *connKey, SClientHbReq *req) { return TSDB_CODE_SUCCESS; } +static int32_t hbGetUserBasicInfo(SClientHbKey *connKey, SClientHbReq *req) { + STscObj *pTscObj = (STscObj *)acquireTscObj(connKey->tscRid); + if (!pTscObj) { + tscWarn("tscObj rid %" PRIx64 " not exist", connKey->tscRid); + return TSDB_CODE_APP_ERROR; + } + + int32_t code = 0; + SUserPassVersion *user = taosMemoryMalloc(sizeof(SUserPassVersion)); + if (!user) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _return; + } + strncpy(user->user, pTscObj->user, TSDB_USER_LEN); + user->version = htonl(pTscObj->passInfo.ver); + + SKv kv = { + .key = HEARTBEAT_KEY_USER_PASSINFO, + .valueLen = sizeof(SUserPassVersion), + .value = user, + }; + + tscDebug("hb got user basic info, valueLen:%d", kv.valueLen); + + if (!req->info) { + req->info = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK); + } + + if (taosHashPut(req->info, &kv.key, sizeof(kv.key), &kv, sizeof(kv)) < 0) { + code = terrno ? terrno : TSDB_CODE_APP_ERROR; + goto _return; + } + +_return: + releaseTscObj(connKey->tscRid); + if (code) { + tscError("hb got user basic info failed since %s", terrstr(code)); + } + + return code; +} + int32_t hbGetExpiredUserInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SClientHbReq *req) { SUserAuthVersion *users = NULL; uint32_t userNum = 0; @@ -620,6 +707,8 @@ int32_t hbQueryHbReqHandle(SClientHbKey *connKey, void *param, SClientHbReq *req hbGetQueryBasicInfo(connKey, req); + hbGetUserBasicInfo(connKey, req); + code = hbGetExpiredUserInfo(connKey, pCatalog, req); if (TSDB_CODE_SUCCESS != code) { return code; diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 60c7b44b3d..182750b821 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -119,6 +119,23 @@ TAOS *taos_connect(const char *ip, const char *user, const char *pass, const cha return NULL; } +int taos_set_notify_cb(TAOS *taos, __taos_notify_fn_t *fp, void *param) { + if (taos == NULL) { + return 0; + } + + STscObj *pObj = acquireTscObj(*(int64_t *)taos); + if (NULL == pObj) { + tscError("invalid parameter for %s", __func__); + terrno = TSDB_CODE_TSC_DISCONNECTED; + return -1; + } + + pObj->passInfo.fp = fp; + + return 0; +} + void taos_close_internal(void *taos) { if (taos == NULL) { return; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index c37c0e830a..edde771e03 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -192,6 +192,8 @@ void *taosDecodeSEpSet(const void *buf, SEpSet *pEp) { static int32_t tSerializeSClientHbReq(SEncoder *pEncoder, const SClientHbReq *pReq) { if (tEncodeSClientHbKey(pEncoder, &pReq->connKey) < 0) return -1; + + if (pReq->connKey.connType == CONN_TYPE__QUERY) { if (tEncodeI64(pEncoder, pReq->app.appId) < 0) return -1; if (tEncodeI32(pEncoder, pReq->app.pid) < 0) return -1; @@ -212,6 +214,7 @@ static int32_t tSerializeSClientHbReq(SEncoder *pEncoder, const SClientHbReq *pR queryNum = 1; if (tEncodeI32(pEncoder, queryNum) < 0) return -1; if (tEncodeU32(pEncoder, pReq->query->connId) < 0) return -1; + if (tEncodeI64(pEncoder, pReq->clusterId) < 0) return -1; int32_t num = taosArrayGetSize(pReq->query->queryDesc); if (tEncodeI32(pEncoder, num) < 0) return -1; @@ -276,7 +279,7 @@ static int32_t tDeserializeSClientHbReq(SDecoder *pDecoder, SClientHbReq *pReq) pReq->query = taosMemoryCalloc(1, sizeof(*pReq->query)); if (NULL == pReq->query) return -1; if (tDecodeU32(pDecoder, &pReq->query->connId) < 0) return -1; - + if (tDecodeI64(pDecoder, &pReq->clusterId) < 0) return -1; int32_t num = 0; if (tDecodeI32(pDecoder, &num) < 0) return -1; if (num > 0) { @@ -2781,6 +2784,59 @@ void tFreeSUserAuthBatchRsp(SUserAuthBatchRsp *pRsp) { taosArrayDestroy(pRsp->pArray); } +int32_t tSerializeSUserPassBatchRsp(void *buf, int32_t bufLen, SUserPassBatchRsp *pRsp) { + SEncoder encoder = {0}; + tEncoderInit(&encoder, buf, bufLen); + + if (tStartEncode(&encoder) < 0) return -1; + + int32_t numOfBatch = taosArrayGetSize(pRsp->pArray); + if (tEncodeI32(&encoder, numOfBatch) < 0) return -1; + for (int32_t i = 0; i < numOfBatch; ++i) { + SGetUserPassRsp *pUserPassRsp = taosArrayGet(pRsp->pArray, i); + if (tEncodeCStr(&encoder, pUserPassRsp->user) < 0) return -1; + if (tEncodeI32(&encoder, pUserPassRsp->version) < 0) return -1; + } + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tEncoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSUserPassBatchRsp(void *buf, int32_t bufLen, SUserPassBatchRsp *pRsp) { + SDecoder decoder = {0}; + tDecoderInit(&decoder, buf, bufLen); + + if (tStartDecode(&decoder) < 0) return -1; + + int32_t numOfBatch = taosArrayGetSize(pRsp->pArray); + if (tDecodeI32(&decoder, &numOfBatch) < 0) return -1; + + pRsp->pArray = taosArrayInit(numOfBatch, sizeof(SGetUserPassRsp)); + if (pRsp->pArray == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + for (int32_t i = 0; i < numOfBatch; ++i) { + SGetUserPassRsp rsp = {0}; + if (tDecodeCStrTo(&decoder, rsp.user) < 0) return -1; + if (tDecodeI32(&decoder, &rsp.version) < 0) return -1; + taosArrayPush(pRsp->pArray, &rsp); + } + tEndDecode(&decoder); + + tDecoderClear(&decoder); + return 0; +} + +void tFreeSUserPassBatchRsp(SUserPassBatchRsp *pRsp) { + if(pRsp) { + taosArrayDestroy(pRsp->pArray); + } +} + int32_t tSerializeSDbCfgReq(void *buf, int32_t bufLen, SDbCfgReq *pReq) { SEncoder encoder = {0}; tEncoderInit(&encoder, buf, bufLen); diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 876f2651bc..de7eafae46 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -278,6 +278,7 @@ typedef struct { int8_t reserve; int32_t acctId; int32_t authVersion; + int32_t passVersion; SHashObj* readDbs; SHashObj* writeDbs; SHashObj* topics; diff --git a/source/dnode/mnode/impl/inc/mndUser.h b/source/dnode/mnode/impl/inc/mndUser.h index 8943ba703e..8a1561ca20 100644 --- a/source/dnode/mnode/impl/inc/mndUser.h +++ b/source/dnode/mnode/impl/inc/mndUser.h @@ -34,6 +34,8 @@ SHashObj *mndDupDbHash(SHashObj *pOld); SHashObj *mndDupTopicHash(SHashObj *pOld); int32_t mndValidateUserAuthInfo(SMnode *pMnode, SUserAuthVersion *pUsers, int32_t numOfUses, void **ppRsp, int32_t *pRspLen); +int32_t mndValidateUserPassInfo(SMnode *pMnode, SUserPassVersion *pUsers, int32_t numOfUses, void **ppRsp, + int32_t *pRspLen); int32_t mndUserRemoveDb(SMnode *pMnode, STrans *pTrans, char *db); int32_t mndUserRemoveTopic(SMnode *pMnode, STrans *pTrans, char *topic); diff --git a/source/dnode/mnode/impl/src/mndDump.c b/source/dnode/mnode/impl/src/mndDump.c index 44a7d49fff..a991bddda8 100644 --- a/source/dnode/mnode/impl/src/mndDump.c +++ b/source/dnode/mnode/impl/src/mndDump.c @@ -421,6 +421,7 @@ void dumpUser(SSdb *pSdb, SJson *json) { tjsonAddStringToObject(item, "updateTime", i642str(pObj->updateTime)); tjsonAddStringToObject(item, "superUser", i642str(pObj->superUser)); tjsonAddStringToObject(item, "authVersion", i642str(pObj->authVersion)); + tjsonAddStringToObject(item, "passVersion", i642str(pObj->passVersion)); tjsonAddStringToObject(item, "numOfReadDbs", i642str(taosHashGetSize(pObj->readDbs))); tjsonAddStringToObject(item, "numOfWriteDbs", i642str(taosHashGetSize(pObj->writeDbs))); sdbRelease(pSdb, pObj); diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index 50e502f4ab..6e78982b03 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -547,6 +547,16 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb } break; } + case HEARTBEAT_KEY_USER_PASSINFO: { + void *rspMsg = NULL; + int32_t rspLen = 0; + mndValidateUserPassInfo(pMnode, kv->value, kv->valueLen / sizeof(SUserPassVersion), &rspMsg, &rspLen); + if (rspMsg && rspLen > 0) { + SKv kv1 = {.key = HEARTBEAT_KEY_USER_PASSINFO, .valueLen = rspLen, .value = rspMsg}; + taosArrayPush(hbRsp.info, &kv1); + } + break; + } default: mError("invalid kv key:%d", kv->key); hbRsp.status = TSDB_CODE_APP_ERROR; diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 92b73aed96..2245e09187 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -22,7 +22,7 @@ #include "mndTrans.h" #include "tbase64.h" -#define USER_VER_NUMBER 2 +#define USER_VER_NUMBER 3 #define USER_RESERVE_SIZE 64 static int32_t mndCreateDefaultUsers(SMnode *pMnode); @@ -142,6 +142,7 @@ SSdbRaw *mndUserActionEncode(SUserObj *pUser) { SDB_SET_INT8(pRaw, dataPos, pUser->enable, _OVER) SDB_SET_INT8(pRaw, dataPos, pUser->reserve, _OVER) SDB_SET_INT32(pRaw, dataPos, pUser->authVersion, _OVER) + SDB_SET_INT32(pRaw, dataPos, pUser->passVersion, _OVER) SDB_SET_INT32(pRaw, dataPos, numOfReadDbs, _OVER) SDB_SET_INT32(pRaw, dataPos, numOfWriteDbs, _OVER) SDB_SET_INT32(pRaw, dataPos, numOfTopics, _OVER) @@ -188,7 +189,7 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) { int8_t sver = 0; if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER; - if (sver != 1 && sver != 2) { + if (sver < 1 || sver > USER_VER_NUMBER) { terrno = TSDB_CODE_SDB_INVALID_DATA_VER; goto _OVER; } @@ -210,6 +211,9 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) { SDB_GET_INT8(pRaw, dataPos, &pUser->enable, _OVER) SDB_GET_INT8(pRaw, dataPos, &pUser->reserve, _OVER) SDB_GET_INT32(pRaw, dataPos, &pUser->authVersion, _OVER) + if (sver >= 3) { + SDB_GET_INT32(pRaw, dataPos, &pUser->passVersion, _OVER) + } int32_t numOfReadDbs = 0; int32_t numOfWriteDbs = 0; @@ -322,6 +326,7 @@ static int32_t mndUserActionUpdate(SSdb *pSdb, SUserObj *pOld, SUserObj *pNew) { taosWLockLatch(&pOld->lock); pOld->updateTime = pNew->updateTime; pOld->authVersion = pNew->authVersion; + pOld->passVersion = pNew->passVersion; pOld->sysInfo = pNew->sysInfo; pOld->enable = pNew->enable; memcpy(pOld->pass, pNew->pass, TSDB_PASSWORD_LEN); @@ -543,10 +548,14 @@ static int32_t mndProcessAlterUserReq(SRpcMsg *pReq) { if (mndUserDupObj(pUser, &newUser) != 0) goto _OVER; + newUser.passVersion = pUser->passVersion; if (alterReq.alterType == TSDB_ALTER_USER_PASSWD) { char pass[TSDB_PASSWORD_LEN + 1] = {0}; taosEncryptPass_c((uint8_t *)alterReq.pass, strlen(alterReq.pass), pass); memcpy(newUser.pass, pass, TSDB_PASSWORD_LEN); + if (0 != strncmp(pUser->pass, pass, TSDB_PASSWORD_LEN)) { + ++newUser.passVersion; + } } if (alterReq.alterType == TSDB_ALTER_USER_SUPERUSER) { @@ -782,6 +791,50 @@ _OVER: return code; } +#if 0 +static int32_t mndProcessGetUserPassReq(SRpcMsg *pReq) { + SMnode *pMnode = pReq->info.node; + int32_t code = -1; + SUserObj *pUser = NULL; + SGetUserPassReq req = {0}; + SGetUserPassRsp rsp = {0}; + + if (tDeserializeSGetUserPassReq(pReq->pCont, pReq->contLen, &req) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto _OVER; + } + + mTrace("user:%s, start to get pass", req.user); + + pUser = mndAcquireUser(pMnode, req.user); + if (pUser == NULL) { + terrno = TSDB_CODE_MND_USER_NOT_EXIST; + goto _OVER; + } + + memcpy(rsp.user, pUser->user, TSDB_USER_LEN); + rsp.version = pUser->passVersion; + + int32_t contLen = tSerializeSGetUserPassRsp(NULL, 0, &rsp); + void *pRsp = rpcMallocCont(contLen); + if (pRsp == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _OVER; + } + + tSerializeSGetUserPassRsp(pRsp, contLen, &rsp); + + pReq->info.rsp = pRsp; + pReq->info.rspLen = contLen; + code = 0; + +_OVER: + + mndReleaseUser(pMnode, pUser); + + return code; +} +#endif static int32_t mndRetrieveUsers(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { SMnode *pMnode = pReq->info.node; @@ -1019,6 +1072,72 @@ _OVER: return code; } +int32_t mndValidateUserPassInfo(SMnode *pMnode, SUserPassVersion *pUsers, int32_t numOfUses, void **ppRsp, + int32_t *pRspLen) { + int32_t code = 0; + SUserPassBatchRsp batchRsp = {0}; + + for (int32_t i = 0; i < numOfUses; ++i) { + SUserObj *pUser = mndAcquireUser(pMnode, pUsers[i].user); + if (pUser == NULL) { + mError("user:%s, failed to validate user pass since %s", pUsers[i].user, terrstr()); + continue; + } + + pUsers[i].version = ntohl(pUsers[i].version); + if (pUser->passVersion <= pUsers[i].version) { + mDebug("user:%s, not update since mnd passVer %d <= client passVer %d", pUsers[i].user, pUser->passVersion, + pUsers[i].version); + mndReleaseUser(pMnode, pUser); + continue; + } + + SGetUserPassRsp rsp = {0}; + memcpy(rsp.user, pUser->user, TSDB_USER_LEN); + rsp.version = pUser->passVersion; + + if (!batchRsp.pArray && !(batchRsp.pArray = taosArrayInit(numOfUses, sizeof(SGetUserPassRsp)))) { + code = TSDB_CODE_OUT_OF_MEMORY; + assert(0); + goto _OVER; + } + + taosArrayPush(batchRsp.pArray, &rsp); + mndReleaseUser(pMnode, pUser); + } + + if (taosArrayGetSize(batchRsp.pArray) <= 0) { + goto _OVER; + } + + int32_t rspLen = tSerializeSUserPassBatchRsp(NULL, 0, &batchRsp); + if (rspLen < 0) { + assert(0); + code = TSDB_CODE_OUT_OF_MEMORY; + goto _OVER; + } + void *pRsp = taosMemoryMalloc(rspLen); + if (pRsp == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + assert(0); + goto _OVER; + } + tSerializeSUserPassBatchRsp(pRsp, rspLen, &batchRsp); + + *ppRsp = pRsp; + *pRspLen = rspLen; + +_OVER: + if (code) { + *ppRsp = NULL; + *pRspLen = 0; + assert(0); + } + + tFreeSUserPassBatchRsp(&batchRsp); + return code; +} + int32_t mndUserRemoveDb(SMnode *pMnode, STrans *pTrans, char *db) { int32_t code = 0; SSdb *pSdb = pMnode->pSdb; From b30d90355aeef51a6a85fd0086a902677b97a9c8 Mon Sep 17 00:00:00 2001 From: kailixu Date: Sun, 9 Apr 2023 18:55:05 +0800 Subject: [PATCH 02/61] chore: more code --- include/libs/catalog/catalog.h | 2 -- source/client/src/clientMain.c | 1 + source/common/src/tmsg.c | 4 --- source/dnode/mnode/impl/src/mndUser.c | 44 --------------------------- 4 files changed, 1 insertion(+), 50 deletions(-) diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index 498a6e4c7d..5895d9404d 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -325,8 +325,6 @@ int32_t catalogChkAuthFromCache(SCatalog* pCtg, const char* user, const char* db int32_t catalogUpdateUserAuthInfo(SCatalog* pCtg, SGetUserAuthRsp* pAuth); -// int32_t catalogUpdateUserPassInfo(SCatalog* pCtg, SGetUserPassRsp* pPass); - int32_t catalogUpdateVgEpSet(SCatalog* pCtg, const char* dbFName, int32_t vgId, SEpSet* epSet); int32_t catalogGetServerVersion(SCatalog* pCtg, SRequestConnInfo* pConn, char** pVersion); diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 182750b821..3969e7e771 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -133,6 +133,7 @@ int taos_set_notify_cb(TAOS *taos, __taos_notify_fn_t *fp, void *param) { pObj->passInfo.fp = fp; + releaseTscObj(*(int64_t *)taos); return 0; } diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index edde771e03..2943de116d 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -192,8 +192,6 @@ void *taosDecodeSEpSet(const void *buf, SEpSet *pEp) { static int32_t tSerializeSClientHbReq(SEncoder *pEncoder, const SClientHbReq *pReq) { if (tEncodeSClientHbKey(pEncoder, &pReq->connKey) < 0) return -1; - - if (pReq->connKey.connType == CONN_TYPE__QUERY) { if (tEncodeI64(pEncoder, pReq->app.appId) < 0) return -1; if (tEncodeI32(pEncoder, pReq->app.pid) < 0) return -1; @@ -214,7 +212,6 @@ static int32_t tSerializeSClientHbReq(SEncoder *pEncoder, const SClientHbReq *pR queryNum = 1; if (tEncodeI32(pEncoder, queryNum) < 0) return -1; if (tEncodeU32(pEncoder, pReq->query->connId) < 0) return -1; - if (tEncodeI64(pEncoder, pReq->clusterId) < 0) return -1; int32_t num = taosArrayGetSize(pReq->query->queryDesc); if (tEncodeI32(pEncoder, num) < 0) return -1; @@ -279,7 +276,6 @@ static int32_t tDeserializeSClientHbReq(SDecoder *pDecoder, SClientHbReq *pReq) pReq->query = taosMemoryCalloc(1, sizeof(*pReq->query)); if (NULL == pReq->query) return -1; if (tDecodeU32(pDecoder, &pReq->query->connId) < 0) return -1; - if (tDecodeI64(pDecoder, &pReq->clusterId) < 0) return -1; int32_t num = 0; if (tDecodeI32(pDecoder, &num) < 0) return -1; if (num > 0) { diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 2245e09187..54e5eac20e 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -791,50 +791,6 @@ _OVER: return code; } -#if 0 -static int32_t mndProcessGetUserPassReq(SRpcMsg *pReq) { - SMnode *pMnode = pReq->info.node; - int32_t code = -1; - SUserObj *pUser = NULL; - SGetUserPassReq req = {0}; - SGetUserPassRsp rsp = {0}; - - if (tDeserializeSGetUserPassReq(pReq->pCont, pReq->contLen, &req) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - goto _OVER; - } - - mTrace("user:%s, start to get pass", req.user); - - pUser = mndAcquireUser(pMnode, req.user); - if (pUser == NULL) { - terrno = TSDB_CODE_MND_USER_NOT_EXIST; - goto _OVER; - } - - memcpy(rsp.user, pUser->user, TSDB_USER_LEN); - rsp.version = pUser->passVersion; - - int32_t contLen = tSerializeSGetUserPassRsp(NULL, 0, &rsp); - void *pRsp = rpcMallocCont(contLen); - if (pRsp == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto _OVER; - } - - tSerializeSGetUserPassRsp(pRsp, contLen, &rsp); - - pReq->info.rsp = pRsp; - pReq->info.rspLen = contLen; - code = 0; - -_OVER: - - mndReleaseUser(pMnode, pUser); - - return code; -} -#endif static int32_t mndRetrieveUsers(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { SMnode *pMnode = pReq->info.node; From 7eb7cb72637b6f4215a3274396e44c4bc7486e15 Mon Sep 17 00:00:00 2001 From: kailixu Date: Sun, 9 Apr 2023 18:57:04 +0800 Subject: [PATCH 03/61] chore: more code --- source/libs/transport/src/transCli.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index c23d6d0a1f..50ed9fa61b 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -462,7 +462,6 @@ void cliHandleExceptImpl(SCliConn* pConn, int32_t code) { if (transQueueEmpty(&pConn->cliMsgs)) { if (pConn->broken == true && CONN_NO_PERSIST_BY_APP(pConn)) { tTrace("%s conn %p handle except, persist:0", CONN_GET_INST_LABEL(pConn), pConn); - if (T_REF_VAL_GET(pConn) > 1) transUnrefCliHandle(pConn); transUnrefCliHandle(pConn); return; } @@ -522,7 +521,6 @@ void cliHandleExceptImpl(SCliConn* pConn, int32_t code) { destroyCmsg(pMsg); tTrace("%s conn %p start to destroy, ref:%d", CONN_GET_INST_LABEL(pConn), pConn, T_REF_VAL_GET(pConn)); } while (!transQueueEmpty(&pConn->cliMsgs)); - if (T_REF_VAL_GET(pConn) > 1) transUnrefCliHandle(pConn); transUnrefCliHandle(pConn); } void cliHandleExcept(SCliConn* conn) { From e689356c6583646048e63ee5e3685103c46a9150 Mon Sep 17 00:00:00 2001 From: kailixu Date: Sun, 9 Apr 2023 19:01:51 +0800 Subject: [PATCH 04/61] chore: more code --- include/common/tmsg.h | 9 +++++---- include/libs/catalog/catalog.h | 5 ----- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 9e2ef087e0..1ea12c3a10 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -705,12 +705,13 @@ int32_t tSerializeSGetUserAuthRsp(void* buf, int32_t bufLen, SGetUserAuthRsp* pR int32_t tDeserializeSGetUserAuthRsp(void* buf, int32_t bufLen, SGetUserAuthRsp* pRsp); void tFreeSGetUserAuthRsp(SGetUserAuthRsp* pRsp); -typedef SGetUserAuthReq SGetUserPassReq; - -typedef struct { +typedef struct SUserPassVersion { char user[TSDB_USER_LEN]; int32_t version; -} SGetUserPassRsp; +} SUserPassVersion; + +typedef SGetUserAuthReq SGetUserPassReq; +typedef SUserPassVersion SGetUserPassRsp; /* * for client side struct, only column id, type, bytes are necessary diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index 5895d9404d..fbb24d2862 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -140,11 +140,6 @@ typedef struct SUserAuthVersion { int32_t version; } SUserAuthVersion; -typedef struct SUserPassVersion { - char user[TSDB_USER_LEN]; - int32_t version; -} SUserPassVersion; - typedef SDbCfgRsp SDbCfgInfo; typedef SUserIndexRsp SIndexInfo; From 8c70dca4a9095430809f881ea46a32545a1d2634 Mon Sep 17 00:00:00 2001 From: kailixu Date: Sun, 9 Apr 2023 19:04:11 +0800 Subject: [PATCH 05/61] chore: more code --- source/common/src/tmsg.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 2943de116d..a59fa7a2d6 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -276,6 +276,7 @@ static int32_t tDeserializeSClientHbReq(SDecoder *pDecoder, SClientHbReq *pReq) pReq->query = taosMemoryCalloc(1, sizeof(*pReq->query)); if (NULL == pReq->query) return -1; if (tDecodeU32(pDecoder, &pReq->query->connId) < 0) return -1; + int32_t num = 0; if (tDecodeI32(pDecoder, &num) < 0) return -1; if (num > 0) { From 76386b9fad07e4c78620bddd64166df87c826f68 Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 10 Apr 2023 13:07:18 +0800 Subject: [PATCH 06/61] enh: assign passVer during taos_connect --- include/client/taos.h | 6 +++++- include/common/tmsg.h | 1 + source/client/src/clientHb.c | 13 ++++++------- source/client/src/clientMsgHandler.c | 1 + source/common/src/tmsg.c | 8 ++++++++ source/dnode/mnode/impl/src/mndProfile.c | 1 + source/dnode/mnode/impl/src/mndUser.c | 2 +- 7 files changed, 23 insertions(+), 9 deletions(-) diff --git a/include/client/taos.h b/include/client/taos.h index bfb29456be..530b40f5aa 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -101,7 +101,7 @@ typedef struct TAOS_FIELD_E { #endif typedef void (*__taos_async_fn_t)(void *param, TAOS_RES *res, int code); -typedef void (*__taos_notify_fn_t)(void *param); +typedef void (*__taos_notify_fn_t)(void *param, void *ext, int type); typedef struct TAOS_MULTI_BIND { int buffer_type; @@ -122,6 +122,10 @@ typedef enum { SET_CONF_RET_ERR_TOO_LONG = -6 } SET_CONF_RET_CODE; +typedef enum { + TAOS_NOTIFY_PASSVER = 1, +} TAOS_NOTIFY_TYPE; + #define RET_MSG_LENGTH 1024 typedef struct setConfRet { SET_CONF_RET_CODE retCode; diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 1ea12c3a10..b7332dd1cc 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -629,6 +629,7 @@ typedef struct { int8_t connType; SEpSet epSet; int32_t svrTimestamp; + int32_t passVer; char sVer[TSDB_VERSION_LEN]; char sDetailVer[128]; } SConnectRsp; diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index f547f9123c..6596bc4bb3 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -54,15 +54,14 @@ static int32_t hbProcessUserPassInfoRsp(void *value, int32_t valueLen, SClientHb STscObj *pTscObj = (STscObj *)acquireTscObj(connKey->tscRid); if (NULL == pTscObj) { tscWarn("tscObj rid %" PRIx64 " not exist", connKey->tscRid); - return TSDB_CODE_SUCCESS; + return code; } SUserPassBatchRsp batchRsp = {0}; if (tDeserializeSUserPassBatchRsp(value, valueLen, &batchRsp) != 0) { - terrno = TSDB_CODE_INVALID_MSG; + code = TSDB_CODE_INVALID_MSG; releaseTscObj(connKey->tscRid); - assert(0); - return -1; + return code; } SPassInfo *passInfo = &pTscObj->passInfo; @@ -70,11 +69,11 @@ static int32_t hbProcessUserPassInfoRsp(void *value, int32_t valueLen, SClientHb for (int32_t i = 0; i < numOfBatchs; ++i) { SGetUserPassRsp *rsp = taosArrayGet(batchRsp.pArray, i); if (0 == strncmp(rsp->user, pTscObj->user, TSDB_USER_LEN)) { - tscError("update user:%s passVer from %d to %d", rsp->user, passInfo->ver, rsp->version); + tscDebug("update passVer of user %s from %d to %d", rsp->user, passInfo->ver, rsp->version); if (atomic_load_32(&passInfo->ver) < rsp->version) { atomic_store_32(&passInfo->ver, rsp->version); if (passInfo->fp) { - (*passInfo->fp)(NULL); + (*passInfo->fp)(&pTscObj->id, NULL, TAOS_NOTIFY_PASSVER); } } } @@ -82,7 +81,7 @@ static int32_t hbProcessUserPassInfoRsp(void *value, int32_t valueLen, SClientHb taosArrayDestroy(batchRsp.pArray); releaseTscObj(connKey->tscRid); - return TSDB_CODE_SUCCESS; + return code; } static int32_t hbGenerateVgInfoFromRsp(SDBVgInfo **pInfo, SUseDbRsp *rsp) { diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index a0146cfa39..52c5fc7940 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -130,6 +130,7 @@ int32_t processConnectRsp(void* param, SDataBuf* pMsg, int32_t code) { lastClusterId = connectRsp.clusterId; pTscObj->connType = connectRsp.connType; + pTscObj->passInfo.ver = connectRsp.passVer; hbRegisterConn(pTscObj->pAppInfo->pAppHbMgr, pTscObj->id, connectRsp.clusterId, connectRsp.connType); diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index a59fa7a2d6..1f9a44b1f4 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -3874,6 +3874,7 @@ int32_t tSerializeSConnectRsp(void *buf, int32_t bufLen, SConnectRsp *pRsp) { if (tEncodeI32(&encoder, pRsp->svrTimestamp) < 0) return -1; if (tEncodeCStr(&encoder, pRsp->sVer) < 0) return -1; if (tEncodeCStr(&encoder, pRsp->sDetailVer) < 0) return -1; + if (tEncodeI32(&encoder, pRsp->passVer) < 0) return -1; tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -3897,6 +3898,13 @@ int32_t tDeserializeSConnectRsp(void *buf, int32_t bufLen, SConnectRsp *pRsp) { if (tDecodeI32(&decoder, &pRsp->svrTimestamp) < 0) return -1; if (tDecodeCStrTo(&decoder, pRsp->sVer) < 0) return -1; if (tDecodeCStrTo(&decoder, pRsp->sDetailVer) < 0) return -1; + + if (!tDecodeIsEnd(&decoder)) { + if (tDecodeI32(&decoder, &pRsp->passVer) < 0) return -1; + } else { + pRsp->passVer = 0; + } + tEndDecode(&decoder); tDecoderClear(&decoder); diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index 6e78982b03..2ebb5aeb99 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -283,6 +283,7 @@ static int32_t mndProcessConnectReq(SRpcMsg *pReq) { connectRsp.connType = connReq.connType; connectRsp.dnodeNum = mndGetDnodeSize(pMnode); connectRsp.svrTimestamp = taosGetTimestampSec(); + connectRsp.passVer = pUser->passVersion; strcpy(connectRsp.sVer, version); snprintf(connectRsp.sDetailVer, sizeof(connectRsp.sDetailVer), "ver:%s\nbuild:%s\ngitinfo:%s", version, buildinfo, diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 54e5eac20e..de4768d746 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -1042,7 +1042,7 @@ int32_t mndValidateUserPassInfo(SMnode *pMnode, SUserPassVersion *pUsers, int32_ pUsers[i].version = ntohl(pUsers[i].version); if (pUser->passVersion <= pUsers[i].version) { - mDebug("user:%s, not update since mnd passVer %d <= client passVer %d", pUsers[i].user, pUser->passVersion, + mTrace("user:%s, not update since mnd passVer %d <= client passVer %d", pUsers[i].user, pUser->passVersion, pUsers[i].version); mndReleaseUser(pMnode, pUser); continue; From f6078dc27e07c2c0263a50ef244988e93402b908 Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 10 Apr 2023 15:11:54 +0800 Subject: [PATCH 07/61] chore: more code --- include/client/taos.h | 2 +- source/client/src/clientMain.c | 10 +++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/include/client/taos.h b/include/client/taos.h index 530b40f5aa..73086ce666 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -123,7 +123,7 @@ typedef enum { } SET_CONF_RET_CODE; typedef enum { - TAOS_NOTIFY_PASSVER = 1, + TAOS_NOTIFY_PASSVER = 0, } TAOS_NOTIFY_TYPE; #define RET_MSG_LENGTH 1024 diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 3969e7e771..a216b2afb2 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -131,7 +131,15 @@ int taos_set_notify_cb(TAOS *taos, __taos_notify_fn_t *fp, void *param) { return -1; } - pObj->passInfo.fp = fp; + int32_t type = param ? *(int32_t *)param : 0; + switch (type) { + case TAOS_NOTIFY_PASSVER: { + pObj->passInfo.fp = fp; + break; + } + default: + break; + } releaseTscObj(*(int64_t *)taos); return 0; From 539337ca0be0a2d91962cf83212757600c0d1e15 Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 10 Apr 2023 16:12:54 +0800 Subject: [PATCH 08/61] chore: code optimization --- source/client/src/clientMain.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index a216b2afb2..2491f5d70a 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -121,14 +121,13 @@ TAOS *taos_connect(const char *ip, const char *user, const char *pass, const cha int taos_set_notify_cb(TAOS *taos, __taos_notify_fn_t *fp, void *param) { if (taos == NULL) { - return 0; + return TSDB_CODE_INVALID_PARA; } STscObj *pObj = acquireTscObj(*(int64_t *)taos); if (NULL == pObj) { tscError("invalid parameter for %s", __func__); - terrno = TSDB_CODE_TSC_DISCONNECTED; - return -1; + return TSDB_CODE_TSC_DISCONNECTED; } int32_t type = param ? *(int32_t *)param : 0; From 5889ea12615b73c4fef2446037ea43d6316c740c Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 17 Apr 2023 14:19:44 +0800 Subject: [PATCH 09/61] chore: support specify param by app --- include/client/taos.h | 2 +- source/client/inc/clientInt.h | 1 + source/client/src/clientHb.c | 2 +- source/client/src/clientMain.c | 4 ++-- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/include/client/taos.h b/include/client/taos.h index 73086ce666..bca93f611f 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -230,7 +230,7 @@ DLL_EXPORT int taos_get_tables_vgId(TAOS *taos, const char *db, const char *tabl DLL_EXPORT int taos_load_table_info(TAOS *taos, const char *tableNameList); -DLL_EXPORT int taos_set_notify_cb(TAOS *taos, __taos_notify_fn_t *fp, void *param); +DLL_EXPORT int taos_set_notify_cb(TAOS *taos, __taos_notify_fn_t *fp, void *param, int type); /* --------------------------schemaless INTERFACE------------------------------- */ diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 8878275d68..93ea433723 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -135,6 +135,7 @@ typedef struct SAppInfo { typedef struct { int32_t ver; + void* param; __taos_notify_fn_t* fp; } SPassInfo; diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 6596bc4bb3..dd40a18c88 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -73,7 +73,7 @@ static int32_t hbProcessUserPassInfoRsp(void *value, int32_t valueLen, SClientHb if (atomic_load_32(&passInfo->ver) < rsp->version) { atomic_store_32(&passInfo->ver, rsp->version); if (passInfo->fp) { - (*passInfo->fp)(&pTscObj->id, NULL, TAOS_NOTIFY_PASSVER); + (*passInfo->fp)(&pTscObj->id, passInfo->param, TAOS_NOTIFY_PASSVER); } } } diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 2491f5d70a..b47e5bf53b 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -119,7 +119,7 @@ TAOS *taos_connect(const char *ip, const char *user, const char *pass, const cha return NULL; } -int taos_set_notify_cb(TAOS *taos, __taos_notify_fn_t *fp, void *param) { +int taos_set_notify_cb(TAOS *taos, __taos_notify_fn_t *fp, void *param, int type) { if (taos == NULL) { return TSDB_CODE_INVALID_PARA; } @@ -130,10 +130,10 @@ int taos_set_notify_cb(TAOS *taos, __taos_notify_fn_t *fp, void *param) { return TSDB_CODE_TSC_DISCONNECTED; } - int32_t type = param ? *(int32_t *)param : 0; switch (type) { case TAOS_NOTIFY_PASSVER: { pObj->passInfo.fp = fp; + pObj->passInfo.param = param; break; } default: From 97aa8aede20394b5d90f819df02cead3532b143e Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 17 Apr 2023 14:26:02 +0800 Subject: [PATCH 10/61] chore: revert --- source/common/src/tmsg.c | 2 +- source/dnode/mnode/impl/src/mndStb.c | 2 +- source/libs/transport/src/transCli.c | 2 ++ 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 0ce46d1cd0..1b0b4b1e4b 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -276,7 +276,7 @@ static int32_t tDeserializeSClientHbReq(SDecoder *pDecoder, SClientHbReq *pReq) pReq->query = taosMemoryCalloc(1, sizeof(*pReq->query)); if (NULL == pReq->query) return -1; if (tDecodeU32(pDecoder, &pReq->query->connId) < 0) return -1; - + int32_t num = 0; if (tDecodeI32(pDecoder, &num) < 0) return -1; if (num > 0) { diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 3f2f4062f1..63bcef2a5b 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -2929,7 +2929,7 @@ static int32_t mndRetrieveStb(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc if (pDb != NULL && pStb->dbUid != pDb->uid) { sdbRelease(pSdb, pStb); - continue; + continue; } cols = 0; diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 50ed9fa61b..c23d6d0a1f 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -462,6 +462,7 @@ void cliHandleExceptImpl(SCliConn* pConn, int32_t code) { if (transQueueEmpty(&pConn->cliMsgs)) { if (pConn->broken == true && CONN_NO_PERSIST_BY_APP(pConn)) { tTrace("%s conn %p handle except, persist:0", CONN_GET_INST_LABEL(pConn), pConn); + if (T_REF_VAL_GET(pConn) > 1) transUnrefCliHandle(pConn); transUnrefCliHandle(pConn); return; } @@ -521,6 +522,7 @@ void cliHandleExceptImpl(SCliConn* pConn, int32_t code) { destroyCmsg(pMsg); tTrace("%s conn %p start to destroy, ref:%d", CONN_GET_INST_LABEL(pConn), pConn, T_REF_VAL_GET(pConn)); } while (!transQueueEmpty(&pConn->cliMsgs)); + if (T_REF_VAL_GET(pConn) > 1) transUnrefCliHandle(pConn); transUnrefCliHandle(pConn); } void cliHandleExcept(SCliConn* conn) { From c0148ed5e195670e22be1c2e74305340c00b73db Mon Sep 17 00:00:00 2001 From: kailixu Date: Tue, 18 Apr 2023 11:11:44 +0800 Subject: [PATCH 11/61] chore: remove the assertion --- source/dnode/mnode/impl/src/mndUser.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 635aa6326a..1c64024eac 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -1455,7 +1455,6 @@ int32_t mndValidateUserPassInfo(SMnode *pMnode, SUserPassVersion *pUsers, int32_ if (!batchRsp.pArray && !(batchRsp.pArray = taosArrayInit(numOfUses, sizeof(SGetUserPassRsp)))) { code = TSDB_CODE_OUT_OF_MEMORY; - assert(0); goto _OVER; } @@ -1469,14 +1468,12 @@ int32_t mndValidateUserPassInfo(SMnode *pMnode, SUserPassVersion *pUsers, int32_ int32_t rspLen = tSerializeSUserPassBatchRsp(NULL, 0, &batchRsp); if (rspLen < 0) { - assert(0); code = TSDB_CODE_OUT_OF_MEMORY; goto _OVER; } void *pRsp = taosMemoryMalloc(rspLen); if (pRsp == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - assert(0); goto _OVER; } tSerializeSUserPassBatchRsp(pRsp, rspLen, &batchRsp); @@ -1488,7 +1485,6 @@ _OVER: if (code) { *ppRsp = NULL; *pRspLen = 0; - assert(0); } tFreeSUserPassBatchRsp(&batchRsp); From 696d294f02cdebf8eb8efb99c99e9de04edaa9a5 Mon Sep 17 00:00:00 2001 From: kailixu Date: Tue, 18 Apr 2023 19:58:02 +0800 Subject: [PATCH 12/61] chore: code optimization --- source/client/src/clientHb.c | 7 ++++++- source/client/src/clientMain.c | 13 +++++++++---- source/dnode/mnode/impl/src/mndUser.c | 1 + 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index dd40a18c88..04e45609e6 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -523,7 +523,12 @@ static int32_t hbGetUserBasicInfo(SClientHbKey *connKey, SClientHbReq *req) { return TSDB_CODE_APP_ERROR; } - int32_t code = 0; + int32_t code = 0; + + if (!pTscObj->passInfo.fp) { + goto _return; + } + SUserPassVersion *user = taosMemoryMalloc(sizeof(SUserPassVersion)); if (!user) { code = TSDB_CODE_OUT_OF_MEMORY; diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index b47e5bf53b..4de8c408c7 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -121,13 +121,15 @@ TAOS *taos_connect(const char *ip, const char *user, const char *pass, const cha int taos_set_notify_cb(TAOS *taos, __taos_notify_fn_t *fp, void *param, int type) { if (taos == NULL) { - return TSDB_CODE_INVALID_PARA; + terrno = TSDB_CODE_INVALID_PARA; + return terrno; } STscObj *pObj = acquireTscObj(*(int64_t *)taos); if (NULL == pObj) { + terrno = TSDB_CODE_TSC_DISCONNECTED; tscError("invalid parameter for %s", __func__); - return TSDB_CODE_TSC_DISCONNECTED; + return terrno; } switch (type) { @@ -136,8 +138,11 @@ int taos_set_notify_cb(TAOS *taos, __taos_notify_fn_t *fp, void *param, int type pObj->passInfo.param = param; break; } - default: - break; + default: { + terrno = TSDB_CODE_INVALID_PARA; + releaseTscObj(*(int64_t *)taos); + return terrno; + } } releaseTscObj(*(int64_t *)taos); diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 1c64024eac..f6e5895cda 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -1455,6 +1455,7 @@ int32_t mndValidateUserPassInfo(SMnode *pMnode, SUserPassVersion *pUsers, int32_ if (!batchRsp.pArray && !(batchRsp.pArray = taosArrayInit(numOfUses, sizeof(SGetUserPassRsp)))) { code = TSDB_CODE_OUT_OF_MEMORY; + mndReleaseUser(pMnode, pUser); goto _OVER; } From dd85357b399786f5ac19fc3a4066e3fa5915aaec Mon Sep 17 00:00:00 2001 From: kailixu Date: Wed, 19 Apr 2023 14:19:36 +0800 Subject: [PATCH 13/61] chore: more code --- source/client/src/clientHb.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 04e45609e6..7f6356284a 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -73,7 +73,7 @@ static int32_t hbProcessUserPassInfoRsp(void *value, int32_t valueLen, SClientHb if (atomic_load_32(&passInfo->ver) < rsp->version) { atomic_store_32(&passInfo->ver, rsp->version); if (passInfo->fp) { - (*passInfo->fp)(&pTscObj->id, passInfo->param, TAOS_NOTIFY_PASSVER); + (*passInfo->fp)(passInfo->param, &passInfo->ver, TAOS_NOTIFY_PASSVER); } } } From 9549c1d929e262dfdc29f6b9d6cfaf5204e826c1 Mon Sep 17 00:00:00 2001 From: kailixu Date: Wed, 19 Apr 2023 14:47:06 +0800 Subject: [PATCH 14/61] chore: fix --- include/client/taos.h | 2 +- source/client/inc/clientInt.h | 6 +++--- source/client/src/clientMain.c | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/include/client/taos.h b/include/client/taos.h index bca93f611f..2bc705a041 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -230,7 +230,7 @@ DLL_EXPORT int taos_get_tables_vgId(TAOS *taos, const char *db, const char *tabl DLL_EXPORT int taos_load_table_info(TAOS *taos, const char *tableNameList); -DLL_EXPORT int taos_set_notify_cb(TAOS *taos, __taos_notify_fn_t *fp, void *param, int type); +DLL_EXPORT int taos_set_notify_cb(TAOS *taos, __taos_notify_fn_t fp, void *param, int type); /* --------------------------schemaless INTERFACE------------------------------- */ diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 93ea433723..e34385bc24 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -134,9 +134,9 @@ typedef struct SAppInfo { } SAppInfo; typedef struct { - int32_t ver; - void* param; - __taos_notify_fn_t* fp; + int32_t ver; + void* param; + __taos_notify_fn_t fp; } SPassInfo; typedef struct STscObj { diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 4de8c408c7..b3f7c844e9 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -119,7 +119,7 @@ TAOS *taos_connect(const char *ip, const char *user, const char *pass, const cha return NULL; } -int taos_set_notify_cb(TAOS *taos, __taos_notify_fn_t *fp, void *param, int type) { +int taos_set_notify_cb(TAOS *taos, __taos_notify_fn_t fp, void *param, int type) { if (taos == NULL) { terrno = TSDB_CODE_INVALID_PARA; return terrno; From fb99654899dc31a4f35fb9faf76e443bd5db6007 Mon Sep 17 00:00:00 2001 From: kailixu Date: Fri, 21 Apr 2023 18:22:44 +0800 Subject: [PATCH 15/61] chore: add UT --- tests/script/api/makefile | 3 + tests/script/api/passwdTest.c | 144 ++++++++++++++++++++++++++++++++++ 2 files changed, 147 insertions(+) create mode 100644 tests/script/api/passwdTest.c diff --git a/tests/script/api/makefile b/tests/script/api/makefile index 6739794cc8..6d55d8a75f 100644 --- a/tests/script/api/makefile +++ b/tests/script/api/makefile @@ -15,8 +15,11 @@ exe: gcc $(CFLAGS) ./stopquery.c -o $(ROOT)stopquery $(LFLAGS) gcc $(CFLAGS) ./dbTableRoute.c -o $(ROOT)dbTableRoute $(LFLAGS) gcc $(CFLAGS) ./insertSameTs.c -o $(ROOT)insertSameTs $(LFLAGS) + gcc $(CFLAGS) ./passwdTest.c -o $(ROOT)passwdTest $(LFLAGS) clean: rm $(ROOT)batchprepare rm $(ROOT)stopquery rm $(ROOT)dbTableRoute + rm $(ROOT)insertSameTs + rm $(ROOT)passwdTest diff --git a/tests/script/api/passwdTest.c b/tests/script/api/passwdTest.c new file mode 100644 index 0000000000..f3161047e8 --- /dev/null +++ b/tests/script/api/passwdTest.c @@ -0,0 +1,144 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +// TAOS standard API example. The same syntax as MySQL, but only a subset +// to compile: gcc -o demo demo.c -ltaos + +#include +#include +#include +#include +#include +#include "taos.h" // TAOS header file + +#define nRepetition 1 +#define nTaos 10 + +void Test(TAOS *taos, char *qstr); +void passVerTestMulti(const char *host, char *qstr); + +int nPassVerNotifiedMulti = 0; + +void __taos_notify_cb(void *param, void *ext, int type) { + switch (type) { + case TAOS_NOTIFY_PASSVER: { + ++nPassVerNotifiedMulti; + printf("%s:%d type:%d user:%s ver:%d\n", __func__, __LINE__, type, param ? (char *)param : "NULL", *(int *)ext); + break; + } + default: + printf("%s:%d unknown type:%d\n", __func__, __LINE__, type); + break; + } +} + +static void queryDB(TAOS *taos, char *command) { + int i; + TAOS_RES *pSql = NULL; + int32_t code = -1; + + for (i = 0; i < nRepetition; ++i) { + if (NULL != pSql) { + taos_free_result(pSql); + pSql = NULL; + } + + pSql = taos_query(taos, command); + code = taos_errno(pSql); + if (0 == code) { + break; + } + } + + if (code != 0) { + fprintf(stderr, "failed to run: %s, reason: %s\n", command, taos_errstr(pSql)); + taos_free_result(pSql); + taos_close(taos); + exit(EXIT_FAILURE); + } else { + fprintf(stderr, "success to run: %s\n", command); + } + + taos_free_result(pSql); +} + +int main(int argc, char *argv[]) { + char qstr[1024]; + + // connect to server + if (argc < 2) { + printf("please input server-ip \n"); + return 0; + } + + TAOS *taos = taos_connect(argv[1], "root", "taosdata", NULL, 0); + if (taos == NULL) { + printf("failed to connect to server, reason:%s\n", "null taos" /*taos_errstr(taos)*/); + exit(1); + } + + passVerTestMulti(argv[1], qstr); + + taos_close(taos); + taos_cleanup(); +} + +void passVerTestMulti(const char *host, char *qstr) { + TAOS *taos[nTaos] = {0}; + char *userName = calloc(1, 24); + strcpy(userName, "root"); + + for (int i = 0; i < nTaos; ++i) { + taos[i] = taos_connect(host, "root", "taosdata", NULL, 0); + if (taos[i] == NULL) { + printf("failed to connect to server, reason:%s\n", "null taos" /*taos_errstr(taos)*/); + exit(1); + } + + int code = taos_set_notify_cb(taos[i], __taos_notify_cb, userName, TAOS_NOTIFY_PASSVER); + + if (code != 0) { + fprintf(stderr, "failed to run: taos_set_notify_cb since %d\n", code); + } else { + fprintf(stderr, "success to run: taos_set_notify_cb\n"); + } + } + + queryDB(taos[0], "create database if not exists demo1 vgroups 1 minrows 10"); + queryDB(taos[0], "create database if not exists demo2 vgroups 1 minrows 10"); + queryDB(taos[0], "create database if not exists demo3 vgroups 1 minrows 10"); + + queryDB(taos[0], "create table demo1.stb (ts timestamp, c1 int) tags(t1 int)"); + queryDB(taos[0], "create table demo2.stb (ts timestamp, c1 int) tags(t1 int)"); + queryDB(taos[0], "create table demo3.stb (ts timestamp, c1 int) tags(t1 int)"); + + strcpy(qstr, "alter user root pass 'taos'"); + queryDB(taos[0], qstr); + + for (int i = 0; i < 10; ++i) { + if (nPassVerNotifiedMulti >= nTaos) break; + sleep(1); + } + + if (nPassVerNotifiedMulti >= nTaos) { + fprintf(stderr, "success to get passVer notification\n"); + } else { + fprintf(stderr, "failed to get passVer notification\n"); + } + + // sleep(1000); + + free(userName); +} \ No newline at end of file From bb3ce6a3d7db12ad58263b2f8bd251c3b4a1d95b Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Sun, 23 Apr 2023 14:54:12 +0800 Subject: [PATCH 16/61] enhance: error code refactor --- include/util/taoserror.h | 14 ++++++-------- source/libs/function/src/tudf.c | 11 ++++------- source/util/src/terror.c | 4 +--- 3 files changed, 11 insertions(+), 18 deletions(-) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 0d9292cc6b..f6195ceee4 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -712,15 +712,13 @@ int32_t* taosGetErrno(); #define TSDB_CODE_UDF_STOPPING TAOS_DEF_ERROR_CODE(0, 0x2901) #define TSDB_CODE_UDF_PIPE_READ_ERR TAOS_DEF_ERROR_CODE(0, 0x2902) #define TSDB_CODE_UDF_PIPE_CONNECT_ERR TAOS_DEF_ERROR_CODE(0, 0x2903) -#define TSDB_CODE_UDF_PIPE_NO_PIPE TAOS_DEF_ERROR_CODE(0, 0x2904) +#define TSDB_CODE_UDF_PIPE_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x2904) #define TSDB_CODE_UDF_LOAD_UDF_FAILURE TAOS_DEF_ERROR_CODE(0, 0x2905) -#define TSDB_CODE_UDF_INVALID_STATE TAOS_DEF_ERROR_CODE(0, 0x2906) -#define TSDB_CODE_UDF_INVALID_INPUT TAOS_DEF_ERROR_CODE(0, 0x2907) -#define TSDB_CODE_UDF_NO_FUNC_HANDLE TAOS_DEF_ERROR_CODE(0, 0x2908) -#define TSDB_CODE_UDF_INVALID_BUFSIZE TAOS_DEF_ERROR_CODE(0, 0x2909) -#define TSDB_CODE_UDF_INVALID_OUTPUT_TYPE TAOS_DEF_ERROR_CODE(0, 0x290A) -#define TSDB_CODE_UDF_SCRIPT_NOT_SUPPORTED TAOS_DEF_ERROR_CODE(0, 0x290B) -#define TSDB_CODE_UDF_FUNC_EXEC_FAILURE TAOS_DEF_ERROR_CODE(0, 0x290C) +#define TSDB_CODE_UDF_INVALID_INPUT TAOS_DEF_ERROR_CODE(0, 0x2906) +#define TSDB_CODE_UDF_INVALID_BUFSIZE TAOS_DEF_ERROR_CODE(0, 0x2907) +#define TSDB_CODE_UDF_INVALID_OUTPUT_TYPE TAOS_DEF_ERROR_CODE(0, 0x2908) +#define TSDB_CODE_UDF_SCRIPT_NOT_SUPPORTED TAOS_DEF_ERROR_CODE(0, 0x2909) +#define TSDB_CODE_UDF_FUNC_EXEC_FAILURE TAOS_DEF_ERROR_CODE(0, 0x290A) // sml #define TSDB_CODE_SML_INVALID_PROTOCOL_TYPE TAOS_DEF_ERROR_CODE(0, 0x3000) diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index 8c8b99a6f8..6b70422ac8 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -1580,7 +1580,7 @@ int32_t udfcStartUvTask(SClientUvTaskNode *uvTask) { case UV_TASK_REQ_RSP: { uv_pipe_t *pipe = uvTask->pipe; if (pipe == NULL) { - code = TSDB_CODE_UDF_PIPE_NO_PIPE; + code = TSDB_CODE_UDF_PIPE_NOT_EXIST; } else { uv_write_t *write = taosMemoryMalloc(sizeof(uv_write_t)); write->data = pipe->data; @@ -1598,7 +1598,7 @@ int32_t udfcStartUvTask(SClientUvTaskNode *uvTask) { case UV_TASK_DISCONNECT: { uv_pipe_t *pipe = uvTask->pipe; if (pipe == NULL) { - code = TSDB_CODE_UDF_PIPE_NO_PIPE; + code = TSDB_CODE_UDF_PIPE_NOT_EXIST; } else { SClientUvConn *conn = pipe->data; QUEUE_INSERT_TAIL(&conn->taskQueue, &uvTask->connTaskQueue); @@ -1759,9 +1759,6 @@ int32_t udfcRunUdfUvTask(SClientUdfTask *task, int8_t uvTaskType) { } int32_t doSetupUdf(char udfName[], UdfcFuncHandle *funcHandle) { - if (gUdfcProxy.udfcState != UDFC_STATE_READY) { - return TSDB_CODE_UDF_INVALID_STATE; - } SClientUdfTask *task = taosMemoryCalloc(1, sizeof(SClientUdfTask)); task->errCode = 0; task->session = taosMemoryCalloc(1, sizeof(SUdfcUvSession)); @@ -1804,7 +1801,7 @@ int32_t callUdf(UdfcFuncHandle handle, int8_t callType, SSDataBlock *input, SUdf SUdfcUvSession *session = (SUdfcUvSession *)handle; if (session->udfUvPipe == NULL) { fnError("No pipe to udfd"); - return TSDB_CODE_UDF_PIPE_NO_PIPE; + return TSDB_CODE_UDF_PIPE_NOT_EXIST; } SClientUdfTask *task = taosMemoryCalloc(1, sizeof(SClientUdfTask)); task->errCode = 0; @@ -1928,7 +1925,7 @@ int32_t doTeardownUdf(UdfcFuncHandle handle) { if (session->udfUvPipe == NULL) { fnError("tear down udf. pipe to udfd does not exist. udf name: %s", session->udfName); taosMemoryFree(session); - return TSDB_CODE_UDF_PIPE_NO_PIPE; + return TSDB_CODE_UDF_PIPE_NOT_EXIST; } SClientUdfTask *task = taosMemoryCalloc(1, sizeof(SClientUdfTask)); diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 34b09761c8..bcfb09d0e4 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -581,11 +581,9 @@ TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_DUP_TIMESTAMP, "Duplicate timestamps TAOS_DEFINE_ERROR(TSDB_CODE_UDF_STOPPING, "udf is stopping") TAOS_DEFINE_ERROR(TSDB_CODE_UDF_PIPE_READ_ERR, "udf pipe read error") TAOS_DEFINE_ERROR(TSDB_CODE_UDF_PIPE_CONNECT_ERR, "udf pipe connect error") -TAOS_DEFINE_ERROR(TSDB_CODE_UDF_PIPE_NO_PIPE, "udf no pipe") +TAOS_DEFINE_ERROR(TSDB_CODE_UDF_PIPE_NOT_EXIST, "udf no pipe") TAOS_DEFINE_ERROR(TSDB_CODE_UDF_LOAD_UDF_FAILURE, "udf load failure") -TAOS_DEFINE_ERROR(TSDB_CODE_UDF_INVALID_STATE, "udf invalid state") TAOS_DEFINE_ERROR(TSDB_CODE_UDF_INVALID_INPUT, "udf invalid function input") -TAOS_DEFINE_ERROR(TSDB_CODE_UDF_NO_FUNC_HANDLE, "udf no function handle") TAOS_DEFINE_ERROR(TSDB_CODE_UDF_INVALID_BUFSIZE, "udf invalid bufsize") TAOS_DEFINE_ERROR(TSDB_CODE_UDF_INVALID_OUTPUT_TYPE, "udf invalid output type") TAOS_DEFINE_ERROR(TSDB_CODE_UDF_SCRIPT_NOT_SUPPORTED, "udf program language not supported") From 8ce07b0cc7959705e740a541671a816e3a49ea2e Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Sun, 23 Apr 2023 15:00:51 +0800 Subject: [PATCH 17/61] fix: change error description of udf pipe not exist --- include/util/taoserror.h | 2 +- source/util/src/terror.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index f6195ceee4..418d5c8af7 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -712,7 +712,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_UDF_STOPPING TAOS_DEF_ERROR_CODE(0, 0x2901) #define TSDB_CODE_UDF_PIPE_READ_ERR TAOS_DEF_ERROR_CODE(0, 0x2902) #define TSDB_CODE_UDF_PIPE_CONNECT_ERR TAOS_DEF_ERROR_CODE(0, 0x2903) -#define TSDB_CODE_UDF_PIPE_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x2904) +#define TSDB_CODE_UDF_PIPE_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x2904) #define TSDB_CODE_UDF_LOAD_UDF_FAILURE TAOS_DEF_ERROR_CODE(0, 0x2905) #define TSDB_CODE_UDF_INVALID_INPUT TAOS_DEF_ERROR_CODE(0, 0x2906) #define TSDB_CODE_UDF_INVALID_BUFSIZE TAOS_DEF_ERROR_CODE(0, 0x2907) diff --git a/source/util/src/terror.c b/source/util/src/terror.c index bcfb09d0e4..dbdb40b93d 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -581,7 +581,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_DUP_TIMESTAMP, "Duplicate timestamps TAOS_DEFINE_ERROR(TSDB_CODE_UDF_STOPPING, "udf is stopping") TAOS_DEFINE_ERROR(TSDB_CODE_UDF_PIPE_READ_ERR, "udf pipe read error") TAOS_DEFINE_ERROR(TSDB_CODE_UDF_PIPE_CONNECT_ERR, "udf pipe connect error") -TAOS_DEFINE_ERROR(TSDB_CODE_UDF_PIPE_NOT_EXIST, "udf no pipe") +TAOS_DEFINE_ERROR(TSDB_CODE_UDF_PIPE_NOT_EXIST, "udf pipe not exist") TAOS_DEFINE_ERROR(TSDB_CODE_UDF_LOAD_UDF_FAILURE, "udf load failure") TAOS_DEFINE_ERROR(TSDB_CODE_UDF_INVALID_INPUT, "udf invalid function input") TAOS_DEFINE_ERROR(TSDB_CODE_UDF_INVALID_BUFSIZE, "udf invalid bufsize") From 5fe99c5ad24a8d96b75716b18d12f6aedc1b4595 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Sun, 23 Apr 2023 15:54:15 +0800 Subject: [PATCH 18/61] feat: support log long query --- include/common/tglobal.h | 8 ++++++ source/client/src/clientImpl.c | 2 +- source/common/src/tglobal.c | 44 +++++++++++++++++++++++++++++++ source/util/src/tlog.c | 48 +++++++++++++++++++++++++++++++--- 4 files changed, 98 insertions(+), 4 deletions(-) diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 6fde7b48a2..25a386b23d 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -24,6 +24,12 @@ extern "C" { #endif +#define SLOW_LOG_TYPE_QUERY 0x1 +#define SLOW_LOG_TYPE_INSERT 0x2 +#define SLOW_LOG_TYPE_OTHERS 0x4 +#define SLOW_LOG_TYPE_ALL 0xFFFFFFFF + + // cluster extern char tsFirst[]; extern char tsSecond[]; @@ -118,6 +124,8 @@ extern int32_t tsRedirectFactor; extern int32_t tsRedirectMaxPeriod; extern int32_t tsMaxRetryWaitTime; extern bool tsUseAdapter; +extern int32_t tsSlowLogThreshold; +extern int32_t tsSlowLogScope; // client extern int32_t tsMinSlidingTime; diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index ce174744ef..b9fa12b587 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1257,7 +1257,7 @@ STscObj* taosConnectImpl(const char* user, const char* auth, const char* db, __t if (pRequest->code != TSDB_CODE_SUCCESS) { const char* errorMsg = (pRequest->code == TSDB_CODE_RPC_FQDN_ERROR) ? taos_errstr(pRequest) : tstrerror(pRequest->code); - fprintf(stderr, "failed to connect to server, reason: %s\n\n", errorMsg); + tscError("failed to connect to server, reason: %s", errorMsg); terrno = pRequest->code; destroyRequest(pRequest); diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index da4a912238..224b0068fe 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -117,6 +117,10 @@ int32_t tsRedirectFactor = 2; int32_t tsRedirectMaxPeriod = 1000; int32_t tsMaxRetryWaitTime = 10000; bool tsUseAdapter = false; +int32_t tsSlowLogThreshold = 3; // seconds +int32_t tsSlowLogScope = SLOW_LOG_TYPE_ALL; + + /* @@ -345,6 +349,8 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { if (cfgAddBool(pCfg, "useAdapter", tsUseAdapter, true) != 0) return -1; if (cfgAddBool(pCfg, "crashReporting", tsEnableCrashReport, true) != 0) return -1; if (cfgAddInt64(pCfg, "queryMaxConcurrentTables", tsQueryMaxConcurrentTables, INT64_MIN, INT64_MAX, 1) != 0) return -1; + if (cfgAddInt32(pCfg, "slowLogThreshold", tsSlowLogThreshold, 0, INT32_MAX, true) != 0) return -1; + if (cfgAddString(pCfg, "slowLogScope", "", true) != 0) return -1; tsNumOfRpcThreads = tsNumOfCores / 2; tsNumOfRpcThreads = TRANGE(tsNumOfRpcThreads, 2, TSDB_MAX_RPC_THREADS); @@ -692,6 +698,36 @@ static void taosSetServerLogCfg(SConfig *pCfg) { metaDebugFlag = cfgGetItem(pCfg, "metaDebugFlag")->i32; } +static int32_t taosSetSlowLogScope(char *pScope) { + if (NULL == pScope || 0 == strlen(pScope)) { + tsSlowLogScope = SLOW_LOG_TYPE_ALL; + return 0; + } + + if (0 == strcasecmp(pScope, "all")) { + tsSlowLogScope = SLOW_LOG_TYPE_ALL; + return 0; + } + + if (0 == strcasecmp(pScope, "query")) { + tsSlowLogScope = SLOW_LOG_TYPE_QUERY; + return 0; + } + + if (0 == strcasecmp(pScope, "insert")) { + tsSlowLogScope = SLOW_LOG_TYPE_INSERT; + return 0; + } + + if (0 == strcasecmp(pScope, "others")) { + tsSlowLogScope = SLOW_LOG_TYPE_OTHERS; + return 0; + } + + uError("Invalid slowLog scope value:%s", pScope); + return -1; +} + static int32_t taosSetClientCfg(SConfig *pCfg) { tstrncpy(tsLocalFqdn, cfgGetItem(pCfg, "fqdn")->str, TSDB_FQDN_LEN); tsServerPort = (uint16_t)cfgGetItem(pCfg, "serverPort")->i32; @@ -742,6 +778,10 @@ static int32_t taosSetClientCfg(SConfig *pCfg) { tsUseAdapter = cfgGetItem(pCfg, "useAdapter")->bval; tsEnableCrashReport = cfgGetItem(pCfg, "crashReporting")->bval; tsQueryMaxConcurrentTables = cfgGetItem(pCfg, "queryMaxConcurrentTables")->i64; + tsSlowLogThreshold = cfgGetItem(pCfg, "slowLogThreshold")->i32; + if (taosSetSlowLogScope(cfgGetItem(pCfg, "slowLogScope")->str)) { + return -1; + } tsMaxRetryWaitTime = cfgGetItem(pCfg, "maxRetryWaitTime")->i32; @@ -1156,6 +1196,10 @@ int32_t taosSetCfg(SConfig *pCfg, char *name) { sDebugFlag = cfgGetItem(pCfg, "sDebugFlag")->i32; } else if (strcasecmp("smaDebugFlag", name) == 0) { smaDebugFlag = cfgGetItem(pCfg, "smaDebugFlag")->i32; + } else if (strcasecmp("slowLogThreshold", name) == 0) { + tsSlowLogThreshold = cfgGetItem(pCfg, "slowLogThreshold")->i32; + } else if (strcasecmp("slowLogScope", name) == 0) { + taosSetSlowLogScope(cfgGetItem(pCfg, "slowLogScope")->str) } break; } diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index a3d3c399ab..10ab389873 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -28,6 +28,7 @@ #define LOG_FILE_NAME_LEN 300 #define LOG_DEFAULT_BUF_SIZE (20 * 1024 * 1024) // 20MB +#define LOG_SLOW_BUF_SIZE (10 * 1024 * 1024) // 10MB #define LOG_DEFAULT_INTERVAL 25 #define LOG_INTERVAL_STEP 5 @@ -62,6 +63,7 @@ typedef struct { pid_t pid; char logName[LOG_FILE_NAME_LEN]; SLogBuff *logHandle; + SLogBuff *slowHandle; TdThreadMutex logMutex; } SLogObj; @@ -136,6 +138,34 @@ static int32_t taosStartLog() { return 0; } +int32_t taosInitSlowLog() { + char fullName[PATH_MAX] = {0}; + char logFileName[64] = {0}; +#ifdef CUS_PROMPT + snprintf(logFileName, 64, "%sSlowLog", CUS_PROMPT); +#else + snprintf(logFileName, 64, "taosSlowLog"); +#endif + + if (strlen(tsLogDir) != 0) { + snprintf(fullName, PATH_MAX, "%s" TD_DIRSEP "%s", tsLogDir, logFileName); + } else { + snprintf(fullName, PATH_MAX, "%s", logFileName); + } + + tsLogObj.slowHandle = taosLogBuffNew(LOG_SLOW_BUF_SIZE); + if (tsLogObj.slowHandle == NULL) return -1; + + taosUmaskFile(0); + tsLogObj.slowHandle->pFile = taosOpenFile(fullName, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND); + if (tsLogObj.slowHandle->pFile == NULL) { + printf("\nfailed to open slow log file:%s, reason:%s\n", fullName, strerror(errno)); + return -1; + } + + return 0; +} + int32_t taosInitLog(const char *logName, int32_t maxFiles) { if (atomic_val_compare_exchange_8(&tsLogInited, 0, 1) != 0) return 0; osUpdate(); @@ -151,6 +181,8 @@ int32_t taosInitLog(const char *logName, int32_t maxFiles) { tsLogObj.logHandle = taosLogBuffNew(LOG_DEFAULT_BUF_SIZE); if (tsLogObj.logHandle == NULL) return -1; if (taosOpenLogFile(fullName, tsNumOfLogLines, maxFiles) < 0) return -1; + + if (taosInitSlowLog() < 0) return -1; if (taosStartLog() < 0) return -1; return 0; } @@ -159,11 +191,23 @@ static void taosStopLog() { if (tsLogObj.logHandle) { tsLogObj.logHandle->stop = 1; } + if (tsLogObj.slowHandle) { + tsLogObj.slowHandle->stop = 1; + } } void taosCloseLog() { + taosStopLog(); + + if (tsLogObj.slowHandle != NULL) { + taosThreadMutexDestroy(&tsLogObj.slowHandle->buffMutex); + taosCloseFile(&tsLogObj.slowHandle->pFile); + taosMemoryFreeClear(tsLogObj.slowHandle->buffer); + memset(&tsLogObj.slowHandle->buffer, 0, sizeof(tsLogObj.slowHandle->buffer)); + taosMemoryFreeClear(tsLogObj.slowHandle); + } + if (tsLogObj.logHandle != NULL) { - taosStopLog(); if (tsLogObj.logHandle != NULL && taosCheckPthreadValid(tsLogObj.logHandle->asyncThread)) { taosThreadJoin(tsLogObj.logHandle->asyncThread, NULL); taosThreadClear(&tsLogObj.logHandle->asyncThread); @@ -176,8 +220,6 @@ void taosCloseLog() { memset(&tsLogObj.logHandle->buffer, 0, sizeof(tsLogObj.logHandle->buffer)); taosThreadMutexDestroy(&tsLogObj.logMutex); taosMemoryFreeClear(tsLogObj.logHandle); - memset(&tsLogObj.logHandle, 0, sizeof(tsLogObj.logHandle)); - tsLogObj.logHandle = NULL; } } From 0256b44be8b7acf9e61daa656016544683cdf773 Mon Sep 17 00:00:00 2001 From: kailixu Date: Sun, 23 Apr 2023 17:45:03 +0800 Subject: [PATCH 19/61] chore: code optimization --- source/client/inc/clientInt.h | 5 +- source/client/src/clientEnv.c | 2 +- source/client/src/clientHb.c | 85 ++++++++++++++++++-------------- source/client/src/clientMain.c | 3 ++ tests/script/api/passwdTest.c | 88 ++++++++++++++++++++++++++-------- 5 files changed, 126 insertions(+), 57 deletions(-) diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index f131bb9aa1..7cd5354240 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -63,6 +63,7 @@ typedef struct { // statistics int32_t reportCnt; int32_t connKeyCnt; + int32_t nPassVerCb; int64_t reportBytes; // not implemented int64_t startTime; // ctl @@ -73,7 +74,7 @@ typedef struct { typedef int32_t (*FHbRspHandle)(SAppHbMgr* pAppHbMgr, SClientHbRsp* pRsp); -typedef int32_t (*FHbReqHandle)(SClientHbKey* connKey, void* param, SClientHbReq* req); +typedef int32_t (*FHbReqHandle)(SClientHbKey* connKey, void* param, SClientHbReq* req, int32_t cb); typedef struct { int8_t inited; @@ -360,7 +361,7 @@ void stopAllRequests(SHashObj* pRequests); // conn level int hbRegisterConn(SAppHbMgr* pAppHbMgr, int64_t tscRefId, int64_t clusterId, int8_t connType); -void hbDeregisterConn(SAppHbMgr* pAppHbMgr, SClientHbKey connKey); +void hbDeregisterConn(SAppHbMgr* pAppHbMgr, SClientHbKey connKey, void* cb); typedef struct SSqlCallbackWrapper { SParseContext* pParseCtx; diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 418103f2a6..99569fdb57 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -239,7 +239,7 @@ void destroyTscObj(void *pObj) { tscTrace("begin to destroy tscObj %" PRIx64 " p:%p", tscId, pTscObj); SClientHbKey connKey = {.tscRid = pTscObj->id, .connType = pTscObj->connType}; - hbDeregisterConn(pTscObj->pAppInfo->pAppHbMgr, connKey); + hbDeregisterConn(pTscObj->pAppInfo->pAppHbMgr, connKey, pTscObj->passInfo.fp); destroyAllRequests(pTscObj->pRequests); taosHashCleanup(pTscObj->pRequests); diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 7f6356284a..49fa2d6a7d 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -24,7 +24,7 @@ static SClientHbMgr clientHbMgr = {0}; static int32_t hbCreateThread(); static void hbStopThread(); -static int32_t hbMqHbReqHandle(SClientHbKey *connKey, void *param, SClientHbReq *req) { return 0; } +static int32_t hbMqHbReqHandle(SClientHbKey *connKey, void *param, SClientHbReq *req, int32_t cb) { return 0; } static int32_t hbMqHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) { return 0; } @@ -49,38 +49,48 @@ static int32_t hbProcessUserAuthInfoRsp(void *value, int32_t valueLen, struct SC return TSDB_CODE_SUCCESS; } -static int32_t hbProcessUserPassInfoRsp(void *value, int32_t valueLen, SClientHbKey *connKey) { - int32_t code = 0; - STscObj *pTscObj = (STscObj *)acquireTscObj(connKey->tscRid); - if (NULL == pTscObj) { - tscWarn("tscObj rid %" PRIx64 " not exist", connKey->tscRid); - return code; - } - +static int32_t hbProcessUserPassInfoRsp(void *value, int32_t valueLen, SClientHbKey *connKey, SAppHbMgr *pAppHbMgr) { + int32_t code = 0; + int32_t numOfBatchs = 0; SUserPassBatchRsp batchRsp = {0}; if (tDeserializeSUserPassBatchRsp(value, valueLen, &batchRsp) != 0) { code = TSDB_CODE_INVALID_MSG; - releaseTscObj(connKey->tscRid); return code; } - SPassInfo *passInfo = &pTscObj->passInfo; - int32_t numOfBatchs = taosArrayGetSize(batchRsp.pArray); - for (int32_t i = 0; i < numOfBatchs; ++i) { - SGetUserPassRsp *rsp = taosArrayGet(batchRsp.pArray, i); - if (0 == strncmp(rsp->user, pTscObj->user, TSDB_USER_LEN)) { - tscDebug("update passVer of user %s from %d to %d", rsp->user, passInfo->ver, rsp->version); - if (atomic_load_32(&passInfo->ver) < rsp->version) { - atomic_store_32(&passInfo->ver, rsp->version); - if (passInfo->fp) { - (*passInfo->fp)(passInfo->param, &passInfo->ver, TAOS_NOTIFY_PASSVER); + numOfBatchs = taosArrayGetSize(batchRsp.pArray); + + SClientHbReq *pReq = NULL; + while ((pReq = taosHashIterate(pAppHbMgr->activeInfo, pReq))) { + STscObj *pTscObj = (STscObj *)acquireTscObj(pReq->connKey.tscRid); + if (!pTscObj) { + continue; + } + SPassInfo *passInfo = &pTscObj->passInfo; + if (!passInfo->fp) { + continue; + } + + for (int32_t i = 0; i < numOfBatchs; ++i) { + SGetUserPassRsp *rsp = taosArrayGet(batchRsp.pArray, i); + if (0 == strncmp(rsp->user, pTscObj->user, TSDB_USER_LEN)) { + int32_t oldVer = atomic_load_32(&passInfo->ver); + if (oldVer < rsp->version) { + atomic_store_32(&passInfo->ver, rsp->version); + tscDebug("update passVer of user %s from %d to %d, tscRid:%" PRIi64 "\n", rsp->user, oldVer, + atomic_load_32(&passInfo->ver), pTscObj->id); + if (passInfo->fp) { + (*passInfo->fp)(passInfo->param, &passInfo->ver, TAOS_NOTIFY_PASSVER); + } } + break; } } + releaseTscObj(pReq->connKey.tscRid); } taosArrayDestroy(batchRsp.pArray); - releaseTscObj(connKey->tscRid); + return code; } @@ -91,7 +101,7 @@ static int32_t hbGenerateVgInfoFromRsp(SDBVgInfo **pInfo, SUseDbRsp *rsp) { code = TSDB_CODE_OUT_OF_MEMORY; return code; } - + vgInfo->vgVersion = rsp->vgVersion; vgInfo->stateTs = rsp->stateTs; vgInfo->hashMethod = rsp->hashMethod; @@ -104,7 +114,7 @@ static int32_t hbGenerateVgInfoFromRsp(SDBVgInfo **pInfo, SUseDbRsp *rsp) { code = TSDB_CODE_OUT_OF_MEMORY; goto _return; } - + for (int32_t j = 0; j < rsp->vgNum; ++j) { SVgroupInfo *pInfo = taosArrayGet(rsp->pVgroupInfos, j); if (taosHashPut(vgInfo->vgHash, &pInfo->vgId, sizeof(int32_t), pInfo, sizeof(SVgroupInfo)) != 0) { @@ -332,7 +342,7 @@ static int32_t hbQueryHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) { break; } - hbProcessUserPassInfoRsp(kv->value, kv->valueLen, &pRsp->connKey); + hbProcessUserPassInfoRsp(kv->value, kv->valueLen, &pRsp->connKey, pAppHbMgr); break; } default: @@ -523,12 +533,7 @@ static int32_t hbGetUserBasicInfo(SClientHbKey *connKey, SClientHbReq *req) { return TSDB_CODE_APP_ERROR; } - int32_t code = 0; - - if (!pTscObj->passInfo.fp) { - goto _return; - } - + int32_t code = 0; SUserPassVersion *user = taosMemoryMalloc(sizeof(SUserPassVersion)); if (!user) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -543,7 +548,8 @@ static int32_t hbGetUserBasicInfo(SClientHbKey *connKey, SClientHbReq *req) { .value = user, }; - tscDebug("hb got user basic info, valueLen:%d", kv.valueLen); + tscDebug("hb got user basic info, valueLen:%d, user:%s, passVer:%d, tscRid:%" PRIi64, kv.valueLen, user->user, + pTscObj->passInfo.ver, connKey->tscRid); if (!req->info) { req->info = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK); @@ -697,7 +703,7 @@ int32_t hbGetAppInfo(int64_t clusterId, SClientHbReq *req) { return TSDB_CODE_SUCCESS; } -int32_t hbQueryHbReqHandle(SClientHbKey *connKey, void *param, SClientHbReq *req) { +int32_t hbQueryHbReqHandle(SClientHbKey *connKey, void *param, SClientHbReq *req, int32_t cb) { int64_t *clusterId = (int64_t *)param; struct SCatalog *pCatalog = NULL; @@ -711,7 +717,7 @@ int32_t hbQueryHbReqHandle(SClientHbKey *connKey, void *param, SClientHbReq *req hbGetQueryBasicInfo(connKey, req); - hbGetUserBasicInfo(connKey, req); + if (cb) hbGetUserBasicInfo(connKey, req); code = hbGetExpiredUserInfo(connKey, pCatalog, req); if (TSDB_CODE_SUCCESS != code) { @@ -764,9 +770,14 @@ SClientHbBatchReq *hbGatherAllInfo(SAppHbMgr *pAppHbMgr) { return NULL; } + int32_t nPassVerCb = atomic_load_32(&pAppHbMgr->nPassVerCb); while (pIter != NULL) { pOneReq = taosArrayPush(pBatchReq->reqs, pOneReq); - code = (*clientHbMgr.reqHandle[pOneReq->connKey.connType])(&pOneReq->connKey, &pOneReq->clusterId, pOneReq); + code = (*clientHbMgr.reqHandle[pOneReq->connKey.connType])(&pOneReq->connKey, &pOneReq->clusterId, pOneReq, + nPassVerCb); + break; + +#if 0 if (code) { pIter = taosHashIterate(pAppHbMgr->activeInfo, pIter); pOneReq = pIter; @@ -775,6 +786,7 @@ SClientHbBatchReq *hbGatherAllInfo(SAppHbMgr *pAppHbMgr) { pIter = taosHashIterate(pAppHbMgr->activeInfo, pIter); pOneReq = pIter; +#endif } releaseTscObj(rid); @@ -1116,7 +1128,7 @@ int hbRegisterConn(SAppHbMgr *pAppHbMgr, int64_t tscRefId, int64_t clusterId, in } } -void hbDeregisterConn(SAppHbMgr *pAppHbMgr, SClientHbKey connKey) { +void hbDeregisterConn(SAppHbMgr *pAppHbMgr, SClientHbKey connKey, void *cb) { SClientHbReq *pReq = taosHashAcquire(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey)); if (pReq) { tFreeClientHbReq(pReq); @@ -1129,4 +1141,7 @@ void hbDeregisterConn(SAppHbMgr *pAppHbMgr, SClientHbKey connKey) { } atomic_sub_fetch_32(&pAppHbMgr->connKeyCnt, 1); + if (cb) { + atomic_sub_fetch_32(&pAppHbMgr->nPassVerCb, 1); + } } diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index b3f7c844e9..3f29249e52 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -136,6 +136,9 @@ int taos_set_notify_cb(TAOS *taos, __taos_notify_fn_t fp, void *param, int type) case TAOS_NOTIFY_PASSVER: { pObj->passInfo.fp = fp; pObj->passInfo.param = param; + if (fp) { + atomic_add_fetch_32(&pObj->pAppInfo->pAppHbMgr->nPassVerCb, 1); + } break; } default: { diff --git a/tests/script/api/passwdTest.c b/tests/script/api/passwdTest.c index f3161047e8..c788851044 100644 --- a/tests/script/api/passwdTest.c +++ b/tests/script/api/passwdTest.c @@ -16,6 +16,11 @@ // TAOS standard API example. The same syntax as MySQL, but only a subset // to compile: gcc -o demo demo.c -ltaos +/** + * passwdTest.c + * - Run the test case in clear TDengine environment with default root passwd 'taosdata' + */ + #include #include #include @@ -23,18 +28,23 @@ #include #include "taos.h" // TAOS header file -#define nRepetition 1 -#define nTaos 10 +#define nDup 1 +#define nRoot 10 +#define nUser 10 +#define USER_LEN 30 void Test(TAOS *taos, char *qstr); +void createUers(TAOS *taos, const char *host, char *qstr); void passVerTestMulti(const char *host, char *qstr); -int nPassVerNotifiedMulti = 0; +int nPassVerNotified = 0; +TAOS *taosu[nRoot] = {0}; +char users[nUser][USER_LEN] = {0}; void __taos_notify_cb(void *param, void *ext, int type) { switch (type) { case TAOS_NOTIFY_PASSVER: { - ++nPassVerNotifiedMulti; + ++nPassVerNotified; printf("%s:%d type:%d user:%s ver:%d\n", __func__, __LINE__, type, param ? (char *)param : "NULL", *(int *)ext); break; } @@ -49,7 +59,7 @@ static void queryDB(TAOS *taos, char *command) { TAOS_RES *pSql = NULL; int32_t code = -1; - for (i = 0; i < nRepetition; ++i) { + for (i = 0; i < nDup; ++i) { if (NULL != pSql) { taos_free_result(pSql); pSql = NULL; @@ -88,19 +98,46 @@ int main(int argc, char *argv[]) { printf("failed to connect to server, reason:%s\n", "null taos" /*taos_errstr(taos)*/); exit(1); } - + createUers(taos, argv[1], qstr); passVerTestMulti(argv[1], qstr); taos_close(taos); taos_cleanup(); } -void passVerTestMulti(const char *host, char *qstr) { - TAOS *taos[nTaos] = {0}; - char *userName = calloc(1, 24); - strcpy(userName, "root"); +void createUers(TAOS *taos, const char *host, char *qstr) { + // users + for (int i = 0; i < nUser; ++i) { + sprintf(users[i], "user%d", i); + sprintf(qstr, "CREATE USER %s PASS 'taosdata'", users[i]); + queryDB(taos, qstr); - for (int i = 0; i < nTaos; ++i) { + taosu[i] = taos_connect(host, users[i], "taosdata", NULL, 0); + if (taosu[i] == NULL) { + printf("failed to connect to server, user:%s, reason:%s\n", users[i], "null taos" /*taos_errstr(taos)*/); + exit(1); + } + + int code = taos_set_notify_cb(taosu[i], __taos_notify_cb, users[i], TAOS_NOTIFY_PASSVER); + + if (code != 0) { + fprintf(stderr, "failed to run: taos_set_notify_cb for user:%s since %d\n", users[i], code); + } else { + fprintf(stderr, "success to run: taos_set_notify_cb for user:%s\n", users[i]); + } + + // alter pass for users + sprintf(qstr, "alter user %s pass 'taos'", users[i]); + queryDB(taos, qstr); + } +} + +void passVerTestMulti(const char *host, char *qstr) { + // root + TAOS *taos[nRoot] = {0}; + char userName[USER_LEN] = "root"; + + for (int i = 0; i < nRoot; ++i) { taos[i] = taos_connect(host, "root", "taosdata", NULL, 0); if (taos[i] == NULL) { printf("failed to connect to server, reason:%s\n", "null taos" /*taos_errstr(taos)*/); @@ -127,18 +164,31 @@ void passVerTestMulti(const char *host, char *qstr) { strcpy(qstr, "alter user root pass 'taos'"); queryDB(taos[0], qstr); - for (int i = 0; i < 10; ++i) { - if (nPassVerNotifiedMulti >= nTaos) break; + // calculate the nPassVerNotified for root and users + int nConn = nRoot + nUser; + + for (int i = 0; i < 15; ++i) { + if (nPassVerNotified >= nConn) break; sleep(1); } - if (nPassVerNotifiedMulti >= nTaos) { - fprintf(stderr, "success to get passVer notification\n"); - } else { - fprintf(stderr, "failed to get passVer notification\n"); + // close the taos_conn + for (int i = 0; i < nRoot; ++i) { + taos_close(taos[i]); + printf("%s:%d close taos[%d]\n", __func__, __LINE__, i); + sleep(1); } - // sleep(1000); + for (int i = 0; i < nUser; ++i) { + taos_close(taosu[i]); + printf("%s:%d close taosu[%d]\n", __func__, __LINE__, i); + sleep(1); + } - free(userName); + if (nPassVerNotified >= nConn) { + fprintf(stderr, "succeed to get passVer notification since nNotify %d >= nConn %d\n", nPassVerNotified, nConn); + } else { + fprintf(stderr, "failed to get passVer notification since nNotify %d < nConn %d\n", nPassVerNotified, nConn); + } + // sleep(300); } \ No newline at end of file From ee1a4b665a16f4e47542135094edb5afb683b784 Mon Sep 17 00:00:00 2001 From: kailixu Date: Sun, 23 Apr 2023 17:48:03 +0800 Subject: [PATCH 20/61] chore: more code --- tests/script/api/passwdTest.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/script/api/passwdTest.c b/tests/script/api/passwdTest.c index c788851044..8a2b0a0390 100644 --- a/tests/script/api/passwdTest.c +++ b/tests/script/api/passwdTest.c @@ -31,7 +31,7 @@ #define nDup 1 #define nRoot 10 #define nUser 10 -#define USER_LEN 30 +#define USER_LEN 24 void Test(TAOS *taos, char *qstr); void createUers(TAOS *taos, const char *host, char *qstr); From 5d4bc82e17ec75f48649caf2b7f51db8cf7ae098 Mon Sep 17 00:00:00 2001 From: kailixu Date: Sun, 23 Apr 2023 19:53:58 +0800 Subject: [PATCH 21/61] chore: more code --- source/client/inc/clientInt.h | 2 +- source/client/src/clientHb.c | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 7cd5354240..1df032c498 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -361,7 +361,7 @@ void stopAllRequests(SHashObj* pRequests); // conn level int hbRegisterConn(SAppHbMgr* pAppHbMgr, int64_t tscRefId, int64_t clusterId, int8_t connType); -void hbDeregisterConn(SAppHbMgr* pAppHbMgr, SClientHbKey connKey, void* cb); +void hbDeregisterConn(SAppHbMgr* pAppHbMgr, SClientHbKey connKey, void* connFp); typedef struct SSqlCallbackWrapper { SParseContext* pParseCtx; diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 49fa2d6a7d..473d7c2e7d 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -101,7 +101,7 @@ static int32_t hbGenerateVgInfoFromRsp(SDBVgInfo **pInfo, SUseDbRsp *rsp) { code = TSDB_CODE_OUT_OF_MEMORY; return code; } - + vgInfo->vgVersion = rsp->vgVersion; vgInfo->stateTs = rsp->stateTs; vgInfo->hashMethod = rsp->hashMethod; @@ -114,7 +114,7 @@ static int32_t hbGenerateVgInfoFromRsp(SDBVgInfo **pInfo, SUseDbRsp *rsp) { code = TSDB_CODE_OUT_OF_MEMORY; goto _return; } - + for (int32_t j = 0; j < rsp->vgNum; ++j) { SVgroupInfo *pInfo = taosArrayGet(rsp->pVgroupInfos, j); if (taosHashPut(vgInfo->vgHash, &pInfo->vgId, sizeof(int32_t), pInfo, sizeof(SVgroupInfo)) != 0) { @@ -717,7 +717,7 @@ int32_t hbQueryHbReqHandle(SClientHbKey *connKey, void *param, SClientHbReq *req hbGetQueryBasicInfo(connKey, req); - if (cb) hbGetUserBasicInfo(connKey, req); + if (cb > 0) hbGetUserBasicInfo(connKey, req); code = hbGetExpiredUserInfo(connKey, pCatalog, req); if (TSDB_CODE_SUCCESS != code) { @@ -1128,7 +1128,7 @@ int hbRegisterConn(SAppHbMgr *pAppHbMgr, int64_t tscRefId, int64_t clusterId, in } } -void hbDeregisterConn(SAppHbMgr *pAppHbMgr, SClientHbKey connKey, void *cb) { +void hbDeregisterConn(SAppHbMgr *pAppHbMgr, SClientHbKey connKey, void *connFp) { SClientHbReq *pReq = taosHashAcquire(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey)); if (pReq) { tFreeClientHbReq(pReq); @@ -1141,7 +1141,7 @@ void hbDeregisterConn(SAppHbMgr *pAppHbMgr, SClientHbKey connKey, void *cb) { } atomic_sub_fetch_32(&pAppHbMgr->connKeyCnt, 1); - if (cb) { + if (connFp) { atomic_sub_fetch_32(&pAppHbMgr->nPassVerCb, 1); } } From 8513a30bfc234e39951802e0a15ac23856181cd2 Mon Sep 17 00:00:00 2001 From: kailixu Date: Sun, 23 Apr 2023 20:01:46 +0800 Subject: [PATCH 22/61] chore: more code --- source/client/inc/clientInt.h | 2 +- source/client/src/clientHb.c | 7 ++++--- source/client/src/clientMain.c | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 1df032c498..211b47b49c 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -63,7 +63,7 @@ typedef struct { // statistics int32_t reportCnt; int32_t connKeyCnt; - int32_t nPassVerCb; + int32_t passKeyCnt; // with passVer call back int64_t reportBytes; // not implemented int64_t startTime; // ctl diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 473d7c2e7d..277f74c77d 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -68,6 +68,7 @@ static int32_t hbProcessUserPassInfoRsp(void *value, int32_t valueLen, SClientHb } SPassInfo *passInfo = &pTscObj->passInfo; if (!passInfo->fp) { + releaseTscObj(pReq->connKey.tscRid); continue; } @@ -770,11 +771,11 @@ SClientHbBatchReq *hbGatherAllInfo(SAppHbMgr *pAppHbMgr) { return NULL; } - int32_t nPassVerCb = atomic_load_32(&pAppHbMgr->nPassVerCb); + int32_t passKeyCnt = atomic_load_32(&pAppHbMgr->passKeyCnt); while (pIter != NULL) { pOneReq = taosArrayPush(pBatchReq->reqs, pOneReq); code = (*clientHbMgr.reqHandle[pOneReq->connKey.connType])(&pOneReq->connKey, &pOneReq->clusterId, pOneReq, - nPassVerCb); + passKeyCnt); break; #if 0 @@ -1142,6 +1143,6 @@ void hbDeregisterConn(SAppHbMgr *pAppHbMgr, SClientHbKey connKey, void *connFp) atomic_sub_fetch_32(&pAppHbMgr->connKeyCnt, 1); if (connFp) { - atomic_sub_fetch_32(&pAppHbMgr->nPassVerCb, 1); + atomic_sub_fetch_32(&pAppHbMgr->passKeyCnt, 1); } } diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 3f29249e52..aed11b4fb1 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -137,7 +137,7 @@ int taos_set_notify_cb(TAOS *taos, __taos_notify_fn_t fp, void *param, int type) pObj->passInfo.fp = fp; pObj->passInfo.param = param; if (fp) { - atomic_add_fetch_32(&pObj->pAppInfo->pAppHbMgr->nPassVerCb, 1); + atomic_add_fetch_32(&pObj->pAppInfo->pAppHbMgr->passKeyCnt, 1); } break; } From dff62bdccd9ab9e59acceef362d1c6a37c594ef9 Mon Sep 17 00:00:00 2001 From: kailixu Date: Sun, 23 Apr 2023 22:43:29 +0800 Subject: [PATCH 23/61] chore: code optimization --- source/client/inc/clientInt.h | 2 +- source/client/src/clientHb.c | 37 +++++++++++++++++++++++++---------- 2 files changed, 28 insertions(+), 11 deletions(-) diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 211b47b49c..c319a6ffa5 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -74,7 +74,7 @@ typedef struct { typedef int32_t (*FHbRspHandle)(SAppHbMgr* pAppHbMgr, SClientHbRsp* pRsp); -typedef int32_t (*FHbReqHandle)(SClientHbKey* connKey, void* param, SClientHbReq* req, int32_t cb); +typedef int32_t (*FHbReqHandle)(SClientHbKey* connKey, void* param, SClientHbReq* req); typedef struct { int8_t inited; diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 277f74c77d..08607b08ea 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -24,7 +24,14 @@ static SClientHbMgr clientHbMgr = {0}; static int32_t hbCreateThread(); static void hbStopThread(); -static int32_t hbMqHbReqHandle(SClientHbKey *connKey, void *param, SClientHbReq *req, int32_t cb) { return 0; } +typedef struct { + union { + int64_t clusterId; + int32_t passKeyCnt; + }; +} SHbParam; + +static int32_t hbMqHbReqHandle(SClientHbKey *connKey, void *param, SClientHbReq *req) { return 0; } static int32_t hbMqHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) { return 0; } @@ -704,21 +711,23 @@ int32_t hbGetAppInfo(int64_t clusterId, SClientHbReq *req) { return TSDB_CODE_SUCCESS; } -int32_t hbQueryHbReqHandle(SClientHbKey *connKey, void *param, SClientHbReq *req, int32_t cb) { - int64_t *clusterId = (int64_t *)param; +int32_t hbQueryHbReqHandle(SClientHbKey *connKey, void *param, SClientHbReq *req) { + SHbParam *hbParam = (SHbParam *)param; struct SCatalog *pCatalog = NULL; - int32_t code = catalogGetHandle(*clusterId, &pCatalog); + int32_t code = catalogGetHandle(hbParam->clusterId, &pCatalog); if (code != TSDB_CODE_SUCCESS) { - tscWarn("catalogGetHandle failed, clusterId:%" PRIx64 ", error:%s", *clusterId, tstrerror(code)); + tscWarn("catalogGetHandle failed, clusterId:%" PRIx64 ", error:%s", hbParam->clusterId, tstrerror(code)); return code; } - hbGetAppInfo(*clusterId, req); + hbGetAppInfo(hbParam->clusterId, req); hbGetQueryBasicInfo(connKey, req); - if (cb > 0) hbGetUserBasicInfo(connKey, req); + if (hbParam->passKeyCnt > 0) { + hbGetUserBasicInfo(connKey, req); + } code = hbGetExpiredUserInfo(connKey, pCatalog, req); if (TSDB_CODE_SUCCESS != code) { @@ -771,11 +780,19 @@ SClientHbBatchReq *hbGatherAllInfo(SAppHbMgr *pAppHbMgr) { return NULL; } - int32_t passKeyCnt = atomic_load_32(&pAppHbMgr->passKeyCnt); while (pIter != NULL) { pOneReq = taosArrayPush(pBatchReq->reqs, pOneReq); - code = (*clientHbMgr.reqHandle[pOneReq->connKey.connType])(&pOneReq->connKey, &pOneReq->clusterId, pOneReq, - passKeyCnt); + SHbParam param; + switch (pOneReq->connKey.connType) { + case CONN_TYPE__QUERY: { + param.clusterId = pOneReq->clusterId; + param.passKeyCnt = atomic_load_32(&pAppHbMgr->passKeyCnt); + break; + } + default: + break; + } + code = (*clientHbMgr.reqHandle[pOneReq->connKey.connType])(&pOneReq->connKey, ¶m, pOneReq); break; #if 0 From 16f039a780b9f712fde8b4861e94f4b0f6ed7f27 Mon Sep 17 00:00:00 2001 From: kailixu Date: Sun, 23 Apr 2023 22:52:20 +0800 Subject: [PATCH 24/61] chore: code optimization --- source/client/inc/clientInt.h | 2 +- source/client/src/clientHb.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index c319a6ffa5..8e20d7d275 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -361,7 +361,7 @@ void stopAllRequests(SHashObj* pRequests); // conn level int hbRegisterConn(SAppHbMgr* pAppHbMgr, int64_t tscRefId, int64_t clusterId, int8_t connType); -void hbDeregisterConn(SAppHbMgr* pAppHbMgr, SClientHbKey connKey, void* connFp); +void hbDeregisterConn(SAppHbMgr* pAppHbMgr, SClientHbKey connKey, void* param); typedef struct SSqlCallbackWrapper { SParseContext* pParseCtx; diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 08607b08ea..3b8739207c 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -1146,7 +1146,7 @@ int hbRegisterConn(SAppHbMgr *pAppHbMgr, int64_t tscRefId, int64_t clusterId, in } } -void hbDeregisterConn(SAppHbMgr *pAppHbMgr, SClientHbKey connKey, void *connFp) { +void hbDeregisterConn(SAppHbMgr *pAppHbMgr, SClientHbKey connKey, void *param) { SClientHbReq *pReq = taosHashAcquire(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey)); if (pReq) { tFreeClientHbReq(pReq); @@ -1159,7 +1159,7 @@ void hbDeregisterConn(SAppHbMgr *pAppHbMgr, SClientHbKey connKey, void *connFp) } atomic_sub_fetch_32(&pAppHbMgr->connKeyCnt, 1); - if (connFp) { + if (param) { atomic_sub_fetch_32(&pAppHbMgr->passKeyCnt, 1); } } From 4701fdfe0606ebf1a3ba9dc8ecd785a7f2fe0521 Mon Sep 17 00:00:00 2001 From: kailixu Date: Sun, 23 Apr 2023 23:11:15 +0800 Subject: [PATCH 25/61] chore: more code --- source/client/src/clientHb.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 3b8739207c..555aefc21b 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -19,11 +19,6 @@ #include "scheduler.h" #include "trpc.h" -static SClientHbMgr clientHbMgr = {0}; - -static int32_t hbCreateThread(); -static void hbStopThread(); - typedef struct { union { int64_t clusterId; @@ -31,6 +26,11 @@ typedef struct { }; } SHbParam; +static SClientHbMgr clientHbMgr = {0}; + +static int32_t hbCreateThread(); +static void hbStopThread(); + static int32_t hbMqHbReqHandle(SClientHbKey *connKey, void *param, SClientHbReq *req) { return 0; } static int32_t hbMqHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) { return 0; } From 63bad2e4fbb770034dfe03d18306ede786127bef Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 24 Apr 2023 09:57:48 +0800 Subject: [PATCH 26/61] chore: code optimization --- source/client/src/clientHb.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 555aefc21b..3b07fcf7ab 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -792,7 +792,13 @@ SClientHbBatchReq *hbGatherAllInfo(SAppHbMgr *pAppHbMgr) { default: break; } - code = (*clientHbMgr.reqHandle[pOneReq->connKey.connType])(&pOneReq->connKey, ¶m, pOneReq); + if (clientHbMgr.reqHandle[pOneReq->connKey.connType]) { + code = (*clientHbMgr.reqHandle[pOneReq->connKey.connType])(&pOneReq->connKey, ¶m, pOneReq); + if (code) { + tscWarn("hbGatherAllInfo failed since %s, tscRid:%" PRIi64 ", connType:%" PRIi8, tstrerror(code), + pOneReq->connKey.tscRid, pOneReq->connKey.connType); + } + } break; #if 0 From a781d435771a56206016a222eeb3f57389a1d1c8 Mon Sep 17 00:00:00 2001 From: cadem Date: Mon, 24 Apr 2023 10:23:43 +0800 Subject: [PATCH 27/61] fix/block-check-to-async-check --- include/libs/sync/sync.h | 1 + include/util/taoserror.h | 2 + source/dnode/mgmt/mgmt_vnode/src/vmHandle.c | 14 ++++++- source/dnode/mnode/impl/src/mndVgroup.c | 2 + source/dnode/vnode/inc/vnode.h | 1 + source/dnode/vnode/src/vnd/vnodeOpen.c | 4 ++ source/libs/sync/src/syncMain.c | 44 +++++++++++++-------- 7 files changed, 50 insertions(+), 18 deletions(-) diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index 33cef538d2..e86a4f9690 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -250,6 +250,7 @@ void syncPreStop(int64_t rid); void syncPostStop(int64_t rid); int32_t syncPropose(int64_t rid, SRpcMsg* pMsg, bool isWeak, int64_t* seq); int32_t syncIsCatchUp(int64_t rid); +ESyncRole syncGetRole(int64_t rid); int32_t syncProcessMsg(int64_t rid, SRpcMsg* pMsg); int32_t syncReconfig(int64_t rid, SSyncCfg* pCfg); int32_t syncBeginSnapshot(int64_t rid, int64_t lastApplyIndex); diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 0d9292cc6b..297ae9de07 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -437,6 +437,8 @@ int32_t* taosGetErrno(); #define TSDB_CODE_VND_STOPPED TAOS_DEF_ERROR_CODE(0, 0x0529) #define TSDB_CODE_VND_DUP_REQUEST TAOS_DEF_ERROR_CODE(0, 0x0530) #define TSDB_CODE_VND_QUERY_BUSY TAOS_DEF_ERROR_CODE(0, 0x0531) +#define TSDB_CODE_VND_NOT_CATCH_UP TAOS_DEF_ERROR_CODE(0, 0x0532) // internal +#define TSDB_CODE_VND_ALREADY_IS_VOTER TAOS_DEF_ERROR_CODE(0, 0x0533) // internal // tsdb #define TSDB_CODE_TDB_INVALID_TABLE_ID TAOS_DEF_ERROR_CODE(0, 0x0600) diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index d7197832e5..0ddc1fcf00 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -336,13 +336,21 @@ int32_t vmProcessAlterVnodeTypeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { SVnodeObj *pVnode = vmAcquireVnode(pMgmt, req.vgId); if (pVnode == NULL) { - dError("vgId:%d, failed to alter hashrange since %s", req.vgId, terrstr()); + dError("vgId:%d, failed to alter vnode type since %s", req.vgId, terrstr()); terrno = TSDB_CODE_VND_NOT_EXIST; return -1; } + if(vnodeGetRole(pVnode->pImpl) == TAOS_SYNC_ROLE_VOTER){ + terrno = TSDB_CODE_VND_ALREADY_IS_VOTER; + vmReleaseVnode(pMgmt, pVnode); + return -1; + } + dInfo("vgId:%d, checking node catch up", req.vgId); - if(vnodeIsCatchUp(pVnode->pImpl) != 0){ + if(vnodeIsCatchUp(pVnode->pImpl) != 1){ + terrno = TSDB_CODE_VND_NOT_CATCH_UP; + vmReleaseVnode(pMgmt, pVnode); return -1; } @@ -365,6 +373,7 @@ int32_t vmProcessAlterVnodeTypeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { req.selfIndex >= req.replica || req.learnerSelfIndex >= req.learnerReplica) { terrno = TSDB_CODE_INVALID_MSG; dError("vgId:%d, failed to alter replica since invalid msg", vgId); + vmReleaseVnode(pMgmt, pVnode); return -1; } @@ -381,6 +390,7 @@ int32_t vmProcessAlterVnodeTypeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { terrno = TSDB_CODE_INVALID_MSG; dError("vgId:%d, dnodeId:%d ep:%s:%u not matched with local dnode", vgId, pReplica->id, pReplica->fqdn, pReplica->port); + vmReleaseVnode(pMgmt, pVnode); return -1; } diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index d674db5b4b..f0bece6e5e 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -1263,6 +1263,8 @@ int32_t mndAddAlterVnodeTypeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, action.pCont = pReq; action.contLen = contLen; action.msgType = TDMT_DND_ALTER_VNODE_TYPE; + action.acceptableCode = TSDB_CODE_VND_ALREADY_IS_VOTER; + action.retryCode = TSDB_CODE_VND_NOT_CATCH_UP; if (mndTransAppendRedoAction(pTrans, &action) != 0) { taosMemoryFree(pReq); diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 7dfaf1508d..828a173108 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -69,6 +69,7 @@ void vnodeGetInfo(SVnode *pVnode, const char **dbname, int32_t *vgId); int32_t vnodeProcessCreateTSma(SVnode *pVnode, void *pCont, uint32_t contLen); int32_t vnodeGetAllTableList(SVnode *pVnode, uint64_t uid, SArray *list); int32_t vnodeIsCatchUp(SVnode *pVnode); +ESyncRole vnodeGetRole(SVnode *pVnode); int32_t vnodeGetCtbIdList(SVnode *pVnode, int64_t suid, SArray *list); int32_t vnodeGetCtbIdListByFilter(SVnode *pVnode, int64_t suid, SArray *list, bool (*filter)(void *arg), void *arg); diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index deeb0af42a..7d41edfdd9 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -431,6 +431,10 @@ int32_t vnodeIsCatchUp(SVnode *pVnode){ return syncIsCatchUp(pVnode->sync); } +ESyncRole vnodeGetRole(SVnode *pVnode){ + return syncGetRole(pVnode->sync); +} + void vnodeStop(SVnode *pVnode) {} int64_t vnodeGetSyncHandle(SVnode *pVnode) { return pVnode->sync; } diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 8a0a6be8fd..fd89d48363 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -580,25 +580,37 @@ int32_t syncIsCatchUp(int64_t rid) { return -1; } - while(1){ - if(pSyncNode->pLogBuf->totalIndex < 0 || pSyncNode->pLogBuf->commitIndex < 0 || - pSyncNode->pLogBuf->totalIndex < pSyncNode->pLogBuf->commitIndex || - pSyncNode->pLogBuf->totalIndex - pSyncNode->pLogBuf->commitIndex > SYNC_LEARNER_CATCHUP){ - sInfo("vgId:%d, Not catch up, wait one second, totalIndex:%" PRId64 " commitIndex:%" PRId64 " matchIndex:%" PRId64, - pSyncNode->vgId, pSyncNode->pLogBuf->totalIndex, pSyncNode->pLogBuf->commitIndex, - pSyncNode->pLogBuf->matchIndex); - taosSsleep(1); - } - else{ - sInfo("vgId:%d, Catch up, totalIndex:%" PRId64 " commitIndex:%" PRId64 " matchIndex:%" PRId64, - pSyncNode->vgId, pSyncNode->pLogBuf->totalIndex, pSyncNode->pLogBuf->commitIndex, - pSyncNode->pLogBuf->matchIndex); - break; - } + int32_t isCatchUp = 0; + if(pSyncNode->pLogBuf->totalIndex < 0 || pSyncNode->pLogBuf->commitIndex < 0 || + pSyncNode->pLogBuf->totalIndex < pSyncNode->pLogBuf->commitIndex || + pSyncNode->pLogBuf->totalIndex - pSyncNode->pLogBuf->commitIndex > SYNC_LEARNER_CATCHUP){ + sInfo("vgId:%d, Not catch up, wait one second, totalIndex:%" PRId64 " commitIndex:%" PRId64 " matchIndex:%" PRId64, + pSyncNode->vgId, pSyncNode->pLogBuf->totalIndex, pSyncNode->pLogBuf->commitIndex, + pSyncNode->pLogBuf->matchIndex); + isCatchUp = 0; + } + else{ + sInfo("vgId:%d, Catch up, totalIndex:%" PRId64 " commitIndex:%" PRId64 " matchIndex:%" PRId64, + pSyncNode->vgId, pSyncNode->pLogBuf->totalIndex, pSyncNode->pLogBuf->commitIndex, + pSyncNode->pLogBuf->matchIndex); + isCatchUp = 1; } syncNodeRelease(pSyncNode); - return 0; + return isCatchUp; +} + +ESyncRole syncGetRole(int64_t rid) { + SSyncNode* pSyncNode = syncNodeAcquire(rid); + if (pSyncNode == NULL) { + sError("sync Node Acquire error since %d", errno); + return -1; + } + + ESyncRole role = pSyncNode->raftCfg.cfg.nodeInfo[pSyncNode->raftCfg.cfg.myIndex].nodeRole; + + syncNodeRelease(pSyncNode); + return role; } int32_t syncNodePropose(SSyncNode* pSyncNode, SRpcMsg* pMsg, bool isWeak, int64_t* seq) { From 470fef645210a6efbaa7837318a49b7a927e01b6 Mon Sep 17 00:00:00 2001 From: cadem Date: Mon, 24 Apr 2023 13:38:55 +0800 Subject: [PATCH 28/61] fix/block check to async check for mnode --- include/dnode/mnode/mnode.h | 2 ++ include/util/taoserror.h | 2 ++ source/dnode/mgmt/mgmt_mnode/src/mmInt.c | 5 +++++ source/dnode/mgmt/mgmt_vnode/src/vmHandle.c | 4 +++- source/dnode/mgmt/node_mgmt/src/dmEnv.c | 12 +++++++++++- source/dnode/mgmt/node_util/inc/dmUtil.h | 2 ++ source/dnode/mnode/impl/src/mndMain.c | 5 +++++ source/dnode/mnode/impl/src/mndMnode.c | 3 ++- 8 files changed, 32 insertions(+), 3 deletions(-) diff --git a/include/dnode/mnode/mnode.h b/include/dnode/mnode/mnode.h index 07a0ca952a..6c3c7497b1 100644 --- a/include/dnode/mnode/mnode.h +++ b/include/dnode/mnode/mnode.h @@ -20,6 +20,7 @@ #include "tmsg.h" #include "tmsgcb.h" #include "trpc.h" +#include "sync.h" #ifdef __cplusplus extern "C" { @@ -73,6 +74,7 @@ int32_t mndStart(SMnode *pMnode); void mndStop(SMnode *pMnode); int32_t mndIsCatchUp(SMnode *pMnode); +ESyncRole mndGetRole(SMnode *pMnode); /** * @brief Get mnode monitor info. diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 297ae9de07..b7325edf9c 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -403,6 +403,8 @@ int32_t* taosGetErrno(); #define TSDB_CODE_SNODE_ALREADY_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x040F) #define TSDB_CODE_SNODE_NOT_FOUND TAOS_DEF_ERROR_CODE(0, 0x0410) #define TSDB_CODE_SNODE_NOT_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x0411) +#define TSDB_CODE_MNODE_NOT_CATCH_UP TAOS_DEF_ERROR_CODE(0, 0x0412) // internal +#define TSDB_CODE_MNODE_ALREADY_IS_VOTER TAOS_DEF_ERROR_CODE(0, 0x0413) // internal // vnode // #define TSDB_CODE_VND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0500) // 2.x diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmInt.c b/source/dnode/mgmt/mgmt_mnode/src/mmInt.c index 05b59b9865..7840528db9 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmInt.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmInt.c @@ -159,6 +159,10 @@ static int32_t mmSyncIsCatchUp(SMnodeMgmt *pMgmt) { return mndIsCatchUp(pMgmt->pMnode); } +static ESyncRole mmSyncGetRole(SMnodeMgmt *pMgmt) { + return mndGetRole(pMgmt->pMnode); +} + SMgmtFunc mmGetMgmtFunc() { SMgmtFunc mgmtFunc = {0}; mgmtFunc.openFp = mmOpen; @@ -170,6 +174,7 @@ SMgmtFunc mmGetMgmtFunc() { mgmtFunc.requiredFp = mmRequire; mgmtFunc.getHandlesFp = mmGetMsgHandles; mgmtFunc.isCatchUpFp = (NodeIsCatchUpFp)mmSyncIsCatchUp; + mgmtFunc.nodeRoleFp = (NodeRole)mmSyncGetRole; return mgmtFunc; } diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index 0ddc1fcf00..4121e14e6d 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -341,7 +341,9 @@ int32_t vmProcessAlterVnodeTypeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { return -1; } - if(vnodeGetRole(pVnode->pImpl) == TAOS_SYNC_ROLE_VOTER){ + ESyncRole role = vnodeGetRole(pVnode->pImpl); + dInfo("vgId:%d, checking node role:%d", req.vgId, role); + if(role == TAOS_SYNC_ROLE_VOTER){ terrno = TSDB_CODE_VND_ALREADY_IS_VOTER; vmReleaseVnode(pMgmt, pVnode); return -1; diff --git a/source/dnode/mgmt/node_mgmt/src/dmEnv.c b/source/dnode/mgmt/node_mgmt/src/dmEnv.c index c53c21cd30..3459af1a3a 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmEnv.c +++ b/source/dnode/mgmt/node_mgmt/src/dmEnv.c @@ -214,9 +214,19 @@ static int32_t dmProcessAlterNodeTypeReq(EDndNodeType ntype, SRpcMsg *pMsg) { pWrapper = &pDnode->wrappers[ntype]; + if(pWrapper->func.nodeRoleFp != NULL){ + ESyncRole role = (*pWrapper->func.nodeRoleFp)(pWrapper->pMgmt); + dInfo("node:%s, checking node role:%d", pWrapper->name, role); + if(role == TAOS_SYNC_ROLE_VOTER){ + terrno = TSDB_CODE_MNODE_ALREADY_IS_VOTER; + return -1; + } + } + if(pWrapper->func.isCatchUpFp != NULL){ dInfo("node:%s, checking node catch up", pWrapper->name); - if(!(*pWrapper->func.isCatchUpFp)(pWrapper->pMgmt) == 0){ + if((*pWrapper->func.isCatchUpFp)(pWrapper->pMgmt) != 1){ + terrno = TSDB_CODE_MNODE_NOT_CATCH_UP; return -1; } } diff --git a/source/dnode/mgmt/node_util/inc/dmUtil.h b/source/dnode/mgmt/node_util/inc/dmUtil.h index 000ce81207..98ef8cd95b 100644 --- a/source/dnode/mgmt/node_util/inc/dmUtil.h +++ b/source/dnode/mgmt/node_util/inc/dmUtil.h @@ -135,6 +135,7 @@ typedef int32_t (*NodeDropFp)(const SMgmtInputOpt *pInput, SRpcMsg *pMsg); typedef int32_t (*NodeRequireFp)(const SMgmtInputOpt *pInput, bool *required); typedef SArray *(*NodeGetHandlesFp)(); // array of SMgmtHandle typedef bool (*NodeIsCatchUpFp)(void *pMgmt); +typedef bool (*NodeRole)(void *pMgmt); typedef struct { NodeOpenFp openFp; @@ -146,6 +147,7 @@ typedef struct { NodeRequireFp requiredFp; NodeGetHandlesFp getHandlesFp; NodeIsCatchUpFp isCatchUpFp; + NodeRole nodeRoleFp; } SMgmtFunc; typedef struct { diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index 13ae4a11d5..a9f52128a6 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -590,6 +590,11 @@ int32_t mndIsCatchUp(SMnode *pMnode) { return syncIsCatchUp(rid); } +ESyncRole mndGetRole(SMnode *pMnode){ + int64_t rid = pMnode->syncMgmt.sync; + return syncGetRole(rid); +} + void mndStop(SMnode *pMnode) { mndSetStop(pMnode); mndSyncStop(pMnode); diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index 53baf843de..5e3476859a 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -322,7 +322,8 @@ static int32_t mndBuildAlterMnodeTypeRedoAction(STrans *pTrans, .pCont = pReq, .contLen = contLen, .msgType = TDMT_DND_ALTER_MNODE_TYPE, - .acceptableCode = TSDB_CODE_MNODE_ALREADY_DEPLOYED, + .retryCode = TSDB_CODE_MNODE_NOT_CATCH_UP, + .acceptableCode = TSDB_CODE_MNODE_ALREADY_IS_VOTER, }; if (mndTransAppendRedoAction(pTrans, &action) != 0) { From d5d113aed72ec1ce5fa2a7ef3437c43e20955269 Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 24 Apr 2023 14:17:13 +0800 Subject: [PATCH 29/61] chore: fix the SHbParam definition --- source/client/src/clientHb.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 3b07fcf7ab..dea58f6a30 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -21,8 +21,10 @@ typedef struct { union { - int64_t clusterId; - int32_t passKeyCnt; + struct { + int64_t clusterId; + int32_t passKeyCnt; + }; }; } SHbParam; From 0a405c10b7f3881278642a0ffa1796b939ccb3ed Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 24 Apr 2023 14:59:52 +0800 Subject: [PATCH 30/61] chore: code optimization --- source/client/src/clientHb.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index dea58f6a30..79435da89f 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -87,11 +87,11 @@ static int32_t hbProcessUserPassInfoRsp(void *value, int32_t valueLen, SClientHb int32_t oldVer = atomic_load_32(&passInfo->ver); if (oldVer < rsp->version) { atomic_store_32(&passInfo->ver, rsp->version); - tscDebug("update passVer of user %s from %d to %d, tscRid:%" PRIi64 "\n", rsp->user, oldVer, - atomic_load_32(&passInfo->ver), pTscObj->id); if (passInfo->fp) { (*passInfo->fp)(passInfo->param, &passInfo->ver, TAOS_NOTIFY_PASSVER); } + tscDebug("update passVer of user %s from %d to %d, tscRid:%" PRIi64, rsp->user, oldVer, + atomic_load_32(&passInfo->ver), pTscObj->id); } break; } From 62dad38d5c492d85f7df5086aab9b707c8faa04b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 24 Apr 2023 15:25:49 +0800 Subject: [PATCH 31/61] refactor: do some internal refactor and add the sample code. --- examples/c/tmq.c | 33 +++++++++++++++++++++++++++++++-- include/client/taos.h | 4 ++-- source/client/src/clientTmq.c | 10 +++++----- 3 files changed, 38 insertions(+), 9 deletions(-) diff --git a/examples/c/tmq.c b/examples/c/tmq.c index 83b40c4f20..94545dfaad 100644 --- a/examples/c/tmq.c +++ b/examples/c/tmq.c @@ -20,7 +20,8 @@ #include #include "taos.h" -static int running = 1; +static int running = 1; +const char* topic_name = "topicname"; static int32_t msg_process(TAOS_RES* msg) { char buf[1024]; @@ -243,7 +244,7 @@ _end: tmq_list_t* build_topic_list() { tmq_list_t* topicList = tmq_list_new(); - int32_t code = tmq_list_append(topicList, "topicname"); + int32_t code = tmq_list_append(topicList, topic_name); if (code) { tmq_list_destroy(topicList); return NULL; @@ -269,6 +270,31 @@ void basic_consume_loop(tmq_t* tmq) { fprintf(stderr, "%d msg consumed, include %d rows\n", msgCnt, totalRows); } +void consume_repeatly(tmq_t* tmq) { + int32_t numOfAssignment = 0; + tmq_topic_assignment* pAssign = NULL; + + int32_t code = tmq_get_topic_assignment(tmq, topic_name, &pAssign, &numOfAssignment); + if (code != 0) { + fprintf(stderr, "failed to get assignment, reason:%s", tmq_err2str(code)); + } + + // seek to the earliest offset + for(int32_t i = 0; i < numOfAssignment; ++i) { + tmq_topic_assignment* p = &pAssign[i]; + + code = tmq_offset_seek(tmq, topic_name, p->vgId, p->begin); + if (code != 0) { + fprintf(stderr, "failed to seek to %ld, reason:%s", p->begin, tmq_err2str(code)); + } + } + + free(pAssign); + + // let's do it again + basic_consume_loop(tmq); +} + int main(int argc, char* argv[]) { int32_t code; @@ -294,10 +320,13 @@ int main(int argc, char* argv[]) { if ((code = tmq_subscribe(tmq, topic_list))) { fprintf(stderr, "Failed to tmq_subscribe(): %s\n", tmq_err2str(code)); } + tmq_list_destroy(topic_list); basic_consume_loop(tmq); + consume_repeatly(tmq); + code = tmq_consumer_close(tmq); if (code) { fprintf(stderr, "Failed to close consumer: %s\n", tmq_err2str(code)); diff --git a/include/client/taos.h b/include/client/taos.h index 05f3c5d11d..e46cd99c73 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -263,7 +263,7 @@ DLL_EXPORT const char *tmq_err2str(int32_t code); /* ------------------------TMQ CONSUMER INTERFACE------------------------ */ typedef struct tmq_topic_assignment { - int32_t vgroupHandle; + int32_t vgId; int64_t currentOffset; int64_t begin; int64_t end; @@ -277,7 +277,7 @@ DLL_EXPORT int32_t tmq_consumer_close(tmq_t *tmq); DLL_EXPORT int32_t tmq_commit_sync(tmq_t *tmq, const TAOS_RES *msg); DLL_EXPORT void tmq_commit_async(tmq_t *tmq, const TAOS_RES *msg, tmq_commit_cb *cb, void *param); DLL_EXPORT int32_t tmq_get_topic_assignment(tmq_t *tmq, const char* pTopicName, tmq_topic_assignment **assignment, int32_t *numOfAssignment); -DLL_EXPORT int32_t tmq_offset_seek(tmq_t *tmq, const char* pTopicName, int32_t vgroupHandle, int64_t offset); +DLL_EXPORT int32_t tmq_offset_seek(tmq_t *tmq, const char* pTopicName, int32_t vgId, int64_t offset); /* ----------------------TMQ CONFIGURATION INTERFACE---------------------- */ diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 87c58ab32f..76fd1d84d0 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -2357,7 +2357,7 @@ static int32_t tmqGetWalInfoCb(void* param, SDataBuf* pMsg, int32_t code) { tmq_topic_assignment assignment = {.begin = pHead->walsver, .end = pHead->walever, .currentOffset = rsp.rspOffset.version, - .vgroupHandle = pParam->vgId}; + .vgId = pParam->vgId}; taosThreadMutexLock(&pCommon->mutex); taosArrayPush(pCommon->pList, &assignment); @@ -2422,7 +2422,7 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a pAssignment->begin = pClientVg->offsetInfo.walVerBegin; pAssignment->end = pClientVg->offsetInfo.walVerEnd; - pAssignment->vgroupHandle = pClientVg->vgId; + pAssignment->vgId = pClientVg->vgId; } if (needFetch) { @@ -2524,7 +2524,7 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a } } -int32_t tmq_offset_seek(tmq_t* tmq, const char* pTopicName, int32_t vgroupHandle, int64_t offset) { +int32_t tmq_offset_seek(tmq_t* tmq, const char* pTopicName, int32_t vgId, int64_t offset) { if (tmq == NULL) { tscError("invalid tmq handle, null"); return TSDB_CODE_INVALID_PARA; @@ -2544,14 +2544,14 @@ int32_t tmq_offset_seek(tmq_t* tmq, const char* pTopicName, int32_t vgroupHandle int32_t numOfVgs = taosArrayGetSize(pTopic->vgs); for (int32_t i = 0; i < numOfVgs; ++i) { SMqClientVg* pClientVg = taosArrayGet(pTopic->vgs, i); - if (pClientVg->vgId == vgroupHandle) { + if (pClientVg->vgId == vgId) { pVg = pClientVg; break; } } if (pVg == NULL) { - tscError("consumer:0x%" PRIx64 " invalid vgroup id:%d", tmq->consumerId, vgroupHandle); + tscError("consumer:0x%" PRIx64 " invalid vgroup id:%d", tmq->consumerId, vgId); return TSDB_CODE_INVALID_PARA; } From 758427b3c41ba71ac559e2be9ba364dafa853d40 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Mon, 24 Apr 2023 15:48:20 +0800 Subject: [PATCH 32/61] fix: use slimit to indicate group for tag scan --- source/libs/executor/inc/executorimpl.h | 1 + source/libs/executor/src/scanoperator.c | 93 ++++++++++++++----------- source/libs/planner/src/planOptimizer.c | 29 ++++++++ 3 files changed, 82 insertions(+), 41 deletions(-) diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 85424fd7de..43ee54f65d 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -366,6 +366,7 @@ typedef struct STagScanInfo { int32_t curPos; SReadHandle readHandle; STableListInfo* pTableListInfo; + SLimitNode* pSlimit; } STagScanInfo; typedef enum EStreamScanMode { diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 2389c7252e..b7315db204 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -2512,6 +2512,53 @@ _error: return NULL; } +static void doTagScanOneTable(SOperatorInfo* pOperator, const SExecTaskInfo* pTaskInfo, STagScanInfo* pInfo, + const SExprInfo* pExprInfo, const SSDataBlock* pRes, int32_t size, const char* str, + int32_t* count, SMetaReader* mr) { + STableKeyInfo* item = tableListGetInfo(pInfo->pTableListInfo, pInfo->curPos); + int32_t code = metaGetTableEntryByUid(mr, item->uid); + tDecoderClear(&(*mr).coder); + if (code != TSDB_CODE_SUCCESS) { + qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", item->uid, tstrerror(terrno), + GET_TASKID(pTaskInfo)); + metaReaderClear(mr); + T_LONG_JMP(pTaskInfo->env, terrno); + } + + for (int32_t j = 0; j < pOperator->exprSupp.numOfExprs; ++j) { + SColumnInfoData* pDst = taosArrayGet(pRes->pDataBlock, pExprInfo[j].base.resSchema.slotId); + + // refactor later + if (fmIsScanPseudoColumnFunc(pExprInfo[j].pExpr->_function.functionId)) { + STR_TO_VARSTR(str, (*mr).me.name); + colDataSetVal(pDst, (*count), str, false); + } else { // it is a tag value + STagVal val = {0}; + val.cid = pExprInfo[j].base.pParam[0].pCol->colId; + const char* p = metaGetTableTagVal((*mr).me.ctbEntry.pTags, pDst->info.type, &val); + + char* data = NULL; + if (pDst->info.type != TSDB_DATA_TYPE_JSON && p != NULL) { + data = tTagValToData((const STagVal*)p, false); + } else { + data = (char*)p; + } + colDataSetVal(pDst, (*count), data, + (data == NULL) || (pDst->info.type == TSDB_DATA_TYPE_JSON && tTagIsJsonNull(data))); + + if (pDst->info.type != TSDB_DATA_TYPE_JSON && p != NULL && IS_VAR_DATA_TYPE(((const STagVal*)p)->type) && + data != NULL) { + taosMemoryFree(data); + } + } + } + + (*count) += 1; + if (++pInfo->curPos >= size) { + setOperatorCompleted(pOperator); + } +} + static SSDataBlock* doTagScan(SOperatorInfo* pOperator) { if (pOperator->status == OP_EXEC_DONE) { return NULL; @@ -2536,47 +2583,10 @@ static SSDataBlock* doTagScan(SOperatorInfo* pOperator) { metaReaderInit(&mr, pInfo->readHandle.meta, 0); while (pInfo->curPos < size && count < pOperator->resultInfo.capacity) { - STableKeyInfo* item = tableListGetInfo(pInfo->pTableListInfo, pInfo->curPos); - int32_t code = metaGetTableEntryByUid(&mr, item->uid); - tDecoderClear(&mr.coder); - if (code != TSDB_CODE_SUCCESS) { - qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", item->uid, tstrerror(terrno), - GET_TASKID(pTaskInfo)); - metaReaderClear(&mr); - T_LONG_JMP(pTaskInfo->env, terrno); - } - - for (int32_t j = 0; j < pOperator->exprSupp.numOfExprs; ++j) { - SColumnInfoData* pDst = taosArrayGet(pRes->pDataBlock, pExprInfo[j].base.resSchema.slotId); - - // refactor later - if (fmIsScanPseudoColumnFunc(pExprInfo[j].pExpr->_function.functionId)) { - STR_TO_VARSTR(str, mr.me.name); - colDataSetVal(pDst, count, str, false); - } else { // it is a tag value - STagVal val = {0}; - val.cid = pExprInfo[j].base.pParam[0].pCol->colId; - const char* p = metaGetTableTagVal(mr.me.ctbEntry.pTags, pDst->info.type, &val); - - char* data = NULL; - if (pDst->info.type != TSDB_DATA_TYPE_JSON && p != NULL) { - data = tTagValToData((const STagVal*)p, false); - } else { - data = (char*)p; - } - colDataSetVal(pDst, count, data, - (data == NULL) || (pDst->info.type == TSDB_DATA_TYPE_JSON && tTagIsJsonNull(data))); - - if (pDst->info.type != TSDB_DATA_TYPE_JSON && p != NULL && IS_VAR_DATA_TYPE(((const STagVal*)p)->type) && - data != NULL) { - taosMemoryFree(data); - } - } - } - - count += 1; - if (++pInfo->curPos >= size) { - setOperatorCompleted(pOperator); + doTagScanOneTable(pOperator, pTaskInfo, pInfo, pExprInfo, pRes, size, str, &count, &mr); + if (pInfo->pSlimit != NULL) { + pInfo->pRes->info.id.groupId = calcGroupId(mr.me.name, strlen(mr.me.name)); + break; } } @@ -2628,6 +2638,7 @@ SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysi pInfo->pRes = createDataBlockFromDescNode(pDescNode); pInfo->readHandle = *pReadHandle; pInfo->curPos = 0; + pInfo->pSlimit = (SLimitNode*)pPhyNode->node.pSlimit; //TODO: slimit now only indicate group setOperatorInfo(pOperator, "TagScanOperator", QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN, false, OP_NOT_OPENED, pInfo, pTaskInfo); diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 07ea110d7e..4ece9be304 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -2418,6 +2418,34 @@ static bool tagScanOptShouldBeOptimized(SLogicNode* pNode) { return true; } +static SLogicNode* tagScanOptFindAncestorWithSlimit(SLogicNode* pTableScanNode) { + SLogicNode* pNode = pTableScanNode->pParent; + while (NULL != pNode) { + if (QUERY_NODE_LOGIC_PLAN_PARTITION == nodeType(pNode) || QUERY_NODE_LOGIC_PLAN_AGG == nodeType(pNode) || + QUERY_NODE_LOGIC_PLAN_WINDOW == nodeType(pNode) || QUERY_NODE_LOGIC_PLAN_SORT == nodeType(pNode)) { + return NULL; + } + if (NULL != pNode->pSlimit) { + return pNode; + } + pNode = pNode->pParent; + } + return NULL; +} + +static void tagScanOptCloneAncestorSlimit(SLogicNode* pTableScanNode) { + if (NULL != pTableScanNode->pSlimit) { + return; + } + + SLogicNode* pNode = tagScanOptFindAncestorWithSlimit(pTableScanNode); + if (NULL != pNode) { + //TODO: only set the slimit now. push down slimit later + pTableScanNode->pSlimit = nodesCloneNode(pNode->pSlimit); + } + return; +} + static int32_t tagScanOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) { SScanLogicNode* pScanNode = (SScanLogicNode*)optFindPossibleNode(pLogicSubplan->pNode, tagScanOptShouldBeOptimized); if (NULL == pScanNode) { @@ -2458,6 +2486,7 @@ static int32_t tagScanOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubp NODES_CLEAR_LIST(pAgg->pChildren); } nodesDestroyNode((SNode*)pAgg); + tagScanOptCloneAncestorSlimit((SLogicNode*)pScanNode); pCxt->optimized = true; return TSDB_CODE_SUCCESS; } From e21fa9fde261f150c0bfc3ddc13dc136c7865428 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 24 Apr 2023 15:48:24 +0800 Subject: [PATCH 33/61] fix(tmq): fix error in unit test. --- source/client/test/clientTests.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index 08d62054e3..f15a93cb2c 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -1144,7 +1144,7 @@ TEST(clientCase, sub_tb_test) { taos_free_result(pRes); } - tmq_offset_seek(tmq, "topic_t1", pAssign[0].vgroupHandle, pAssign[0].begin); + tmq_offset_seek(tmq, "topic_t1", pAssign[0].vgId, pAssign[0].begin); } tmq_consumer_close(tmq); From 5a512eb76791a879687a16dd27f4a28b8ae224e1 Mon Sep 17 00:00:00 2001 From: cadem Date: Mon, 24 Apr 2023 16:06:56 +0800 Subject: [PATCH 34/61] remove null pointer --- source/libs/sync/src/syncMain.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 19bb126cb6..cb1919d16e 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -2156,7 +2156,7 @@ static void syncNodeEqPeerHeartbeatTimer(void* param, void* tmrId) { SSyncHbTimerData* pData = syncHbTimerDataAcquire(hbDataRid); if (pData == NULL) { - sError("hb timer get pData NULL, rid:%" PRId64 " addr:%" PRId64, hbDataRid, pData->destId.addr); + sError("hb timer get pData NULL, %" PRId64, hbDataRid); return; } From 9551a269b31e38e333e8c1942db2f74e865c6143 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 24 Apr 2023 16:45:34 +0800 Subject: [PATCH 35/61] feat: support log slow query by default --- include/util/taoserror.h | 9 +- include/util/tlog.h | 6 ++ source/client/src/clientEnv.c | 21 +++-- source/client/src/clientImpl.c | 5 ++ source/client/src/clientSml.c | 80 +++++++++++++---- source/common/src/tdataformat.c | 8 +- source/common/src/tglobal.c | 10 ++- source/util/src/terror.c | 3 +- source/util/src/tlog.c | 151 ++++++++++++++++++-------------- 9 files changed, 194 insertions(+), 99 deletions(-) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 0d9292cc6b..0511474fdb 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -103,7 +103,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_CHECKSUM_ERROR TAOS_DEF_ERROR_CODE(0, 0x011F) // internal #define TSDB_CODE_COMPRESS_ERROR TAOS_DEF_ERROR_CODE(0, 0x0120) -#define TSDB_CODE_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0121) // +#define TSDB_CODE_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0121) #define TSDB_CODE_CFG_NOT_FOUND TAOS_DEF_ERROR_CODE(0, 0x0122) #define TSDB_CODE_REPEAT_INIT TAOS_DEF_ERROR_CODE(0, 0x0123) #define TSDB_CODE_DUP_KEY TAOS_DEF_ERROR_CODE(0, 0x0124) @@ -118,9 +118,10 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MSG_ENCODE_ERROR TAOS_DEF_ERROR_CODE(0, 0x012D) #define TSDB_CODE_NO_ENOUGH_DISKSPACE TAOS_DEF_ERROR_CODE(0, 0x012E) -#define TSDB_CODE_APP_IS_STARTING TAOS_DEF_ERROR_CODE(0, 0x0130) // -#define TSDB_CODE_APP_IS_STOPPING TAOS_DEF_ERROR_CODE(0, 0x0131) // -#define TSDB_CODE_IVLD_DATA_FMT TAOS_DEF_ERROR_CODE(0, 0x0132) // +#define TSDB_CODE_APP_IS_STARTING TAOS_DEF_ERROR_CODE(0, 0x0130) +#define TSDB_CODE_APP_IS_STOPPING TAOS_DEF_ERROR_CODE(0, 0x0131) +#define TSDB_CODE_INVALID_DATA_FMT TAOS_DEF_ERROR_CODE(0, 0x0132) +#define TSDB_CODE_INVALID_CFG_VALUE TAOS_DEF_ERROR_CODE(0, 0x0133) //client #define TSDB_CODE_TSC_INVALID_OPERATION TAOS_DEF_ERROR_CODE(0, 0x0200) diff --git a/include/util/tlog.h b/include/util/tlog.h index 0071b3d32c..d267d5876a 100644 --- a/include/util/tlog.h +++ b/include/util/tlog.h @@ -83,6 +83,12 @@ void taosPrintLongString(const char *flags, ELogLevel level, int32_t dflag, cons #endif ; +void taosPrintSlowLog(const char *format, ...) +#ifdef __GNUC__ + __attribute__((format(printf, 1, 2))) +#endif + ; + bool taosAssertDebug(bool condition, const char *file, int32_t line, const char *format, ...); bool taosAssertRelease(bool condition); diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 418103f2a6..6691253585 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -42,7 +42,7 @@ SAppInfo appInfo; int64_t lastClusterId = 0; int32_t clientReqRefPool = -1; int32_t clientConnRefPool = -1; -int32_t clientStop = 0; +int32_t clientStop = -1; int32_t timestampDeltaLimit = 900; // s @@ -69,7 +69,6 @@ static int32_t registerRequest(SRequestObj *pRequest, STscObj *pTscObj) { } static void deregisterRequest(SRequestObj *pRequest) { - const static int64_t SLOW_QUERY_INTERVAL = 3000000L; // todo configurable if (pRequest == NULL) { tscError("pRequest == NULL"); return; @@ -80,6 +79,7 @@ static void deregisterRequest(SRequestObj *pRequest) { int32_t currentInst = atomic_sub_fetch_64((int64_t *)&pActivity->currentRequests, 1); int32_t num = atomic_sub_fetch_32(&pTscObj->numOfReqs, 1); + int32_t reqType = SLOW_LOG_TYPE_OTHERS; int64_t duration = taosGetTimestampUs() - pRequest->metric.start; tscDebug("0x%" PRIx64 " free Request from connObj: 0x%" PRIx64 ", reqId:0x%" PRIx64 @@ -95,6 +95,7 @@ static void deregisterRequest(SRequestObj *pRequest) { duration, pRequest->metric.parseCostUs, pRequest->metric.ctgCostUs, pRequest->metric.analyseCostUs, pRequest->metric.planCostUs, pRequest->metric.execCostUs); atomic_add_fetch_64((int64_t *)&pActivity->insertElapsedTime, duration); + reqType = SLOW_LOG_TYPE_INSERT; } else if (QUERY_NODE_SELECT_STMT == pRequest->stmtType) { tscDebug("query duration %" PRId64 "us: parseCost:%" PRId64 "us, ctgCost:%" PRId64 "us, analyseCost:%" PRId64 "us, planCost:%" PRId64 "us, exec:%" PRId64 "us", @@ -102,12 +103,16 @@ static void deregisterRequest(SRequestObj *pRequest) { pRequest->metric.planCostUs, pRequest->metric.execCostUs); atomic_add_fetch_64((int64_t *)&pActivity->queryElapsedTime, duration); + reqType = SLOW_LOG_TYPE_QUERY; } } - if (duration >= SLOW_QUERY_INTERVAL) { + if (duration >= (tsSlowLogThreshold * 1000000UL)) { atomic_add_fetch_64((int64_t *)&pActivity->numOfSlowQueries, 1); - tscWarnL("slow query: %s, duration:%" PRId64, pRequest->sqlstr, duration); + if (tsSlowLogScope & reqType) { + taosPrintSlowLog("PID:%d, Conn:%u, QID:0x%" PRIx64 ", Start:%" PRId64 ", Duration:%" PRId64 "us, SQL:%s", + taosGetPId(), pTscObj->connId, pRequest->requestId, pRequest->metric.start, duration, pRequest->sqlstr); + } } releaseTscObj(pTscObj->id); @@ -427,8 +432,12 @@ static void *tscCrashReportThreadFp(void *param) { } #endif + if (-1 != atomic_val_compare_exchange_32(&clientStop, -1, 0)) { + return NULL; + } + while (1) { - if (clientStop) break; + if (clientStop > 0) break; if (loopTimes++ < reportPeriodNum) { taosMsleep(sleepTime); continue; @@ -466,7 +475,7 @@ static void *tscCrashReportThreadFp(void *param) { loopTimes = 0; } - clientStop = -1; + clientStop = -2; return NULL; } diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index b9fa12b587..f8eade1d7c 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1248,6 +1248,11 @@ STscObj* taosConnectImpl(const char* user, const char* auth, const char* db, __t return NULL; } + pRequest->sqlstr = taosStrdup("taos_connect"); + if (pRequest->sqlstr) { + pRequest->sqlLen = strlen(pRequest->sqlstr); + } + SMsgSendInfo* body = buildConnectMsg(pRequest); int64_t transporterId = 0; diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 17150286e1..011a1b3111 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -660,6 +660,7 @@ static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns, SMCreateStbReq pReq = {0}; int32_t code = TSDB_CODE_SUCCESS; SCmdMsgInfo pCmdMsg = {0}; + char *pSql = NULL; // put front for free pReq.numOfColumns = taosArrayGetSize(pColumns); @@ -667,7 +668,27 @@ static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns, pReq.numOfTags = taosArrayGetSize(pTags); pReq.pTags = pTags; - code = buildRequest(info->taos->id, "", 0, NULL, false, &pRequest, 0); + if (action == SCHEMA_ACTION_CREATE_STABLE) { + pReq.colVer = 1; + pReq.tagVer = 1; + pReq.suid = 0; + pReq.source = TD_REQ_FROM_APP; + pSql = "sml_create_stable"; + } else if (action == SCHEMA_ACTION_ADD_TAG || action == SCHEMA_ACTION_CHANGE_TAG_SIZE) { + pReq.colVer = pTableMeta->sversion; + pReq.tagVer = pTableMeta->tversion + 1; + pReq.suid = pTableMeta->uid; + pReq.source = TD_REQ_FROM_TAOX; + pSql = (action == SCHEMA_ACTION_ADD_TAG) ? "sml_add_tag" : "sml_modify_tag_size"; + } else if (action == SCHEMA_ACTION_ADD_COLUMN || action == SCHEMA_ACTION_CHANGE_COLUMN_SIZE) { + pReq.colVer = pTableMeta->sversion + 1; + pReq.tagVer = pTableMeta->tversion; + pReq.suid = pTableMeta->uid; + pReq.source = TD_REQ_FROM_TAOX; + pSql = (action == SCHEMA_ACTION_ADD_COLUMN) ? "sml_add_column" : "sml_modify_column_size"; + } + + code = buildRequest(info->taos->id, pSql, strlen(pSql), NULL, false, &pRequest, 0); if (code != TSDB_CODE_SUCCESS) { goto end; } @@ -678,23 +699,6 @@ static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns, goto end; } - if (action == SCHEMA_ACTION_CREATE_STABLE) { - pReq.colVer = 1; - pReq.tagVer = 1; - pReq.suid = 0; - pReq.source = TD_REQ_FROM_APP; - } else if (action == SCHEMA_ACTION_ADD_TAG || action == SCHEMA_ACTION_CHANGE_TAG_SIZE) { - pReq.colVer = pTableMeta->sversion; - pReq.tagVer = pTableMeta->tversion + 1; - pReq.suid = pTableMeta->uid; - pReq.source = TD_REQ_FROM_TAOX; - } else if (action == SCHEMA_ACTION_ADD_COLUMN || action == SCHEMA_ACTION_CHANGE_COLUMN_SIZE) { - pReq.colVer = pTableMeta->sversion + 1; - pReq.tagVer = pTableMeta->tversion; - pReq.suid = pTableMeta->uid; - pReq.source = TD_REQ_FROM_TAOX; - } - if (pReq.numOfTags == 0) { pReq.numOfTags = 1; SField field = {0}; @@ -1514,6 +1518,44 @@ static int smlProcess(SSmlHandle *info, char *lines[], char *rawLine, char *rawL return code; } +void smlSetReqSQL(SRequestObj *request, char *lines[], char *rawLine, char *rawLineEnd) { + if (tsSlowLogScope & SLOW_LOG_TYPE_INSERT) { + int32_t len = 0; + int32_t rlen = 0; + char* p = NULL; + + if (lines && lines[0]) { + len = strlen(lines[0]); + p = lines[0]; + } else if (rawLine) { + if (rawLineEnd) { + len = rawLineEnd - rawLine; + } else { + len = strlen(rawLine); + } + p = rawLine; + } + + if (NULL == p) { + return; + } + + rlen = TMIN(len, TSDB_MAX_ALLOWED_SQL_LEN); + rlen = TMAX(rlen, 0); + + char *sql = taosMemoryMalloc(rlen + 1); + if (NULL == sql) { + uError("malloc %d for sml sql failed", rlen + 1); + return; + } + memcpy(sql, p, rlen); + sql[rlen] = 0; + + request->sqlstr = sql; + request->sqlLen = rlen; + } +} + TAOS_RES *taos_schemaless_insert_inner(TAOS *taos, char *lines[], char *rawLine, char *rawLineEnd, int numLines, int protocol, int precision, int32_t ttl, int64_t reqid) { int32_t code = TSDB_CODE_SUCCESS; @@ -1546,6 +1588,8 @@ TAOS_RES *taos_schemaless_insert_inner(TAOS *taos, char *lines[], char *rawLine, info->msgBuf.len = ERROR_MSG_BUF_DEFAULT_SIZE; info->lineNum = numLines; + smlSetReqSQL(request, lines, rawLine, rawLineEnd); + SSmlMsgBuf msg = {ERROR_MSG_BUF_DEFAULT_SIZE, request->msgBuf}; if (request->pDb == NULL) { request->code = TSDB_CODE_PAR_DB_NOT_SPECIFIED; diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index d6ab974c6c..0cbc5b2230 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -500,7 +500,7 @@ int32_t tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal) break; default: ASSERTS(0, "invalid row format"); - return TSDB_CODE_IVLD_DATA_FMT; + return TSDB_CODE_INVALID_DATA_FMT; } if (bv == BIT_FLG_NONE) { @@ -938,7 +938,7 @@ static int32_t tRowTupleUpsertColData(SRow *pRow, STSchema *pTSchema, SColData * break; default: ASSERTS(0, "Invalid row flag"); - return TSDB_CODE_IVLD_DATA_FMT; + return TSDB_CODE_INVALID_DATA_FMT; } while (pColData) { @@ -963,7 +963,7 @@ static int32_t tRowTupleUpsertColData(SRow *pRow, STSchema *pTSchema, SColData * break; default: ASSERTS(0, "Invalid row flag"); - return TSDB_CODE_IVLD_DATA_FMT; + return TSDB_CODE_INVALID_DATA_FMT; } if (bv == BIT_FLG_NONE) { @@ -1054,7 +1054,7 @@ static int32_t tRowKVUpsertColData(SRow *pRow, STSchema *pTSchema, SColData *aCo pData = pv + ((uint32_t *)pKVIdx->idx)[iCol]; } else { ASSERTS(0, "Invalid KV row format"); - return TSDB_CODE_IVLD_DATA_FMT; + return TSDB_CODE_INVALID_DATA_FMT; } int16_t cid; diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 224b0068fe..0a29a415f0 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -724,7 +724,13 @@ static int32_t taosSetSlowLogScope(char *pScope) { return 0; } + if (0 == strcasecmp(pScope, "none")) { + tsSlowLogScope = 0; + return 0; + } + uError("Invalid slowLog scope value:%s", pScope); + terrno = TSDB_CODE_INVALID_CFG_VALUE; return -1; } @@ -1199,7 +1205,9 @@ int32_t taosSetCfg(SConfig *pCfg, char *name) { } else if (strcasecmp("slowLogThreshold", name) == 0) { tsSlowLogThreshold = cfgGetItem(pCfg, "slowLogThreshold")->i32; } else if (strcasecmp("slowLogScope", name) == 0) { - taosSetSlowLogScope(cfgGetItem(pCfg, "slowLogScope")->str) + if (taosSetSlowLogScope(cfgGetItem(pCfg, "slowLogScope")->str)) { + return -1; + } } break; } diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 34b09761c8..29e76bfaf4 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -98,7 +98,8 @@ TAOS_DEFINE_ERROR(TSDB_CODE_NO_ENOUGH_DISKSPACE, "No enough disk space" TAOS_DEFINE_ERROR(TSDB_CODE_APP_IS_STARTING, "Database is starting up") TAOS_DEFINE_ERROR(TSDB_CODE_APP_IS_STOPPING, "Database is closing down") -TAOS_DEFINE_ERROR(TSDB_CODE_IVLD_DATA_FMT, "Invalid data format") +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_DATA_FMT, "Invalid data format") +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_CFG_VALUE, "Invalid configuration value") //client TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_OPERATION, "Invalid operation") diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 10ab389873..688d893d53 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -24,7 +24,7 @@ #define LOG_MAX_LINE_SIZE (10024) #define LOG_MAX_LINE_BUFFER_SIZE (LOG_MAX_LINE_SIZE + 3) #define LOG_MAX_LINE_DUMP_SIZE (1024 * 1024) -#define LOG_MAX_LINE_DUMP_BUFFER_SIZE (LOG_MAX_LINE_DUMP_SIZE + 3) +#define LOG_MAX_LINE_DUMP_BUFFER_SIZE (LOG_MAX_LINE_DUMP_SIZE + 128) #define LOG_FILE_NAME_LEN 300 #define LOG_DEFAULT_BUF_SIZE (20 * 1024 * 1024) // 20MB @@ -52,6 +52,8 @@ typedef struct { int32_t stop; TdThread asyncThread; TdThreadMutex buffMutex; + int32_t writeInterval; + int32_t lastDuration; } SLogBuff; typedef struct { @@ -71,7 +73,6 @@ extern SConfig *tsCfg; static int8_t tsLogInited = 0; static SLogObj tsLogObj = {.fileNum = 1}; static int64_t tsAsyncLogLostLines = 0; -static int32_t tsWriteInterval = LOG_DEFAULT_INTERVAL; static int32_t tsDaylightActive; /* Currently in daylight saving time. */ bool tsLogEmbedded = 0; @@ -84,6 +85,7 @@ int64_t tsNumOfErrorLogs = 0; int64_t tsNumOfInfoLogs = 0; int64_t tsNumOfDebugLogs = 0; int64_t tsNumOfTraceLogs = 0; +int64_t tsNumOfSlowLogs = 0; // log int32_t dDebugFlag = 131; @@ -203,7 +205,6 @@ void taosCloseLog() { taosThreadMutexDestroy(&tsLogObj.slowHandle->buffMutex); taosCloseFile(&tsLogObj.slowHandle->pFile); taosMemoryFreeClear(tsLogObj.slowHandle->buffer); - memset(&tsLogObj.slowHandle->buffer, 0, sizeof(tsLogObj.slowHandle->buffer)); taosMemoryFreeClear(tsLogObj.slowHandle); } @@ -217,7 +218,6 @@ void taosCloseLog() { taosThreadMutexDestroy(&tsLogObj.logHandle->buffMutex); taosCloseFile(&tsLogObj.logHandle->pFile); taosMemoryFreeClear(tsLogObj.logHandle->buffer); - memset(&tsLogObj.logHandle->buffer, 0, sizeof(tsLogObj.logHandle->buffer)); taosThreadMutexDestroy(&tsLogObj.logMutex); taosMemoryFreeClear(tsLogObj.logHandle); } @@ -555,10 +555,9 @@ void taosPrintLongString(const char *flags, ELogLevel level, int32_t dflag, cons va_list argpointer; va_start(argpointer, format); - len += vsnprintf(buffer + len, LOG_MAX_LINE_DUMP_BUFFER_SIZE - len, format, argpointer); + len += vsnprintf(buffer + len, LOG_MAX_LINE_DUMP_BUFFER_SIZE - 2 - len, format, argpointer); va_end(argpointer); - if (len > LOG_MAX_LINE_DUMP_SIZE) len = LOG_MAX_LINE_DUMP_SIZE; buffer[len++] = '\n'; buffer[len] = 0; @@ -566,6 +565,31 @@ void taosPrintLongString(const char *flags, ELogLevel level, int32_t dflag, cons taosMemoryFree(buffer); } +void taosPrintSlowLog(const char *format, ...) { + if (!osLogSpaceAvailable()) return; + + char *buffer = taosMemoryMalloc(LOG_MAX_LINE_DUMP_BUFFER_SIZE); + int32_t len = taosBuildLogHead(buffer, ""); + + va_list argpointer; + va_start(argpointer, format); + len += vsnprintf(buffer + len, LOG_MAX_LINE_DUMP_BUFFER_SIZE - 2 - len, format, argpointer); + va_end(argpointer); + + buffer[len++] = '\n'; + buffer[len] = 0; + + atomic_add_fetch_64(&tsNumOfSlowLogs, 1); + + if (tsAsyncLog) { + taosPushLogBuffer(tsLogObj.slowHandle, buffer, len); + } else { + taosWriteFile(tsLogObj.slowHandle->pFile, buffer, len); + } + + taosMemoryFree(buffer); +} + void taosDumpData(unsigned char *msg, int32_t len) { if (!osLogSpaceAvailable()) return; taosUpdateLogNums(DEBUG_DUMP); @@ -610,6 +634,7 @@ static SLogBuff *taosLogBuffNew(int32_t bufSize) { LOG_BUF_SIZE(pLogBuf) = bufSize; pLogBuf->minBuffSize = bufSize / 10; pLogBuf->stop = 0; + pLogBuf->writeInterval = LOG_DEFAULT_INTERVAL; if (taosThreadMutexInit(&LOG_BUF_MUTEX(pLogBuf), NULL) < 0) goto _err; // tsem_init(&(pLogBuf->buffNotEmpty), 0, 0); @@ -693,83 +718,78 @@ static int32_t taosGetLogRemainSize(SLogBuff *pLogBuf, int32_t start, int32_t en } static void taosWriteLog(SLogBuff *pLogBuf) { - static int32_t lastDuration = 0; - int32_t remainChecked = 0; - int32_t start, end, pollSize; + int32_t start = LOG_BUF_START(pLogBuf); + int32_t end = LOG_BUF_END(pLogBuf); - do { - if (remainChecked == 0) { - start = LOG_BUF_START(pLogBuf); - end = LOG_BUF_END(pLogBuf); + if (start == end) { + dbgEmptyW++; + pLogBuf->writeInterval = LOG_MAX_INTERVAL; + return; + } - if (start == end) { - dbgEmptyW++; - tsWriteInterval = LOG_MAX_INTERVAL; - return; - } - - pollSize = taosGetLogRemainSize(pLogBuf, start, end); - if (pollSize < pLogBuf->minBuffSize) { - lastDuration += tsWriteInterval; - if (lastDuration < LOG_MAX_WAIT_MSEC) { - break; - } - } - - lastDuration = 0; + int32_t pollSize = taosGetLogRemainSize(pLogBuf, start, end); + if (pollSize < pLogBuf->minBuffSize) { + pLogBuf->lastDuration += pLogBuf->writeInterval; + if (pLogBuf->lastDuration < LOG_MAX_WAIT_MSEC) { + return; } + } - if (start < end) { - taosWriteFile(pLogBuf->pFile, LOG_BUF_BUFFER(pLogBuf) + start, pollSize); - } else { - int32_t tsize = LOG_BUF_SIZE(pLogBuf) - start; - taosWriteFile(pLogBuf->pFile, LOG_BUF_BUFFER(pLogBuf) + start, tsize); + pLogBuf->lastDuration = 0; - taosWriteFile(pLogBuf->pFile, LOG_BUF_BUFFER(pLogBuf), end); + if (start < end) { + taosWriteFile(pLogBuf->pFile, LOG_BUF_BUFFER(pLogBuf) + start, pollSize); + } else { + int32_t tsize = LOG_BUF_SIZE(pLogBuf) - start; + taosWriteFile(pLogBuf->pFile, LOG_BUF_BUFFER(pLogBuf) + start, tsize); + + taosWriteFile(pLogBuf->pFile, LOG_BUF_BUFFER(pLogBuf), end); + } + + dbgWN++; + dbgWSize += pollSize; + + if (pollSize < pLogBuf->minBuffSize) { + dbgSmallWN++; + if (pLogBuf->writeInterval < LOG_MAX_INTERVAL) { + pLogBuf->writeInterval += LOG_INTERVAL_STEP; } - - dbgWN++; - dbgWSize += pollSize; - - if (pollSize < pLogBuf->minBuffSize) { - dbgSmallWN++; - if (tsWriteInterval < LOG_MAX_INTERVAL) { - tsWriteInterval += LOG_INTERVAL_STEP; - } - } else if (pollSize > LOG_BUF_SIZE(pLogBuf) / 3) { - dbgBigWN++; - tsWriteInterval = LOG_MIN_INTERVAL; - } else if (pollSize > LOG_BUF_SIZE(pLogBuf) / 4) { - if (tsWriteInterval > LOG_MIN_INTERVAL) { - tsWriteInterval -= LOG_INTERVAL_STEP; - } + } else if (pollSize > LOG_BUF_SIZE(pLogBuf) / 3) { + dbgBigWN++; + pLogBuf->writeInterval = LOG_MIN_INTERVAL; + } else if (pollSize > LOG_BUF_SIZE(pLogBuf) / 4) { + if (pLogBuf->writeInterval > LOG_MIN_INTERVAL) { + pLogBuf->writeInterval -= LOG_INTERVAL_STEP; } + } - LOG_BUF_START(pLogBuf) = (LOG_BUF_START(pLogBuf) + pollSize) % LOG_BUF_SIZE(pLogBuf); + LOG_BUF_START(pLogBuf) = (LOG_BUF_START(pLogBuf) + pollSize) % LOG_BUF_SIZE(pLogBuf); - start = LOG_BUF_START(pLogBuf); - end = LOG_BUF_END(pLogBuf); + start = LOG_BUF_START(pLogBuf); + end = LOG_BUF_END(pLogBuf); - pollSize = taosGetLogRemainSize(pLogBuf, start, end); - if (pollSize < pLogBuf->minBuffSize) { - break; - } + pollSize = taosGetLogRemainSize(pLogBuf, start, end); + if (pollSize < pLogBuf->minBuffSize) { + return; + } - tsWriteInterval = LOG_MIN_INTERVAL; - - remainChecked = 1; - } while (1); + pLogBuf->writeInterval = 0; } static void *taosAsyncOutputLog(void *param) { - SLogBuff *pLogBuf = (SLogBuff *)param; + SLogBuff *pLogBuf = (SLogBuff *)tsLogObj.logHandle; + SLogBuff *pSlowBuf = (SLogBuff *)tsLogObj.slowHandle; + setThreadName("log"); int32_t count = 0; int32_t updateCron = 0; + int32_t writeInterval = 0; + while (1) { - count += tsWriteInterval; + writeInterval = TMIN(pLogBuf->writeInterval, pSlowBuf->writeInterval); + count += writeInterval; updateCron++; - taosMsleep(tsWriteInterval); + taosMsleep(writeInterval); if (count > 1000) { osUpdate(); count = 0; @@ -777,13 +797,14 @@ static void *taosAsyncOutputLog(void *param) { // Polling the buffer taosWriteLog(pLogBuf); + taosWriteLog(pSlowBuf); if (updateCron >= 3600 * 24 * 40 / 2) { taosUpdateDaylight(); updateCron = 0; } - if (pLogBuf->stop) break; + if (pLogBuf->stop || pSlowBuf->stop) break; } return NULL; From 809fe165e78cb8cc01e9ecad8410beea0e0f1299 Mon Sep 17 00:00:00 2001 From: sunpeng Date: Mon, 24 Apr 2023 17:54:01 +0800 Subject: [PATCH 36/61] docs: add docs for `taoskeeper with prometheus` (#21052) --- docs/en/14-reference/14-taosKeeper.md | 26 ++++++++++++++++++++++++++ docs/zh/14-reference/14-taosKeeper.md | 26 ++++++++++++++++++++++++++ 2 files changed, 52 insertions(+) diff --git a/docs/en/14-reference/14-taosKeeper.md b/docs/en/14-reference/14-taosKeeper.md index 9c4a2da921..cfc2554387 100644 --- a/docs/en/14-reference/14-taosKeeper.md +++ b/docs/en/14-reference/14-taosKeeper.md @@ -223,3 +223,29 @@ Content-Length: 19 {"version":"1.0.0"} ``` + +### taoskeeper with Prometheus + +There is `/metrics` api in taoskeeper provide TDengine metric data for Prometheus. + +#### scrape config + +Scrape config in Prometheus specifies a set of targets and parameters describing how to scrape metric data from endpoint. For more information, please reference to [Prometheus documents](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config). + +``` +# A scrape configuration containing exactly one endpoint to scrape: +# Here it's Prometheus itself. +scrape_configs: + - job_name: "taoskeeper" + # metrics_path defaults to '/metrics' + # scheme defaults to 'http'. + static_configs: + - targets: ["localhost:6043"] +``` + +#### Dashboard + +There is a dashboard named `TaosKeeper Prometheus Dashboard for 3.x`, which provides a monitoring dashboard similar to TInsight. + +In Grafana, click the Dashboard menu and click `import`, enter the dashboard ID `18587` and click the `Load` button. Then finished importing `TaosKeeper Prometheus Dashboard for 3.x` dashboard. + diff --git a/docs/zh/14-reference/14-taosKeeper.md b/docs/zh/14-reference/14-taosKeeper.md index 03ca30781f..b4d35fb240 100644 --- a/docs/zh/14-reference/14-taosKeeper.md +++ b/docs/zh/14-reference/14-taosKeeper.md @@ -226,3 +226,29 @@ Content-Length: 19 {"version":"1.0.0"} ``` + +### 集成 Prometheus + +taoskeeper 提供了 `/metrics` 接口，返回了 Prometheus 格式的监控数据，Prometheus 可以从 taoskeeper 抽取监控数据，实现通过 Prometheus 监控 TDengine 的目的。 + +#### 抽取配置 + +Prometheus 提供了 `scrape_configs` 配置如何从 endpoint 抽取监控数据，通常只需要修改 `static_configs` 中的 targets 配置为 taoskeeper 的 endpoint 地址，更多配置信息请参考 [Prometheus 配置文档](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config)。 + +``` +# A scrape configuration containing exactly one endpoint to scrape: +# Here it's Prometheus itself. +scrape_configs: + - job_name: "taoskeeper" + # metrics_path defaults to '/metrics' + # scheme defaults to 'http'. + static_configs: + - targets: ["localhost:6043"] +``` + +#### Dashboard + +我们提供了 `TaosKeeper Prometheus Dashboard for 3.x` dashboard，提供了和 TDinsight 类似的监控 dashboard。 + +在 Grafana Dashboard 菜单点击 `import`，dashboard ID 填写 `18587`，点击 `Load` 按钮即可导入 `TaosKeeper Prometheus Dashboard for 3.x` dashboard。 + From 817dc57611969230e92469d5e5bddd0850192f7f Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 24 Apr 2023 19:00:07 +0800 Subject: [PATCH 37/61] Update 14-stream.md --- docs/zh/12-taos-sql/14-stream.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/docs/zh/12-taos-sql/14-stream.md b/docs/zh/12-taos-sql/14-stream.md index 12e466f349..9bbb551c87 100644 --- a/docs/zh/12-taos-sql/14-stream.md +++ b/docs/zh/12-taos-sql/14-stream.md @@ -213,3 +213,29 @@ DELETE_MARK time ``` DELETE_MARK用于删除缓存的窗口状态，也就是删除流计算的中间结果。如果不设置，默认值是10年 T = 最新事件时间 - DELETE_MARK + +## 流式计算支持的函数 + +1. 所有的单行函数均可用于流计算中。 +2. 以下 19 个聚合函数不能在创建流计算的 SQL 语句中使用 +``` +leastsquares +percentile +top +bottom +elapsed +interp +derivative +irate +twa +histogram +diff +statecount +stateduration +csum +mavg +sample +tail +unique +mode +``` From de1161dafb47fdd7142b610f21bb4491f3dd9f08 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 24 Apr 2023 19:22:44 +0800 Subject: [PATCH 38/61] enh: ignore single row null data type validation --- source/common/src/tdataformat.c | 8 +++++--- source/libs/parser/src/parInsertStmt.c | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index d6ab974c6c..f379084cf5 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -2503,9 +2503,11 @@ _exit: int32_t tColDataAddValueByBind(SColData *pColData, TAOS_MULTI_BIND *pBind) { int32_t code = 0; - ASSERT(pColData->type == pBind->buffer_type); - - if (IS_VAR_DATA_TYPE(pBind->buffer_type)) { // var-length data type + if (!(pBind->num == 1 && pBind->is_null && *pBind->is_null)) { + ASSERT(pColData->type == pBind->buffer_type); + } + + if (IS_VAR_DATA_TYPE(pColData->type)) { // var-length data type for (int32_t i = 0; i < pBind->num; ++i) { if (pBind->is_null && pBind->is_null[i]) { code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NULL](pColData, NULL, 0); diff --git a/source/libs/parser/src/parInsertStmt.c b/source/libs/parser/src/parInsertStmt.c index 01a635e4b2..922a0f45ff 100644 --- a/source/libs/parser/src/parInsertStmt.c +++ b/source/libs/parser/src/parInsertStmt.c @@ -251,7 +251,7 @@ int32_t qBindStmtColsValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, in goto _return; } - if (bind[c].buffer_type != pColSchema->type) { + if ((!(rowNum == 1 && bind[c].is_null && *bind[c].is_null)) && bind[c].buffer_type != pColSchema->type) { // for rowNum ==1 , connector may not set buffer_type code = buildInvalidOperationMsg(&pBuf, "column type mis-match with buffer type"); goto _return; } From 28bff83096e9906f5233f59c864792cff040949a Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 24 Apr 2023 19:29:33 +0800 Subject: [PATCH 39/61] fix: log thread quit issue --- source/util/src/tlog.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 688d893d53..baf6a9f319 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -201,6 +201,11 @@ static void taosStopLog() { void taosCloseLog() { taosStopLog(); + if (tsLogObj.logHandle != NULL && taosCheckPthreadValid(tsLogObj.logHandle->asyncThread)) { + taosThreadJoin(tsLogObj.logHandle->asyncThread, NULL); + taosThreadClear(&tsLogObj.logHandle->asyncThread); + } + if (tsLogObj.slowHandle != NULL) { taosThreadMutexDestroy(&tsLogObj.slowHandle->buffMutex); taosCloseFile(&tsLogObj.slowHandle->pFile); @@ -209,10 +214,6 @@ void taosCloseLog() { } if (tsLogObj.logHandle != NULL) { - if (tsLogObj.logHandle != NULL && taosCheckPthreadValid(tsLogObj.logHandle->asyncThread)) { - taosThreadJoin(tsLogObj.logHandle->asyncThread, NULL); - taosThreadClear(&tsLogObj.logHandle->asyncThread); - } tsLogInited = 0; taosThreadMutexDestroy(&tsLogObj.logHandle->buffMutex); From 655774c32e4b69d29dc3c0509e2a9dedaa2112df Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 25 Apr 2023 09:00:41 +0800 Subject: [PATCH 40/61] fix: move error msg from tsc to shell --- tools/shell/src/shellEngine.c | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 910b067d4e..478801d3e7 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -1115,6 +1115,7 @@ int32_t shellExecute() { } if (shell.conn == NULL) { + printf("failed to connect to server, reason: %s\n", taos_errstr(NULL)); fflush(stdout); return -1; } From 93ab81437f57733e536e108d8e34ad51115f94ff Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Tue, 25 Apr 2023 11:08:58 +0800 Subject: [PATCH 41/61] enhance: code refactor and operator full implement slimit --- source/libs/executor/src/scanoperator.c | 31 ++++++++++++++++--------- source/libs/planner/src/planOptimizer.c | 2 ++ 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index b7315db204..6d62d55024 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -2512,9 +2512,11 @@ _error: return NULL; } -static void doTagScanOneTable(SOperatorInfo* pOperator, const SExecTaskInfo* pTaskInfo, STagScanInfo* pInfo, - const SExprInfo* pExprInfo, const SSDataBlock* pRes, int32_t size, const char* str, - int32_t* count, SMetaReader* mr) { +static void doTagScanOneTable(SOperatorInfo* pOperator, const SSDataBlock* pRes, int32_t count, SMetaReader* mr) { + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + STagScanInfo* pInfo = pOperator->info; + SExprInfo* pExprInfo = &pOperator->exprSupp.pExprInfo[0]; + STableKeyInfo* item = tableListGetInfo(pInfo->pTableListInfo, pInfo->curPos); int32_t code = metaGetTableEntryByUid(mr, item->uid); tDecoderClear(&(*mr).coder); @@ -2525,13 +2527,14 @@ static void doTagScanOneTable(SOperatorInfo* pOperator, const SExecTaskInfo* pTa T_LONG_JMP(pTaskInfo->env, terrno); } + char str[512]; for (int32_t j = 0; j < pOperator->exprSupp.numOfExprs; ++j) { SColumnInfoData* pDst = taosArrayGet(pRes->pDataBlock, pExprInfo[j].base.resSchema.slotId); // refactor later if (fmIsScanPseudoColumnFunc(pExprInfo[j].pExpr->_function.functionId)) { STR_TO_VARSTR(str, (*mr).me.name); - colDataSetVal(pDst, (*count), str, false); + colDataSetVal(pDst, (count), str, false); } else { // it is a tag value STagVal val = {0}; val.cid = pExprInfo[j].base.pParam[0].pCol->colId; @@ -2543,7 +2546,7 @@ static void doTagScanOneTable(SOperatorInfo* pOperator, const SExecTaskInfo* pTa } else { data = (char*)p; } - colDataSetVal(pDst, (*count), data, + colDataSetVal(pDst, (count), data, (data == NULL) || (pDst->info.type == TSDB_DATA_TYPE_JSON && tTagIsJsonNull(data))); if (pDst->info.type != TSDB_DATA_TYPE_JSON && p != NULL && IS_VAR_DATA_TYPE(((const STagVal*)p)->type) && @@ -2552,11 +2555,6 @@ static void doTagScanOneTable(SOperatorInfo* pOperator, const SExecTaskInfo* pTa } } } - - (*count) += 1; - if (++pInfo->curPos >= size) { - setOperatorCompleted(pOperator); - } } static SSDataBlock* doTagScan(SOperatorInfo* pOperator) { @@ -2583,9 +2581,20 @@ static SSDataBlock* doTagScan(SOperatorInfo* pOperator) { metaReaderInit(&mr, pInfo->readHandle.meta, 0); while (pInfo->curPos < size && count < pOperator->resultInfo.capacity) { - doTagScanOneTable(pOperator, pTaskInfo, pInfo, pExprInfo, pRes, size, str, &count, &mr); + doTagScanOneTable(pOperator, pRes, count, &mr); + ++count; + if (++pInfo->curPos >= size) { + setOperatorCompleted(pOperator); + } + // each table with tbname is a group, hence its own block, but only group when slimit exists for performance reason. if (pInfo->pSlimit != NULL) { + if (pInfo->curPos < pInfo->pSlimit->offset) { + continue; + } pInfo->pRes->info.id.groupId = calcGroupId(mr.me.name, strlen(mr.me.name)); + if (pInfo->curPos >= (pInfo->pSlimit->offset + pInfo->pSlimit->limit) - 1) { + setOperatorCompleted(pOperator); + } break; } } diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 4ece9be304..4f8b57de5f 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -2442,6 +2442,8 @@ static void tagScanOptCloneAncestorSlimit(SLogicNode* pTableScanNode) { if (NULL != pNode) { //TODO: only set the slimit now. push down slimit later pTableScanNode->pSlimit = nodesCloneNode(pNode->pSlimit); + ((SLimitNode*)pTableScanNode->pSlimit)->limit += ((SLimitNode*)pTableScanNode->pSlimit)->offset; + ((SLimitNode*)pTableScanNode->pSlimit)->offset = 0; } return; } From ae773fee8b17102feca3bfbe7a4ce2c6112b8dc3 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Tue, 25 Apr 2023 14:43:22 +0800 Subject: [PATCH 42/61] feature: add test case --- tests/parallel_test/cases.task | 1 + tests/script/tsim/query/tag_scan.sim | 48 ++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+) create mode 100644 tests/script/tsim/query/tag_scan.sim diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index dda4ec3e84..9f9679ae35 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -887,6 +887,7 @@ ,,y,script,./test.sh -f tsim/query/emptyTsRange.sim ,,y,script,./test.sh -f tsim/query/partitionby.sim ,,y,script,./test.sh -f tsim/query/tableCount.sim +,,y,script,./test.sh -f tsim/query/tag_scan.sim ,,y,script,./test.sh -f tsim/query/nullColSma.sim ,,y,script,./test.sh -f tsim/qnode/basic1.sim ,,y,script,./test.sh -f tsim/snode/basic1.sim diff --git a/tests/script/tsim/query/tag_scan.sim b/tests/script/tsim/query/tag_scan.sim new file mode 100644 index 0000000000..03e3a20632 --- /dev/null +++ b/tests/script/tsim/query/tag_scan.sim @@ -0,0 +1,48 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sql connect +sql drop database if exists test +sql create database test; +sql use test; + +sql create table st(ts timestamp, f int) tags (t int); +sql insert into ct1 using st tags(1) values(now, 1); +sql insert into ct2 using st tags(2) values(now, 2); +sql insert into ct3 using st tags(3) values(now, 3); +sql insert into ct4 using st tags(4) values(now, 4); + +sql create table st2(ts timestamp, f int) tags (t int); +sql insert into ct21 using st2 tags(1) values(now, 1); +sql insert into ct22 using st2 tags(2) values(now, 2); +sql insert into ct23 using st2 tags(3) values(now, 3); +sql insert into ct24 using st2 tags(4) values(now, 4); + +sql select tbname, 1 from st group by tbname order by tbname; +print $rows $data00 $data10 $data20 +if $rows != 4 then + return -1 +endi +if $data00 != @ct1@ then + return -1 +endi +if $data10 != @ct2@ then + return -1 +endi +sql select tbname, 1 from st group by tbname slimit 0, 1; +print $rows +if $rows != 1 then + return -1 +endi +sql select tbname, 1 from st group by tbname slimit 2, 2; +print $rows $data00 $data10 +if $rows != 2 then + return -1 +endi +sql select tbname, 1 from st group by tbname order by tbname slimit 0, 1; +print $rows $data00 $data10 $data20 +if $rows != 4 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT From 2fd9640a39ce88025fbe21ba73e10e6717ff2cc3 Mon Sep 17 00:00:00 2001 From: cadem Date: Tue, 25 Apr 2023 15:15:28 +0800 Subject: [PATCH 43/61] change learner config format --- source/dnode/vnode/src/vnd/vnodeCfg.c | 20 +++++++------------- source/libs/sync/src/syncRaftCfg.c | 14 ++++++-------- 2 files changed, 13 insertions(+), 21 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeCfg.c b/source/dnode/vnode/src/vnd/vnodeCfg.c index 65f32b0a85..511ba9cc24 100644 --- a/source/dnode/vnode/src/vnd/vnodeCfg.c +++ b/source/dnode/vnode/src/vnd/vnodeCfg.c @@ -60,19 +60,19 @@ int vnodeCheckCfg(const SVnodeCfg *pCfg) { const char* vnodeRoleToStr(ESyncRole role) { switch (role) { case TAOS_SYNC_ROLE_VOTER: - return "voter"; + return "true"; case TAOS_SYNC_ROLE_LEARNER: - return "learner"; + return "false"; default: return "unknown"; } } const ESyncRole vnodeStrToRole(char* str) { - if(strcmp(str, "voter") == 0){ + if(strcmp(str, "true") == 0){ return TAOS_SYNC_ROLE_VOTER; } - if(strcmp(str, "learner") == 0){ + if(strcmp(str, "false") == 0){ return TAOS_SYNC_ROLE_LEARNER; } @@ -139,7 +139,6 @@ int vnodeEncodeConfig(const void *pObj, SJson *pJson) { if (tjsonAddIntegerToObject(pJson, "hashSuffix", pCfg->hashSuffix) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "syncCfg.replicaNum", pCfg->syncCfg.replicaNum) < 0) return -1; - if (tjsonAddIntegerToObject(pJson, "syncCfg.totalReplicaNum", pCfg->syncCfg.totalReplicaNum) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "syncCfg.myIndex", pCfg->syncCfg.myIndex) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "vndStats.stables", pCfg->vndStats.numOfSTables) < 0) return -1; @@ -161,7 +160,7 @@ int vnodeEncodeConfig(const void *pObj, SJson *pJson) { if (tjsonAddStringToObject(info, "nodeFqdn", pNode->nodeFqdn) < 0) return -1; if (tjsonAddIntegerToObject(info, "nodeId", pNode->nodeId) < 0) return -1; if (tjsonAddIntegerToObject(info, "clusterId", pNode->clusterId) < 0) return -1; - if (tjsonAddStringToObject(info, "nodeRole", vnodeRoleToStr(pNode->nodeRole)) < 0) return -1; + if (tjsonAddStringToObject(info, "isReplica", vnodeRoleToStr(pNode->nodeRole)) < 0) return -1; if (tjsonAddItemToArray(nodeInfo, info) < 0) return -1; vDebug("vgId:%d, encode config, replica:%d ep:%s:%u dnode:%d", pCfg->vgId, i, pNode->nodeFqdn, pNode->nodePort, pNode->nodeId); @@ -259,8 +258,6 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) { tjsonGetNumberValue(pJson, "syncCfg.replicaNum", pCfg->syncCfg.replicaNum, code); if (code < 0) return -1; - tjsonGetNumberValue(pJson, "syncCfg.totalReplicaNum", pCfg->syncCfg.totalReplicaNum, code); - if (code < 0) return -1; tjsonGetNumberValue(pJson, "syncCfg.myIndex", pCfg->syncCfg.myIndex, code); if (code < 0) return -1; @@ -277,10 +274,7 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) { SJson *nodeInfo = tjsonGetObjectItem(pJson, "syncCfg.nodeInfo"); int arraySize = tjsonGetArraySize(nodeInfo); - if(pCfg->syncCfg.totalReplicaNum == 0 && pCfg->syncCfg.replicaNum > 0){ - pCfg->syncCfg.totalReplicaNum = pCfg->syncCfg.replicaNum; - } - if (arraySize != pCfg->syncCfg.totalReplicaNum) return -1; + pCfg->syncCfg.totalReplicaNum = arraySize; vDebug("vgId:%d, decode config, replicas:%d totalReplicas:%d selfIndex:%d", pCfg->vgId, pCfg->syncCfg.replicaNum, pCfg->syncCfg.totalReplicaNum, pCfg->syncCfg.myIndex); @@ -296,7 +290,7 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) { tjsonGetNumberValue(info, "clusterId", pNode->clusterId, code); if (code < 0) return -1; char role[10] = {0}; - code = tjsonGetStringValue(info, "nodeRole", role); + code = tjsonGetStringValue(info, "isReplica", role); if (code < 0) return -1; if(strlen(role) != 0){ pNode->nodeRole = vnodeStrToRole(role); diff --git a/source/libs/sync/src/syncRaftCfg.c b/source/libs/sync/src/syncRaftCfg.c index 576b9d62f5..480ed4b2af 100644 --- a/source/libs/sync/src/syncRaftCfg.c +++ b/source/libs/sync/src/syncRaftCfg.c @@ -21,19 +21,19 @@ const char* syncRoleToStr(ESyncRole role) { switch (role) { case TAOS_SYNC_ROLE_VOTER: - return "voter"; + return "true"; case TAOS_SYNC_ROLE_LEARNER: - return "learner"; + return "false"; default: return "unknown"; } } const ESyncRole syncStrToRole(char* str) { - if(strcmp(str, "voter") == 0){ + if(strcmp(str, "true") == 0){ return TAOS_SYNC_ROLE_VOTER; } - if(strcmp(str, "learner") == 0){ + if(strcmp(str, "false") == 0){ return TAOS_SYNC_ROLE_LEARNER; } @@ -42,7 +42,6 @@ const ESyncRole syncStrToRole(char* str) { static int32_t syncEncodeSyncCfg(const void *pObj, SJson *pJson) { SSyncCfg *pCfg = (SSyncCfg *)pObj; - if (tjsonAddDoubleToObject(pJson, "totalReplicaNum", pCfg->totalReplicaNum) < 0) return -1; if (tjsonAddDoubleToObject(pJson, "replicaNum", pCfg->replicaNum) < 0) return -1; if (tjsonAddDoubleToObject(pJson, "myIndex", pCfg->myIndex) < 0) return -1; @@ -56,7 +55,7 @@ static int32_t syncEncodeSyncCfg(const void *pObj, SJson *pJson) { if (tjsonAddStringToObject(info, "nodeFqdn", pCfg->nodeInfo[i].nodeFqdn) < 0) return -1; if (tjsonAddIntegerToObject(info, "nodeId", pCfg->nodeInfo[i].nodeId) < 0) return -1; if (tjsonAddIntegerToObject(info, "clusterId", pCfg->nodeInfo[i].clusterId) < 0) return -1; - if (tjsonAddStringToObject(info, "nodeRole", syncRoleToStr(pCfg->nodeInfo[i].nodeRole)) < 0) return -1; + if (tjsonAddStringToObject(info, "isReplica", syncRoleToStr(pCfg->nodeInfo[i].nodeRole)) < 0) return -1; if (tjsonAddItemToArray(nodeInfo, info) < 0) return -1; } @@ -133,7 +132,6 @@ static int32_t syncDecodeSyncCfg(const SJson *pJson, void *pObj) { SSyncCfg *pCfg = (SSyncCfg *)pObj; int32_t code = 0; - tjsonGetInt32ValueFromDouble(pJson, "totalReplicaNum", pCfg->totalReplicaNum, code); tjsonGetInt32ValueFromDouble(pJson, "replicaNum", pCfg->replicaNum, code); if (code < 0) return -1; tjsonGetInt32ValueFromDouble(pJson, "myIndex", pCfg->myIndex, code); @@ -153,7 +151,7 @@ static int32_t syncDecodeSyncCfg(const SJson *pJson, void *pObj) { tjsonGetNumberValue(info, "nodeId", pCfg->nodeInfo[i].nodeId, code); tjsonGetNumberValue(info, "clusterId", pCfg->nodeInfo[i].clusterId, code); char role[10] = {0}; - code = tjsonGetStringValue(info, "nodeRole", role); + code = tjsonGetStringValue(info, "isReplica", role); if(code < 0) return -1; if(strlen(role) != 0){ pCfg->nodeInfo[i].nodeRole = syncStrToRole(role); From 12f28a4a4381790e281d669ca04fcbdd0ecfaca4 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Mon, 24 Apr 2023 17:37:38 +0800 Subject: [PATCH 44/61] enh: unify error msg for no disk space --- source/libs/executor/src/groupoperator.c | 4 ++-- source/libs/executor/src/timewindowoperator.c | 4 ++-- source/libs/executor/src/tlinearhash.c | 4 ++-- source/libs/executor/src/tsort.c | 9 ++++----- source/libs/function/src/builtinsimpl.c | 8 ++++++-- source/libs/function/src/tpercentile.c | 2 +- source/libs/function/src/udfd.c | 2 +- 7 files changed, 18 insertions(+), 15 deletions(-) diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index 3d9bacf39f..612ecb4684 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -871,9 +871,9 @@ SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartition getBufferPgSize(pInfo->binfo.pRes->info.rowSize, &defaultPgsz, &defaultBufsz); if (!osTempSpaceAvailable()) { - terrno = TSDB_CODE_NO_AVAIL_DISK; + terrno = TSDB_CODE_NO_DISKSPACE; pTaskInfo->code = terrno; - qError("Create partition operator info failed since %s", terrstr(terrno)); + qError("Create partition operator info failed since %s, tempDir:%s", terrstr(), tsTempDir); goto _error; } diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 007a6f63d1..7afda3f597 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -2911,8 +2911,8 @@ int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, SqlFunctionCtx* pCtx, bufSize = pageSize * 4; } if (!osTempSpaceAvailable()) { - terrno = TSDB_CODE_NO_AVAIL_DISK; - qError("Init stream agg supporter failed since %s", terrstr(terrno)); + terrno = TSDB_CODE_NO_DISKSPACE; + qError("Init stream agg supporter failed since %s, tempDir:%s", terrstr(), tsTempDir); return terrno; } int32_t code = createDiskbasedBuf(&pSup->pResultBuf, pageSize, bufSize, "function", tsTempDir); diff --git a/source/libs/executor/src/tlinearhash.c b/source/libs/executor/src/tlinearhash.c index 2cba3855c7..023583fcde 100644 --- a/source/libs/executor/src/tlinearhash.c +++ b/source/libs/executor/src/tlinearhash.c @@ -248,8 +248,8 @@ SLHashObj* tHashInit(int32_t inMemPages, int32_t pageSize, _hash_fn_t fn, int32_ } if (!osTempSpaceAvailable()) { - terrno = TSDB_CODE_NO_AVAIL_DISK; - printf("tHash Init failed since %s", terrstr(terrno)); + terrno = TSDB_CODE_NO_DISKSPACE; + printf("tHash Init failed since %s, tempDir:%s", terrstr(), tsTempDir); taosMemoryFree(pHashObj); return NULL; } diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c index 6c8e581b3f..da5f65fdf2 100644 --- a/source/libs/executor/src/tsort.c +++ b/source/libs/executor/src/tsort.c @@ -195,8 +195,8 @@ static int32_t doAddToBuf(SSDataBlock* pDataBlock, SSortHandle* pHandle) { if (pHandle->pBuf == NULL) { if (!osTempSpaceAvailable()) { - terrno = TSDB_CODE_NO_AVAIL_DISK; - qError("Add to buf failed since %s", terrstr(terrno)); + terrno = TSDB_CODE_NO_DISKSPACE; + qError("Add to buf failed since %s, tempDir:%s", terrstr(), tsTempDir); return terrno; } @@ -261,9 +261,8 @@ static int32_t sortComparInit(SMsortComparParam* pParam, SArray* pSources, int32 // multi-pass internal merge sort is required if (pHandle->pBuf == NULL) { if (!osTempSpaceAvailable()) { - code = TSDB_CODE_NO_AVAIL_DISK; - terrno = code; - qError("Sort compare init failed since %s, %s", tstrerror(code), pHandle->idStr); + code = terrno = TSDB_CODE_NO_DISKSPACE; + qError("Sort compare init failed since %s, tempDir:%s, idStr:%s", terrstr(), tsTempDir, pHandle->idStr); return code; } diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index a8ecd9b0a2..ead53aade6 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -855,7 +855,9 @@ int32_t setSelectivityValue(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, const STu int32_t numOfCols = pCtx->subsidiaries.num; const char* p = loadTupleData(pCtx, pTuplePos); if (p == NULL) { - terrno = TSDB_CODE_NO_AVAIL_DISK; + terrno = TSDB_CODE_NOT_FOUND; + qError("Load tuple data failed since %s, groupId:%" PRIu64 ", ts:%" PRId64, terrstr(), + pTuplePos->streamTupleKey.groupId, pTuplePos->streamTupleKey.ts); return terrno; } @@ -5098,7 +5100,9 @@ int32_t modeFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { if (maxCount != 0) { const char* pData = loadTupleData(pCtx, &resDataPos); if (pData == NULL) { - code = TSDB_CODE_NO_AVAIL_DISK; + code = terrno = TSDB_CODE_NOT_FOUND; + qError("Load tuple data failed since %s, groupId:%" PRIu64 ", ts:%" PRId64, terrstr(), + resDataPos.streamTupleKey.groupId, resDataPos.streamTupleKey.ts); modeFunctionCleanup(pInfo); return code; } diff --git a/source/libs/function/src/tpercentile.c b/source/libs/function/src/tpercentile.c index de381fadbd..2edfb33e81 100644 --- a/source/libs/function/src/tpercentile.c +++ b/source/libs/function/src/tpercentile.c @@ -277,7 +277,7 @@ tMemBucket *tMemBucketCreate(int16_t nElemSize, int16_t dataType, double minval, resetSlotInfo(pBucket); if (!osTempSpaceAvailable()) { - terrno = TSDB_CODE_NO_AVAIL_DISK; + terrno = TSDB_CODE_NO_DISKSPACE; // qError("MemBucket create disk based Buf failed since %s", terrstr(terrno)); tMemBucketDestroy(pBucket); return NULL; diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index 5034af2f82..d2553ba96d 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -844,7 +844,7 @@ void udfdGetFuncBodyPath(const SUdf *udf, char *path) { int32_t udfdSaveFuncBodyToFile(SFuncInfo *pFuncInfo, SUdf *udf) { if (!osDataSpaceAvailable()) { - terrno = TSDB_CODE_NO_AVAIL_DISK; + terrno = TSDB_CODE_NO_DISKSPACE; fnError("udfd create shared library failed since %s", terrstr(terrno)); return terrno; } From 7e2dee8a0f5d7a7c8e10e66a3d43d1a803966593 Mon Sep 17 00:00:00 2001 From: cadem Date: Tue, 25 Apr 2023 15:49:17 +0800 Subject: [PATCH 45/61] filter voter when agree upon --- source/libs/sync/src/syncCommit.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/source/libs/sync/src/syncCommit.c b/source/libs/sync/src/syncCommit.c index 2501b4df8b..d3397f8e88 100644 --- a/source/libs/sync/src/syncCommit.c +++ b/source/libs/sync/src/syncCommit.c @@ -75,10 +75,12 @@ bool syncNodeAgreedUpon(SSyncNode* pNode, SyncIndex index) { SSyncIndexMgr* pMatches = pNode->pMatchIndex; ASSERT(pNode->replicaNum == pMatches->replicaNum); - for (int i = 0; i < pNode->replicaNum; i++) { - SyncIndex matchIndex = pMatches->index[i]; - if (matchIndex >= index) { - count++; + for (int i = 0; i < pNode->totalReplicaNum; i++) { + if(pNode->raftCfg.cfg.nodeInfo[i].nodeRole == TAOS_SYNC_ROLE_VOTER){ + SyncIndex matchIndex = pMatches->index[i]; + if (matchIndex >= index) { + count++; + } } } From f720feb56d7a20399b6862d76e5813e98637a94f Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 25 Apr 2023 16:58:00 +0800 Subject: [PATCH 46/61] Update 14-stream.md --- docs/zh/12-taos-sql/14-stream.md | 46 ++++++++++++++++---------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/docs/zh/12-taos-sql/14-stream.md b/docs/zh/12-taos-sql/14-stream.md index 9bbb551c87..237d39104f 100644 --- a/docs/zh/12-taos-sql/14-stream.md +++ b/docs/zh/12-taos-sql/14-stream.md @@ -216,26 +216,26 @@ T = 最新事件时间 - DELETE_MARK ## 流式计算支持的函数 -1. 所有的单行函数均可用于流计算中。 -2. 以下 19 个聚合函数不能在创建流计算的 SQL 语句中使用 -``` -leastsquares -percentile -top -bottom -elapsed -interp -derivative -irate -twa -histogram -diff -statecount -stateduration -csum -mavg -sample -tail -unique -mode -``` +1. 所有的[单行函数](../function/#单行函数)均可用于流计算。 +2. 以下 19 个聚合/选择函数不能在创建流计算的 SQL 语句，[系统信息函数](../function/#系统信息函数)其他类型的聚合函数均可用于流计算。 + +- [leastsquares](../function/#leastsquares) +- [percentile](../function/#percentile) +- [top](../function/#leastsquares) +- [bottom](../function/#top) +- [elapsed](../function/#leastsquares) +- [interp](../function/#elapsed) +- [derivative](../function/#derivative) +- [irate](../function/#irate) +- [twa](../function/#twa) +- [histogram](../function/#histogram) +- [diff](../function/#diff) +- [statecount](../function/#statecount) +- [stateduration](../function/#stateduration) +- [csum](../function/#csum) +- [mavg](../function/#mavg) +- [sample](../function/#sample) +- [tail](../function/#tail) +- [unique](../function/#unique) +- [mode](../function/#mode) + From 19976122f4e602dcd6a14094079806214353cb8c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 25 Apr 2023 17:00:03 +0800 Subject: [PATCH 47/61] Update 14-stream.md --- docs/zh/12-taos-sql/14-stream.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/zh/12-taos-sql/14-stream.md b/docs/zh/12-taos-sql/14-stream.md index 237d39104f..b08804e5c4 100644 --- a/docs/zh/12-taos-sql/14-stream.md +++ b/docs/zh/12-taos-sql/14-stream.md @@ -216,8 +216,8 @@ T = 最新事件时间 - DELETE_MARK ## 流式计算支持的函数 -1. 所有的[单行函数](../function/#单行函数)均可用于流计算。 -2. 以下 19 个聚合/选择函数不能在创建流计算的 SQL 语句，[系统信息函数](../function/#系统信息函数)其他类型的聚合函数均可用于流计算。 +1. 所有的 [单行函数](../function/#单行函数) 均可用于流计算。 +2. 以下 19 个聚合/选择函数 不能 应用在创建流计算的 SQL 语句，[系统信息函数](../function/#系统信息函数) 也不能用于流计算中。此外的其他类型的函数均可用于流计算。 - [leastsquares](../function/#leastsquares) - [percentile](../function/#percentile) From 693bc09f193b5255cba8b8421856deafe5b08a10 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Tue, 25 Apr 2023 17:36:13 +0800 Subject: [PATCH 48/61] enh: comment off unused error codes in sync and wal --- include/util/taoserror.h | 18 +++++++++--------- source/libs/sync/src/syncMain.c | 4 +--- source/util/src/terror.c | 7 ------- 3 files changed, 10 insertions(+), 19 deletions(-) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 142427ca6b..9cb0f5e564 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -539,17 +539,17 @@ int32_t* taosGetErrno(); // #define TSDB_CODE_SYN_INVALID_CHECKSUM TAOS_DEF_ERROR_CODE(0, 0x0908) // 2.x // #define TSDB_CODE_SYN_INVALID_MSGLEN TAOS_DEF_ERROR_CODE(0, 0x0909) // 2.x // #define TSDB_CODE_SYN_INVALID_MSGTYPE TAOS_DEF_ERROR_CODE(0, 0x090A) // 2.x -#define TSDB_CODE_SYN_IS_LEADER TAOS_DEF_ERROR_CODE(0, 0x090B) +// #define TSDB_CODE_SYN_IS_LEADER TAOS_DEF_ERROR_CODE(0, 0x090B) // unused #define TSDB_CODE_SYN_NOT_LEADER TAOS_DEF_ERROR_CODE(0, 0x090C) -#define TSDB_CODE_SYN_ONE_REPLICA TAOS_DEF_ERROR_CODE(0, 0x090D) -#define TSDB_CODE_SYN_NOT_IN_NEW_CONFIG TAOS_DEF_ERROR_CODE(0, 0x090E) -#define TSDB_CODE_SYN_NEW_CONFIG_ERROR TAOS_DEF_ERROR_CODE(0, 0x090F) // internal -#define TSDB_CODE_SYN_RECONFIG_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0910) +// #define TSDB_CODE_SYN_ONE_REPLICA TAOS_DEF_ERROR_CODE(0, 0x090D) // unused +// #define TSDB_CODE_SYN_NOT_IN_NEW_CONFIG TAOS_DEF_ERROR_CODE(0, 0x090E) // unused +#define TSDB_CODE_SYN_NEW_CONFIG_ERROR TAOS_DEF_ERROR_CODE(0, 0x090F) // internal +// #define TSDB_CODE_SYN_RECONFIG_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0910) // unused #define TSDB_CODE_SYN_PROPOSE_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0911) -#define TSDB_CODE_SYN_STANDBY_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0912) -#define TSDB_CODE_SYN_BATCH_ERROR TAOS_DEF_ERROR_CODE(0, 0x0913) +// #define TSDB_CODE_SYN_STANDBY_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0912) // unused +// #define TSDB_CODE_SYN_BATCH_ERROR TAOS_DEF_ERROR_CODE(0, 0x0913) // unused #define TSDB_CODE_SYN_RESTORING TAOS_DEF_ERROR_CODE(0, 0x0914) -#define TSDB_CODE_SYN_INVALID_SNAPSHOT_MSG TAOS_DEF_ERROR_CODE(0, 0x0915) // internal +#define TSDB_CODE_SYN_INVALID_SNAPSHOT_MSG TAOS_DEF_ERROR_CODE(0, 0x0915) // internal #define TSDB_CODE_SYN_BUFFER_FULL TAOS_DEF_ERROR_CODE(0, 0x0916) #define TSDB_CODE_SYN_WRITE_STALL TAOS_DEF_ERROR_CODE(0, 0x0917) #define TSDB_CODE_SYN_NEGO_WIN_EXCEEDED TAOS_DEF_ERROR_CODE(0, 0X0918) @@ -573,7 +573,7 @@ int32_t* taosGetErrno(); // wal // #define TSDB_CODE_WAL_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x1000) // 2.x #define TSDB_CODE_WAL_FILE_CORRUPTED TAOS_DEF_ERROR_CODE(0, 0x1001) -#define TSDB_CODE_WAL_SIZE_LIMIT TAOS_DEF_ERROR_CODE(0, 0x1002) +// #define TSDB_CODE_WAL_SIZE_LIMIT TAOS_DEF_ERROR_CODE(0, 0x1002) // unused #define TSDB_CODE_WAL_INVALID_VER TAOS_DEF_ERROR_CODE(0, 0x1003) // #define TSDB_CODE_WAL_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x1004) // 2.x #define TSDB_CODE_WAL_LOG_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x1005) diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index cb1919d16e..499df4a98b 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -463,8 +463,7 @@ bool syncSnapshotRecving(int64_t rid) { int32_t syncNodeLeaderTransfer(SSyncNode* pSyncNode) { if (pSyncNode->peersNum == 0) { sDebug("vgId:%d, only one replica, cannot leader transfer", pSyncNode->vgId); - terrno = TSDB_CODE_SYN_ONE_REPLICA; - return -1; + return 0; } int32_t ret = 0; @@ -486,7 +485,6 @@ int32_t syncNodeLeaderTransfer(SSyncNode* pSyncNode) { int32_t syncNodeLeaderTransferTo(SSyncNode* pSyncNode, SNodeInfo newLeader) { if (pSyncNode->replicaNum == 1) { sDebug("vgId:%d, only one replica, cannot leader transfer", pSyncNode->vgId); - terrno = TSDB_CODE_SYN_ONE_REPLICA; return -1; } diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 1d9d45f26f..2ef3c23f0a 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -414,15 +414,9 @@ TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_TABLE_LIMITED, "Table creation limite // sync TAOS_DEFINE_ERROR(TSDB_CODE_SYN_TIMEOUT, "Sync timeout") -TAOS_DEFINE_ERROR(TSDB_CODE_SYN_IS_LEADER, "Sync is leader") TAOS_DEFINE_ERROR(TSDB_CODE_SYN_NOT_LEADER, "Sync leader is unreachable") -TAOS_DEFINE_ERROR(TSDB_CODE_SYN_ONE_REPLICA, "Sync one replica") -TAOS_DEFINE_ERROR(TSDB_CODE_SYN_NOT_IN_NEW_CONFIG, "Sync not in new config") TAOS_DEFINE_ERROR(TSDB_CODE_SYN_NEW_CONFIG_ERROR, "Sync new config error") -TAOS_DEFINE_ERROR(TSDB_CODE_SYN_RECONFIG_NOT_READY, "Sync not ready for reconfig") TAOS_DEFINE_ERROR(TSDB_CODE_SYN_PROPOSE_NOT_READY, "Sync not ready for propose") -TAOS_DEFINE_ERROR(TSDB_CODE_SYN_STANDBY_NOT_READY, "Sync not ready for standby") -TAOS_DEFINE_ERROR(TSDB_CODE_SYN_BATCH_ERROR, "Sync batch error") TAOS_DEFINE_ERROR(TSDB_CODE_SYN_RESTORING, "Sync leader is restoring") TAOS_DEFINE_ERROR(TSDB_CODE_SYN_INVALID_SNAPSHOT_MSG, "Sync invalid snapshot msg") TAOS_DEFINE_ERROR(TSDB_CODE_SYN_BUFFER_FULL, "Sync buffer is full") @@ -445,7 +439,6 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TQ_NO_COMMITTED_OFFSET, "TQ no committed offse // wal TAOS_DEFINE_ERROR(TSDB_CODE_WAL_FILE_CORRUPTED, "WAL file is corrupted") -TAOS_DEFINE_ERROR(TSDB_CODE_WAL_SIZE_LIMIT, "WAL size exceeds limit") TAOS_DEFINE_ERROR(TSDB_CODE_WAL_INVALID_VER, "WAL use invalid version") TAOS_DEFINE_ERROR(TSDB_CODE_WAL_LOG_NOT_EXIST, "WAL log not exist") TAOS_DEFINE_ERROR(TSDB_CODE_WAL_CHKSUM_MISMATCH, "WAL checksum mismatch") From fee4050d79c24bd6b65b3d15218a428641dead67 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Tue, 25 Apr 2023 18:05:30 +0800 Subject: [PATCH 49/61] enh: rename error code TSDB_CODE_SYN_NEGO_WIN_EXCEEDED --- include/util/taoserror.h | 2 +- source/libs/sync/src/syncPipeline.c | 2 +- source/util/src/terror.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 9cb0f5e564..a709ccf10c 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -552,7 +552,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_SYN_INVALID_SNAPSHOT_MSG TAOS_DEF_ERROR_CODE(0, 0x0915) // internal #define TSDB_CODE_SYN_BUFFER_FULL TAOS_DEF_ERROR_CODE(0, 0x0916) #define TSDB_CODE_SYN_WRITE_STALL TAOS_DEF_ERROR_CODE(0, 0x0917) -#define TSDB_CODE_SYN_NEGO_WIN_EXCEEDED TAOS_DEF_ERROR_CODE(0, 0X0918) +#define TSDB_CODE_SYN_NEGOTIATION_WIN_FULL TAOS_DEF_ERROR_CODE(0, 0x0918) #define TSDB_CODE_SYN_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x09FF) // tq diff --git a/source/libs/sync/src/syncPipeline.c b/source/libs/sync/src/syncPipeline.c index 85c18220a6..8bb72de518 100644 --- a/source/libs/sync/src/syncPipeline.c +++ b/source/libs/sync/src/syncPipeline.c @@ -54,7 +54,7 @@ int32_t syncLogBufferAppend(SSyncLogBuffer* pBuf, SSyncNode* pNode, SSyncRaftEnt } if (pNode->restoreFinish && index - pBuf->commitIndex >= TSDB_SYNC_NEGOTIATION_WIN) { - terrno = TSDB_CODE_SYN_NEGO_WIN_EXCEEDED; + terrno = TSDB_CODE_SYN_NEGOTIATION_WIN_FULL; sError("vgId:%d, failed to append since %s, index:%" PRId64 ", commit-index:%" PRId64, pNode->vgId, terrstr(), index, pBuf->commitIndex); goto _err; diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 2ef3c23f0a..c1ae901203 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -421,7 +421,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_SYN_RESTORING, "Sync leader is restor TAOS_DEFINE_ERROR(TSDB_CODE_SYN_INVALID_SNAPSHOT_MSG, "Sync invalid snapshot msg") TAOS_DEFINE_ERROR(TSDB_CODE_SYN_BUFFER_FULL, "Sync buffer is full") TAOS_DEFINE_ERROR(TSDB_CODE_SYN_WRITE_STALL, "Sync write stall") -TAOS_DEFINE_ERROR(TSDB_CODE_SYN_NEGO_WIN_EXCEEDED, "Sync negotiation win exceeded") +TAOS_DEFINE_ERROR(TSDB_CODE_SYN_NEGOTIATION_WIN_FULL, "Sync negotiation win is full") TAOS_DEFINE_ERROR(TSDB_CODE_SYN_INTERNAL_ERROR, "Sync internal error") //tq From 23a915da3bc95c5313efbc3fae9a230469f1afb4 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Tue, 25 Apr 2023 19:11:42 +0800 Subject: [PATCH 50/61] enh: adjust error msgs in sync and wal --- source/util/src/terror.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/util/src/terror.c b/source/util/src/terror.c index c1ae901203..6fc16ad4a9 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -416,7 +416,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_TABLE_LIMITED, "Table creation limite TAOS_DEFINE_ERROR(TSDB_CODE_SYN_TIMEOUT, "Sync timeout") TAOS_DEFINE_ERROR(TSDB_CODE_SYN_NOT_LEADER, "Sync leader is unreachable") TAOS_DEFINE_ERROR(TSDB_CODE_SYN_NEW_CONFIG_ERROR, "Sync new config error") -TAOS_DEFINE_ERROR(TSDB_CODE_SYN_PROPOSE_NOT_READY, "Sync not ready for propose") +TAOS_DEFINE_ERROR(TSDB_CODE_SYN_PROPOSE_NOT_READY, "Sync not ready to propose") TAOS_DEFINE_ERROR(TSDB_CODE_SYN_RESTORING, "Sync leader is restoring") TAOS_DEFINE_ERROR(TSDB_CODE_SYN_INVALID_SNAPSHOT_MSG, "Sync invalid snapshot msg") TAOS_DEFINE_ERROR(TSDB_CODE_SYN_BUFFER_FULL, "Sync buffer is full") @@ -439,7 +439,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TQ_NO_COMMITTED_OFFSET, "TQ no committed offse // wal TAOS_DEFINE_ERROR(TSDB_CODE_WAL_FILE_CORRUPTED, "WAL file is corrupted") -TAOS_DEFINE_ERROR(TSDB_CODE_WAL_INVALID_VER, "WAL use invalid version") +TAOS_DEFINE_ERROR(TSDB_CODE_WAL_INVALID_VER, "WAL invalid version") TAOS_DEFINE_ERROR(TSDB_CODE_WAL_LOG_NOT_EXIST, "WAL log not exist") TAOS_DEFINE_ERROR(TSDB_CODE_WAL_CHKSUM_MISMATCH, "WAL checksum mismatch") TAOS_DEFINE_ERROR(TSDB_CODE_WAL_LOG_INCOMPLETE, "WAL log incomplete") From 519dc0ead0e04fae914aa5d4f791e4424c170a73 Mon Sep 17 00:00:00 2001 From: Adam Ji Date: Wed, 26 Apr 2023 09:24:29 +0800 Subject: [PATCH 51/61] docs: fix tmq create database args (#21081) --- docs/en/07-develop/07-tmq.mdx | 2 +- docs/zh/07-develop/07-tmq.mdx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/07-develop/07-tmq.mdx b/docs/en/07-develop/07-tmq.mdx index 7465cc0a12..739eabd1cf 100644 --- a/docs/en/07-develop/07-tmq.mdx +++ b/docs/en/07-develop/07-tmq.mdx @@ -222,7 +222,7 @@ A database including one supertable and two subtables is created as follows: ```sql DROP DATABASE IF EXISTS tmqdb; -CREATE DATABASE tmqdb; +CREATE DATABASE tmqdb WAL_RETENTION_PERIOD 3600; CREATE TABLE tmqdb.stb (ts TIMESTAMP, c1 INT, c2 FLOAT, c3 VARCHAR(16)) TAGS(t1 INT, t3 VARCHAR(16)); CREATE TABLE tmqdb.ctb0 USING tmqdb.stb TAGS(0, "subtable0"); CREATE TABLE tmqdb.ctb1 USING tmqdb.stb TAGS(1, "subtable1"); diff --git a/docs/zh/07-develop/07-tmq.mdx b/docs/zh/07-develop/07-tmq.mdx index 036c9aa86e..a3833438a2 100644 --- a/docs/zh/07-develop/07-tmq.mdx +++ b/docs/zh/07-develop/07-tmq.mdx @@ -221,7 +221,7 @@ void Close() ```sql DROP DATABASE IF EXISTS tmqdb; -CREATE DATABASE tmqdb; +CREATE DATABASE tmqdb WAL_RETENTION_PERIOD 3600; CREATE TABLE tmqdb.stb (ts TIMESTAMP, c1 INT, c2 FLOAT, c3 VARCHAR(16)) TAGS(t1 INT, t3 VARCHAR(16)); CREATE TABLE tmqdb.ctb0 USING tmqdb.stb TAGS(0, "subtable0"); CREATE TABLE tmqdb.ctb1 USING tmqdb.stb TAGS(1, "subtable1"); From 41cc572cd8b8fff0915b432e6bed69aa50924f8d Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 26 Apr 2023 12:41:26 +0800 Subject: [PATCH 52/61] docs: update connector matrix for 3.0 (#21086) * docs: update csharp connector status * docs: fix csharp ws bulk pulling * docs: clarify database param is optional to websocket dsn * docs: fix java connector mistake * fix: a few typos * fix: many typos * docs: java connector support subscription over websocket * fix: update 3.0 connector feature matrix --- docs/en/14-reference/03-connector/index.mdx | 2 +- docs/zh/08-connector/index.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/14-reference/03-connector/index.mdx b/docs/en/14-reference/03-connector/index.mdx index a35d5bc2d1..28b7b83b58 100644 --- a/docs/en/14-reference/03-connector/index.mdx +++ b/docs/en/14-reference/03-connector/index.mdx @@ -62,7 +62,7 @@ The different database framework specifications for various programming language | **Regular Query** | Support | Support | Support | Support | Support | Support | | **Parameter Binding** | Not Supported | Not Supported | Support | Support | Not Supported | Support | | **Subscription (TMQ) ** | Supported | Support | Support | Not Supported | Not Supported | Support | -| **Schemaless** | Not Supported | Not Supported | Not Supported | Not Supported | Not Supported | Not Supported | +| **Schemaless** | Supported | Not Supported | Not Supported | Not Supported | Not Supported | Not Supported | | **Bulk Pulling (based on WebSocket) ** | Support | Support | Support | Support | Support | Support | | **DataFrame** | Not Supported | Support | Not Supported | Not Supported | Not Supported | Not Supported | diff --git a/docs/zh/08-connector/index.md b/docs/zh/08-connector/index.md index eb1f3a9a9a..bb8c95a15a 100644 --- a/docs/zh/08-connector/index.md +++ b/docs/zh/08-connector/index.md @@ -61,7 +61,7 @@ TDengine 版本更新往往会增加新的功能特性，列表中的连接器 | **普通查询** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | | **参数绑定** | 暂不支持 | 暂不支持 | 支持 | 支持 | 暂不支持 | 支持 | | **数据订阅（TMQ）** | 支持 | 支持 | 支持 | 暂不支持 | 暂不支持 | 支持 | -| **Schemaless** | 暂不支持 | 暂不支持 | 暂不支持 | 暂不支持 | 暂不支持 | 暂不支持 | +| **Schemaless** | 支持 | 暂不支持 | 暂不支持 | 暂不支持 | 暂不支持 | 暂不支持 | | **批量拉取（基于 WebSocket）** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | | **DataFrame** | 不支持 | 支持 | 不支持 | 不支持 | 不支持 | 不支持 | From 3822a5858b05483f9abbf058bd2d8dbdcce411e0 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Wed, 26 Apr 2023 16:03:37 +0800 Subject: [PATCH 53/61] sim/valgrind: suppress rocks reachables --- tests/script/local.supp | 227 ++++++++++++++++++++++++++++++++++++++++ tests/script/sh/exec.sh | 6 +- 2 files changed, 231 insertions(+), 2 deletions(-) create mode 100644 tests/script/local.supp diff --git a/tests/script/local.supp b/tests/script/local.supp new file mode 100644 index 0000000000..562cddba46 --- /dev/null +++ b/tests/script/local.supp @@ -0,0 +1,227 @@ +{ + + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + fun:_ZN9__gnu_cxx13new_allocatorINSt8__detail10_Hash_nodeISt4pairIKjPFvPvEELb0EEEE8allocateEmPKv + fun:_ZNSt16allocator_traitsISaINSt8__detail10_Hash_nodeISt4pairIKjPFvPvEELb0EEEEE8allocateERS9_m + fun:_ZNSt8__detail16_Hashtable_allocISaINS_10_Hash_nodeISt4pairIKjPFvPvEELb0EEEEE16_M_allocate_nodeIJRKSt21piecewise_construct_tSt5tupleIJRS3_EESF_IJEEEEEPS8_DpOT_ + fun:_ZNSt10_HashtableIjSt4pairIKjPFvPvEESaIS5_ENSt8__detail10_Select1stESt8equal_toIjESt4hashIjENS7_18_Mod_range_hashingENS7_20_Default_ranged_hashENS7_20_Prime_rehash_policyENS7_17_Hashtable_traitsILb0ELb0ELb1EEEE12_Scoped_nodeC1IJRKSt21piecewise_construct_tSt5tupleIJRS1_EESO_IJEEEEEPNS7_16_Hashtable_allocISaINS7_10_Hash_nodeIS5_Lb0EEEEEEDpOT_ + fun:_ZNSt8__detail9_Map_baseIjSt4pairIKjPFvPvEESaIS6_ENS_10_Select1stESt8equal_toIjESt4hashIjENS_18_Mod_range_hashingENS_20_Default_ranged_hashENS_20_Prime_rehash_policyENS_17_Hashtable_traitsILb0ELb0ELb1EEELb1EEixERS2_ + fun:_ZNSt13unordered_mapIjPFvPvESt4hashIjESt8equal_toIjESaISt4pairIKjS2_EEEixERS8_ + fun:_ZN7rocksdb14ThreadLocalPtr10StaticMeta10SetHandlerEjPFvPvE + fun:_ZN7rocksdb14ThreadLocalPtrC1EPFvPvE + fun:_ZN7rocksdb16ColumnFamilyDataC1EjRKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEPNS_7VersionEPNS_5CacheEPNS_18WriteBufferManagerERKNS_19ColumnFamilyOptionsERKNS_18ImmutableDBOptionsERKNS_11FileOptionsEPNS_15ColumnFamilySetEPNS_16BlockCacheTracerERKSt10shared_ptrINS_8IOTracerEES8_ + fun:_ZN7rocksdb15ColumnFamilySetC1ERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEPKNS_18ImmutableDBOptionsERKNS_11FileOptionsEPNS_5CacheEPNS_18WriteBufferManagerEPNS_15WriteControllerEPNS_16BlockCacheTracerERKSt10shared_ptrINS_8IOTracerEES8_ + fun:_ZN7rocksdb10VersionSetC1ERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEPKNS_18ImmutableDBOptionsERKNS_11FileOptionsEPNS_5CacheEPNS_18WriteBufferManagerEPNS_15WriteControllerEPNS_16BlockCacheTracerERKSt10shared_ptrINS_8IOTracerEES8_ + fun:_ZN7rocksdb6DBImplC1ERKNS_9DBOptionsERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEbbb + fun:_ZN7rocksdb6DBImpl4OpenERKNS_9DBOptionsERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEERKSt6vectorINS_22ColumnFamilyDescriptorESaISD_EEPSC_IPNS_18ColumnFamilyHandleESaISJ_EEPPNS_2DBEbb + fun:_ZN7rocksdb2DB4OpenERKNS_9DBOptionsERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEERKSt6vectorINS_22ColumnFamilyDescriptorESaISD_EEPSC_IPNS_18ColumnFamilyHandleESaISJ_EEPPS0_ + fun:_ZN7rocksdb2DB4OpenERKNS_7OptionsERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEPPS0_ + fun:rocksdb_open + fun:tsdbOpenRocksCache + fun:tsdbOpenCache + fun:tsdbOpen +} +{ + + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + fun:_ZN9__gnu_cxx13new_allocatorINSt8__detail10_Hash_nodeISt4pairIKjPFvPvEELb0EEEE8allocateEmPKv + fun:_ZNSt16allocator_traitsISaINSt8__detail10_Hash_nodeISt4pairIKjPFvPvEELb0EEEEE8allocateERS9_m + fun:_ZNSt8__detail16_Hashtable_allocISaINS_10_Hash_nodeISt4pairIKjPFvPvEELb0EEEEE16_M_allocate_nodeIJRKSt21piecewise_construct_tSt5tupleIJRS3_EESF_IJEEEEEPS8_DpOT_ + fun:_ZNSt10_HashtableIjSt4pairIKjPFvPvEESaIS5_ENSt8__detail10_Select1stESt8equal_toIjESt4hashIjENS7_18_Mod_range_hashingENS7_20_Default_ranged_hashENS7_20_Prime_rehash_policyENS7_17_Hashtable_traitsILb0ELb0ELb1EEEE12_Scoped_nodeC1IJRKSt21piecewise_construct_tSt5tupleIJRS1_EESO_IJEEEEEPNS7_16_Hashtable_allocISaINS7_10_Hash_nodeIS5_Lb0EEEEEEDpOT_ + fun:_ZNSt8__detail9_Map_baseIjSt4pairIKjPFvPvEESaIS6_ENS_10_Select1stESt8equal_toIjESt4hashIjENS_18_Mod_range_hashingENS_20_Default_ranged_hashENS_20_Prime_rehash_policyENS_17_Hashtable_traitsILb0ELb0ELb1EEELb1EEixERS2_ + fun:_ZNSt13unordered_mapIjPFvPvESt4hashIjESt8equal_toIjESaISt4pairIKjS2_EEEixERS8_ + fun:_ZN7rocksdb14ThreadLocalPtr10StaticMeta10SetHandlerEjPFvPvE + fun:_ZN7rocksdb14ThreadLocalPtrC1EPFvPvE + fun:_ZN7rocksdb16ColumnFamilyDataC1EjRKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEPNS_7VersionEPNS_5CacheEPNS_18WriteBufferManagerERKNS_19ColumnFamilyOptionsERKNS_18ImmutableDBOptionsERKNS_11FileOptionsEPNS_15ColumnFamilySetEPNS_16BlockCacheTracerERKSt10shared_ptrINS_8IOTracerEES8_ + fun:_ZN7rocksdb15ColumnFamilySet18CreateColumnFamilyERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEjPNS_7VersionERKNS_19ColumnFamilyOptionsE + fun:_ZN7rocksdb10VersionSet18CreateColumnFamilyERKNS_19ColumnFamilyOptionsEPKNS_11VersionEditE + fun:_ZN7rocksdb18VersionEditHandler15CreateCfAndInitERKNS_19ColumnFamilyOptionsERKNS_11VersionEditE + fun:_ZN7rocksdb18VersionEditHandler10InitializeEv + fun:_ZN7rocksdb22VersionEditHandlerBase7IterateERNS_3log6ReaderEPNS_6StatusE + fun:_ZN7rocksdb10VersionSet7RecoverERKSt6vectorINS_22ColumnFamilyDescriptorESaIS2_EEbPNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEE + fun:_ZN7rocksdb6DBImpl7RecoverERKSt6vectorINS_22ColumnFamilyDescriptorESaIS2_EEbbbPm + fun:_ZN7rocksdb6DBImpl4OpenERKNS_9DBOptionsERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEERKSt6vectorINS_22ColumnFamilyDescriptorESaISD_EEPSC_IPNS_18ColumnFamilyHandleESaISJ_EEPPNS_2DBEbb + fun:_ZN7rocksdb2DB4OpenERKNS_9DBOptionsERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEERKSt6vectorINS_22ColumnFamilyDescriptorESaISD_EEPSC_IPNS_18ColumnFamilyHandleESaISJ_EEPPS0_ + fun:_ZN7rocksdb2DB4OpenERKNS_7OptionsERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEPPS0_ +} +{ + + Memcheck:Leak + match-leak-kinds: reachable + fun:calloc + fun:__cxa_thread_atexit_impl + fun:__cxa_thread_atexit + fun:__tls_init + fun:_ZTWN7rocksdb12perf_contextE + fun:_ZN7rocksdb17InstrumentedMutex4LockEv + fun:_ZN7rocksdb21InstrumentedMutexLockC1EPNS_17InstrumentedMutexE + fun:_ZN7rocksdb5Timer8ShutdownEv + fun:_ZN7rocksdb5TimerD1Ev + fun:_ZNKSt14default_deleteIN7rocksdb5TimerEEclEPS1_ + fun:_ZNSt10unique_ptrIN7rocksdb5TimerESt14default_deleteIS1_EED1Ev + fun:_ZN7rocksdb21PeriodicWorkSchedulerD1Ev + fun:__run_exit_handlers + fun:exit + fun:(below main) +} +{ + + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + fun:_ZN7rocksdb24CacheEntryStatsCollectorINS_13InternalStats19CacheEntryRoleStatsEE9GetSharedEPNS_5CacheEPNS_11SystemClockEPSt10shared_ptrIS3_E + fun:_ZN7rocksdb13InternalStatsC1EiPNS_11SystemClockEPNS_16ColumnFamilyDataE + fun:_ZN7rocksdb16ColumnFamilyDataC1EjRKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEPNS_7VersionEPNS_5CacheEPNS_18WriteBufferManagerERKNS_19ColumnFamilyOptionsERKNS_18ImmutableDBOptionsERKNS_11FileOptionsEPNS_15ColumnFamilySetEPNS_16BlockCacheTracerERKSt10shared_ptrINS_8IOTracerEES8_ + fun:_ZN7rocksdb15ColumnFamilySet18CreateColumnFamilyERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEjPNS_7VersionERKNS_19ColumnFamilyOptionsE + fun:_ZN7rocksdb10VersionSet18CreateColumnFamilyERKNS_19ColumnFamilyOptionsEPKNS_11VersionEditE + fun:_ZN7rocksdb18VersionEditHandler15CreateCfAndInitERKNS_19ColumnFamilyOptionsERKNS_11VersionEditE + fun:_ZN7rocksdb18VersionEditHandler10InitializeEv + fun:_ZN7rocksdb22VersionEditHandlerBase7IterateERNS_3log6ReaderEPNS_6StatusE + fun:_ZN7rocksdb10VersionSet7RecoverERKSt6vectorINS_22ColumnFamilyDescriptorESaIS2_EEbPNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEE + fun:_ZN7rocksdb6DBImpl7RecoverERKSt6vectorINS_22ColumnFamilyDescriptorESaIS2_EEbbbPm + fun:_ZN7rocksdb6DBImpl4OpenERKNS_9DBOptionsERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEERKSt6vectorINS_22ColumnFamilyDescriptorESaISD_EEPSC_IPNS_18ColumnFamilyHandleESaISJ_EEPPNS_2DBEbb + fun:_ZN7rocksdb2DB4OpenERKNS_9DBOptionsERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEERKSt6vectorINS_22ColumnFamilyDescriptorESaISD_EEPSC_IPNS_18ColumnFamilyHandleESaISJ_EEPPS0_ + fun:_ZN7rocksdb2DB4OpenERKNS_7OptionsERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEPPS0_ + fun:rocksdb_open + fun:tsdbOpenRocksCache + fun:tsdbOpenCache + fun:tsdbOpen + fun:vnodeOpen + fun:vmProcessCreateVnodeReq +} +{ + + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + fun:_ZN7rocksdb12_GLOBAL__N_111GetRegistryEv + fun:_ZN7rocksdb23CopyCacheDeleterRoleMapEv + fun:_ZN7rocksdb13InternalStats19CacheEntryRoleStats15BeginCollectionEPNS_5CacheEPNS_11SystemClockEm + fun:_ZN7rocksdb24CacheEntryStatsCollectorINS_13InternalStats19CacheEntryRoleStatsEE12CollectStatsEii + fun:_ZN7rocksdb13InternalStats22CollectCacheEntryStatsEb + fun:_ZN7rocksdb6DBImpl9DumpStatsEv + fun:_ZZN7rocksdb21PeriodicWorkScheduler8RegisterEPNS_6DBImplEjjENKUlvE_clEv + fun:_ZSt13__invoke_implIvRZN7rocksdb21PeriodicWorkScheduler8RegisterEPNS0_6DBImplEjjEUlvE_JEET_St14__invoke_otherOT0_DpOT1_ + fun:_ZSt10__invoke_rIvRZN7rocksdb21PeriodicWorkScheduler8RegisterEPNS0_6DBImplEjjEUlvE_JEENSt9enable_ifIXsrSt6__and_IJSt7is_voidIT_ESt14__is_invocableIT0_JDpT1_EEEE5valueES9_E4typeEOSC_DpOSD_ + fun:_ZNSt17_Function_handlerIFvvEZN7rocksdb21PeriodicWorkScheduler8RegisterEPNS1_6DBImplEjjEUlvE_E9_M_invokeERKSt9_Any_data + fun:_ZNKSt8functionIFvvEEclEv + fun:_ZN7rocksdb5Timer3RunEv + fun:_ZSt13__invoke_implIvMN7rocksdb5TimerEFvvEPS1_JEET_St21__invoke_memfun_derefOT0_OT1_DpOT2_ + fun:_ZSt8__invokeIMN7rocksdb5TimerEFvvEJPS1_EENSt15__invoke_resultIT_JDpT0_EE4typeEOS6_DpOS7_ + fun:_ZNSt6thread8_InvokerISt5tupleIJMN7rocksdb5TimerEFvvEPS3_EEE9_M_invokeIJLm0ELm1EEEEvSt12_Index_tupleIJXspT_EEE + fun:_ZNSt6thread8_InvokerISt5tupleIJMN7rocksdb5TimerEFvvEPS3_EEEclEv + fun:_ZNSt6thread11_State_implINS_8_InvokerISt5tupleIJMN7rocksdb5TimerEFvvEPS4_EEEEE6_M_runEv + obj:/usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.28 + fun:start_thread +} +{ + + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + fun:_ZN9__gnu_cxx13new_allocatorIPNSt8__detail15_Hash_node_baseEE8allocateEmPKv + fun:_ZNSt16allocator_traitsISaIPNSt8__detail15_Hash_node_baseEEE8allocateERS3_m + fun:_ZNSt8__detail16_Hashtable_allocISaINS_10_Hash_nodeIPN7rocksdb16ThreadStatusDataELb0EEEEE19_M_allocate_bucketsEm + fun:_ZNSt10_HashtableIPN7rocksdb16ThreadStatusDataES2_SaIS2_ENSt8__detail9_IdentityESt8equal_toIS2_ESt4hashIS2_ENS4_18_Mod_range_hashingENS4_20_Default_ranged_hashENS4_20_Prime_rehash_policyENS4_17_Hashtable_traitsILb0ELb1ELb1EEEE19_M_allocate_bucketsEm + fun:_ZNSt10_HashtableIPN7rocksdb16ThreadStatusDataES2_SaIS2_ENSt8__detail9_IdentityESt8equal_toIS2_ESt4hashIS2_ENS4_18_Mod_range_hashingENS4_20_Default_ranged_hashENS4_20_Prime_rehash_policyENS4_17_Hashtable_traitsILb0ELb1ELb1EEEE13_M_rehash_auxEmSt17integral_constantIbLb1EE + fun:_ZNSt10_HashtableIPN7rocksdb16ThreadStatusDataES2_SaIS2_ENSt8__detail9_IdentityESt8equal_toIS2_ESt4hashIS2_ENS4_18_Mod_range_hashingENS4_20_Default_ranged_hashENS4_20_Prime_rehash_policyENS4_17_Hashtable_traitsILb0ELb1ELb1EEEE9_M_rehashEmRKm + fun:_ZNSt10_HashtableIPN7rocksdb16ThreadStatusDataES2_SaIS2_ENSt8__detail9_IdentityESt8equal_toIS2_ESt4hashIS2_ENS4_18_Mod_range_hashingENS4_20_Default_ranged_hashENS4_20_Prime_rehash_policyENS4_17_Hashtable_traitsILb0ELb1ELb1EEEE21_M_insert_unique_nodeERKS2_mmPNS4_10_Hash_nodeIS2_Lb0EEEm + fun:_ZNSt10_HashtableIPN7rocksdb16ThreadStatusDataES2_SaIS2_ENSt8__detail9_IdentityESt8equal_toIS2_ESt4hashIS2_ENS4_18_Mod_range_hashingENS4_20_Default_ranged_hashENS4_20_Prime_rehash_policyENS4_17_Hashtable_traitsILb0ELb1ELb1EEEE9_M_insertIRKS2_NS4_10_AllocNodeISaINS4_10_Hash_nodeIS2_Lb0EEEEEEEESt4pairINS4_14_Node_iteratorIS2_Lb1ELb0EEEbEOT_RKT0_St17integral_constantIbLb1EEm + fun:_ZNSt8__detail12_Insert_baseIPN7rocksdb16ThreadStatusDataES3_SaIS3_ENS_9_IdentityESt8equal_toIS3_ESt4hashIS3_ENS_18_Mod_range_hashingENS_20_Default_ranged_hashENS_20_Prime_rehash_policyENS_17_Hashtable_traitsILb0ELb1ELb1EEEE6insertERKS3_ + fun:_ZNSt13unordered_setIPN7rocksdb16ThreadStatusDataESt4hashIS2_ESt8equal_toIS2_ESaIS2_EE6insertERKS2_ + fun:_ZN7rocksdb19ThreadStatusUpdater14RegisterThreadENS_12ThreadStatus10ThreadTypeEm + fun:_ZN7rocksdb16ThreadStatusUtil14RegisterThreadEPKNS_3EnvENS_12ThreadStatus10ThreadTypeE + fun:_ZN7rocksdb14ThreadPoolImpl4Impl15BGThreadWrapperEPv + fun:_ZSt13__invoke_implIvPFvPvEJPN7rocksdb16BGThreadMetadataEEET_St14__invoke_otherOT0_DpOT1_ + fun:_ZSt8__invokeIPFvPvEJPN7rocksdb16BGThreadMetadataEEENSt15__invoke_resultIT_JDpT0_EE4typeEOS7_DpOS8_ + fun:_ZNSt6thread8_InvokerISt5tupleIJPFvPvEPN7rocksdb16BGThreadMetadataEEEE9_M_invokeIJLm0ELm1EEEEvSt12_Index_tupleIJXspT_EEE + fun:_ZNSt6thread8_InvokerISt5tupleIJPFvPvEPN7rocksdb16BGThreadMetadataEEEEclEv + fun:_ZNSt6thread11_State_implINS_8_InvokerISt5tupleIJPFvPvEPN7rocksdb16BGThreadMetadataEEEEEE6_M_runEv + obj:/usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.28 +} +{ + + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + fun:_ZN9__gnu_cxx13new_allocatorIPNSt8__detail15_Hash_node_baseEE8allocateEmPKv + fun:_ZNSt16allocator_traitsISaIPNSt8__detail15_Hash_node_baseEEE8allocateERS3_m + fun:_ZNSt8__detail16_Hashtable_allocISaINS_10_Hash_nodeISt4pairIKjPFvPvEELb0EEEEE19_M_allocate_bucketsEm + fun:_ZNSt10_HashtableIjSt4pairIKjPFvPvEESaIS5_ENSt8__detail10_Select1stESt8equal_toIjESt4hashIjENS7_18_Mod_range_hashingENS7_20_Default_ranged_hashENS7_20_Prime_rehash_policyENS7_17_Hashtable_traitsILb0ELb0ELb1EEEE19_M_allocate_bucketsEm + fun:_ZNSt10_HashtableIjSt4pairIKjPFvPvEESaIS5_ENSt8__detail10_Select1stESt8equal_toIjESt4hashIjENS7_18_Mod_range_hashingENS7_20_Default_ranged_hashENS7_20_Prime_rehash_policyENS7_17_Hashtable_traitsILb0ELb0ELb1EEEE13_M_rehash_auxEmSt17integral_constantIbLb1EE + fun:_ZNSt10_HashtableIjSt4pairIKjPFvPvEESaIS5_ENSt8__detail10_Select1stESt8equal_toIjESt4hashIjENS7_18_Mod_range_hashingENS7_20_Default_ranged_hashENS7_20_Prime_rehash_policyENS7_17_Hashtable_traitsILb0ELb0ELb1EEEE9_M_rehashEmRKm + fun:_ZNSt10_HashtableIjSt4pairIKjPFvPvEESaIS5_ENSt8__detail10_Select1stESt8equal_toIjESt4hashIjENS7_18_Mod_range_hashingENS7_20_Default_ranged_hashENS7_20_Prime_rehash_policyENS7_17_Hashtable_traitsILb0ELb0ELb1EEEE21_M_insert_unique_nodeERS1_mmPNS7_10_Hash_nodeIS5_Lb0EEEm + fun:_ZNSt8__detail9_Map_baseIjSt4pairIKjPFvPvEESaIS6_ENS_10_Select1stESt8equal_toIjESt4hashIjENS_18_Mod_range_hashingENS_20_Default_ranged_hashENS_20_Prime_rehash_policyENS_17_Hashtable_traitsILb0ELb0ELb1EEELb1EEixERS2_ + fun:_ZNSt13unordered_mapIjPFvPvESt4hashIjESt8equal_toIjESaISt4pairIKjS2_EEEixERS8_ + fun:_ZN7rocksdb14ThreadLocalPtr10StaticMeta10SetHandlerEjPFvPvE + fun:_ZN7rocksdb14ThreadLocalPtrC1EPFvPvE + fun:_ZN7rocksdb16ColumnFamilyDataC1EjRKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEPNS_7VersionEPNS_5CacheEPNS_18WriteBufferManagerERKNS_19ColumnFamilyOptionsERKNS_18ImmutableDBOptionsERKNS_11FileOptionsEPNS_15ColumnFamilySetEPNS_16BlockCacheTracerERKSt10shared_ptrINS_8IOTracerEES8_ + fun:_ZN7rocksdb15ColumnFamilySetC1ERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEPKNS_18ImmutableDBOptionsERKNS_11FileOptionsEPNS_5CacheEPNS_18WriteBufferManagerEPNS_15WriteControllerEPNS_16BlockCacheTracerERKSt10shared_ptrINS_8IOTracerEES8_ + fun:_ZN7rocksdb10VersionSetC1ERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEPKNS_18ImmutableDBOptionsERKNS_11FileOptionsEPNS_5CacheEPNS_18WriteBufferManagerEPNS_15WriteControllerEPNS_16BlockCacheTracerERKSt10shared_ptrINS_8IOTracerEES8_ + fun:_ZN7rocksdb6DBImplC1ERKNS_9DBOptionsERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEbbb + fun:_ZN7rocksdb6DBImpl4OpenERKNS_9DBOptionsERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEERKSt6vectorINS_22ColumnFamilyDescriptorESaISD_EEPSC_IPNS_18ColumnFamilyHandleESaISJ_EEPPNS_2DBEbb + fun:_ZN7rocksdb2DB4OpenERKNS_9DBOptionsERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEERKSt6vectorINS_22ColumnFamilyDescriptorESaISD_EEPSC_IPNS_18ColumnFamilyHandleESaISJ_EEPPS0_ + fun:_ZN7rocksdb2DB4OpenERKNS_7OptionsERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEPPS0_ + fun:rocksdb_open +} +{ + + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + fun:_ZN7rocksdb12_GLOBAL__N_125CreateThreadStatusUpdaterEv + fun:_ZN7rocksdb12_GLOBAL__N_18PosixEnvC1Ev + fun:_ZN7rocksdb3Env7DefaultEv + fun:_ZN7rocksdb9DBOptionsC1Ev + fun:_ZN7rocksdb7OptionsC1Ev + fun:_ZN7rocksdb18ImmutableCFOptionsC1Ev + fun:_Z41__static_initialization_and_destruction_0ii + fun:_GLOBAL__sub_I_cf_options.cc + fun:__libc_csu_init + fun:(below main) +} +{ + + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + fun:_ZN7rocksdb14ThreadLocalPtr8InstanceEv + fun:_ZN7rocksdb14ThreadLocalPtr14InitSingletonsEv + fun:_ZN7rocksdb3Env7DefaultEv + fun:_ZN7rocksdb9DBOptionsC1Ev + fun:_ZN7rocksdb7OptionsC1Ev + fun:_ZN7rocksdb18ImmutableCFOptionsC1Ev + fun:_Z41__static_initialization_and_destruction_0ii + fun:_GLOBAL__sub_I_cf_options.cc + fun:__libc_csu_init + fun:(below main) +} +{ + + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + fun:_ZN7rocksdb24CacheEntryStatsCollectorINS_13InternalStats19CacheEntryRoleStatsEE9GetSharedEPNS_5CacheEPNS_11SystemClockEPSt10shared_ptrIS3_E + fun:_ZN7rocksdb13InternalStatsC1EiPNS_11SystemClockEPNS_16ColumnFamilyDataE + fun:_ZN7rocksdb16ColumnFamilyDataC1EjRKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEPNS_7VersionEPNS_5CacheEPNS_18WriteBufferManagerERKNS_19ColumnFamilyOptionsERKNS_18ImmutableDBOptionsERKNS_11FileOptionsEPNS_15ColumnFamilySetEPNS_16BlockCacheTracerERKSt10shared_ptrINS_8IOTracerEES8_ + fun:_ZN7rocksdb15ColumnFamilySet18CreateColumnFamilyERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEjPNS_7VersionERKNS_19ColumnFamilyOptionsE + fun:_ZN7rocksdb10VersionSet18CreateColumnFamilyERKNS_19ColumnFamilyOptionsEPKNS_11VersionEditE + fun:_ZN7rocksdb18VersionEditHandler15CreateCfAndInitERKNS_19ColumnFamilyOptionsERKNS_11VersionEditE + fun:_ZN7rocksdb18VersionEditHandler10InitializeEv + fun:_ZN7rocksdb22VersionEditHandlerBase7IterateERNS_3log6ReaderEPNS_6StatusE + fun:_ZN7rocksdb10VersionSet7RecoverERKSt6vectorINS_22ColumnFamilyDescriptorESaIS2_EEbPNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEE + fun:_ZN7rocksdb6DBImpl7RecoverERKSt6vectorINS_22ColumnFamilyDescriptorESaIS2_EEbbbPm + fun:_ZN7rocksdb6DBImpl4OpenERKNS_9DBOptionsERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEERKSt6vectorINS_22ColumnFamilyDescriptorESaISD_EEPSC_IPNS_18ColumnFamilyHandleESaISJ_EEPPNS_2DBEbb + fun:_ZN7rocksdb2DB4OpenERKNS_9DBOptionsERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEERKSt6vectorINS_22ColumnFamilyDescriptorESaISD_EEPSC_IPNS_18ColumnFamilyHandleESaISJ_EEPPS0_ + fun:_ZN7rocksdb2DB4OpenERKNS_7OptionsERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEPPS0_ + fun:rocksdb_open + fun:tsdbOpenRocksCache + fun:tsdbOpenCache + fun:tsdbOpen + fun:vnodeOpen + fun:vmOpenVnodeInThread +} diff --git a/tests/script/sh/exec.sh b/tests/script/sh/exec.sh index f548a4cc41..5061688de2 100755 --- a/tests/script/sh/exec.sh +++ b/tests/script/sh/exec.sh @@ -109,8 +109,10 @@ if [ "$EXEC_OPTON" = "start" ]; then if [ "$VALGRIND_OPTION" = "true" ]; then TT=`date +%s` #mkdir ${LOG_DIR}/${TT} - echo "nohup valgrind --log-file=${LOG_DIR}/valgrind-taosd-${NODE_NAME}-${TT}.log --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all --num-callers=20 -v -v --workaround-gcc296-bugs=yes $EXE_DIR/taosd -c $CFG_DIR > /dev/null 2>&1 &" - nohup valgrind --log-file=${LOG_DIR}/valgrind-taosd-${NODE_NAME}-${TT}.log --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all --num-callers=20 -v -v --workaround-gcc296-bugs=yes $EXE_DIR/taosd -c $CFG_DIR > /dev/null 2>&1 & + #echo "nohup valgrind --log-file=${LOG_DIR}/valgrind-taosd-${NODE_NAME}-${TT}.log --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all --num-callers=20 -v -v --workaround-gcc296-bugs=yes $EXE_DIR/taosd -c $CFG_DIR > /dev/null 2>&1 &" + #nohup valgrind --log-file=${LOG_DIR}/valgrind-taosd-${NODE_NAME}-${TT}.log --gen-suppressions=all --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all --num-callers=20 -v -v --workaround-gcc296-bugs=yes $EXE_DIR/taosd -c $CFG_DIR > /dev/null 2>&1 & + echo "nohup valgrind --log-file=${LOG_DIR}/valgrind-taosd-${NODE_NAME}-${TT}.log --suppressions=${SCRIPT_DIR}/local.supp --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all --num-callers=20 -v -v --workaround-gcc296-bugs=yes $EXE_DIR/taosd -c $CFG_DIR > /dev/null 2>&1 &" + nohup valgrind --log-file=${LOG_DIR}/valgrind-taosd-${NODE_NAME}-${TT}.log --suppressions=${SCRIPT_DIR}/local.supp --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all --num-callers=20 -v -v --workaround-gcc296-bugs=yes $EXE_DIR/taosd -c $CFG_DIR > /dev/null 2>&1 & else echo "nohup $EXE_DIR/taosd -c $CFG_DIR > /dev/null 2>&1 &" nohup $EXE_DIR/taosd -c $CFG_DIR > /dev/null 2> $ASAN_DIR/$NODE_NAME.asan & From cf57a3f3f6a012c05e1db43de6e86b3ec83b3c2d Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 26 Apr 2023 16:30:40 +0800 Subject: [PATCH 54/61] Update 10-function.md --- docs/en/12-taos-sql/10-function.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/12-taos-sql/10-function.md b/docs/en/12-taos-sql/10-function.md index 5c1a833e05..0b5f9103c7 100644 --- a/docs/en/12-taos-sql/10-function.md +++ b/docs/en/12-taos-sql/10-function.md @@ -5,9 +5,9 @@ description: This document describes the standard SQL functions available in TDe toc_max_heading_level: 4 --- -## Single Row Functions +## Scalar Functions -Single row functions return a result for each row. +Scalar functions return one result for each row. ### Mathematical Functions From a98cf1277f25097d948667cbf084a28a4dc3cd92 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 26 Apr 2023 16:33:23 +0800 Subject: [PATCH 55/61] Update 14-stream.md --- docs/en/12-taos-sql/14-stream.md | 34 ++++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/docs/en/12-taos-sql/14-stream.md b/docs/en/12-taos-sql/14-stream.md index b8f6c3a163..eed1713a3c 100644 --- a/docs/en/12-taos-sql/14-stream.md +++ b/docs/en/12-taos-sql/14-stream.md @@ -13,8 +13,11 @@ Because stream processing is built in to TDengine, you are no longer reliant on ```sql CREATE STREAM [IF NOT EXISTS] stream_name [stream_options] INTO stb_name SUBTABLE(expression) AS subquery stream_options: { - TRIGGER [AT_ONCE | WINDOW_CLOSE | MAX_DELAY time] - WATERMARK time + TRIGGER [AT_ONCE | WINDOW_CLOSE | MAX_DELAY time] + WATERMARK time + IGNORE EXPIRED [0|1] + DELETE_MARK time + FILL_HISTORY [0|1] } ``` @@ -141,3 +144,30 @@ The data in expired windows is tagged as expired. TDengine stream processing pro 2. Recalculate the data. In this method, all data in the window is reobtained from the database and recalculated. The latest results are then returned. In both of these methods, configuring the watermark is essential for obtaining accurate results (if expired data is dropped) and avoiding repeated triggers that affect system performance (if expired data is recalculated). + +## Supported functions + +All scalar functions are available in stream processing. +All System Information Functions are not available for stream processing. +All aggregate and selection functions are available in stream processing, except following functions + +- [leastsquares](../function/#leastsquares) +- [percentile](../function/#percentile) +- [top](../function/#leastsquares) +- [bottom](../function/#top) +- [elapsed](../function/#leastsquares) +- [interp](../function/#elapsed) +- [derivative](../function/#derivative) +- [irate](../function/#irate) +- [twa](../function/#twa) +- [histogram](../function/#histogram) +- [diff](../function/#diff) +- [statecount](../function/#statecount) +- [stateduration](../function/#stateduration) +- [csum](../function/#csum) +- [mavg](../function/#mavg) +- [sample](../function/#sample) +- [tail](../function/#tail) +- [unique](../function/#unique) +- [mode](../function/#mode) + From 24e8ff79702d290aec3e61d637c26261d6db07b8 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 26 Apr 2023 16:35:45 +0800 Subject: [PATCH 56/61] Update 14-stream.md --- docs/en/12-taos-sql/14-stream.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/12-taos-sql/14-stream.md b/docs/en/12-taos-sql/14-stream.md index eed1713a3c..9d1b63ce81 100644 --- a/docs/en/12-taos-sql/14-stream.md +++ b/docs/en/12-taos-sql/14-stream.md @@ -147,7 +147,7 @@ In both of these methods, configuring the watermark is essential for obtaining a ## Supported functions -All scalar functions are available in stream processing. +All [scalar functions](../function/#scalar-functions) are available in stream processing. All System Information Functions are not available for stream processing. All aggregate and selection functions are available in stream processing, except following functions From 7113ded06a2ae7c335fd5e2bdf014e1bbdfdc166 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 26 Apr 2023 16:48:21 +0800 Subject: [PATCH 57/61] docs: fix typos and use 3.1.0 in java connector for3.0 (#21095) * docs: update csharp connector status * docs: fix csharp ws bulk pulling * docs: clarify database param is optional to websocket dsn * docs: fix java connector mistake * fix: a few typos * fix: many typos * docs: java connector support subscription over websocket * fix: update 3.0 connector feature matrix * docs: fix typos and use 3.1.0 in jdbc driver --- docs/en/12-taos-sql/07-tag-index.md | 4 ++-- docs/en/12-taos-sql/29-changes.md | 2 +- docs/en/14-reference/03-connector/03-cpp.mdx | 6 +++--- docs/en/14-reference/03-connector/04-java.mdx | 8 ++++---- docs/zh/08-connector/14-java.mdx | 8 ++++---- 5 files changed, 14 insertions(+), 14 deletions(-) diff --git a/docs/en/12-taos-sql/07-tag-index.md b/docs/en/12-taos-sql/07-tag-index.md index cb2a61d3e8..af1d0a352e 100644 --- a/docs/en/12-taos-sql/07-tag-index.md +++ b/docs/en/12-taos-sql/07-tag-index.md @@ -6,7 +6,7 @@ description: Use Tag Index to Improve Query Performance ## Introduction -Prior to TDengine 3.0.3.0 (excluded)，only one index is created by default on the first tag of each super talbe, but it's not allowed to dynamically create index on any other tags. From version 3.0.30, you can dynamically create index on any tag of any type. The index created automatically by TDengine is still valid. Query performance can benefit from indexes if you use properly. +Prior to TDengine 3.0.3.0 (excluded)，only one index is created by default on the first tag of each super table, but it's not allowed to dynamically create index on any other tags. From version 3.0.30, you can dynamically create index on any tag of any type. The index created automatically by TDengine is still valid. Query performance can benefit from indexes if you use properly. ## Syntax @@ -48,4 +48,4 @@ You can also add filter conditions to limit the results. 6. You can' create index on a normal table or a child table. -7. If the unique values of a tag column are too few, it's better not to create index on such tag columns, the benefit would be very small. \ No newline at end of file +7. If the unique values of a tag column are too few, it's better not to create index on such tag columns, the benefit would be very small. diff --git a/docs/en/12-taos-sql/29-changes.md b/docs/en/12-taos-sql/29-changes.md index f4606f263f..086aee59fe 100644 --- a/docs/en/12-taos-sql/29-changes.md +++ b/docs/en/12-taos-sql/29-changes.md @@ -27,7 +27,7 @@ The following data types can be used in the schema for standard tables. | - | :------- | :-------- | :------- | | 1 | ALTER ACCOUNT | Deprecated| This Enterprise Edition-only statement has been removed. It returns the error "This statement is no longer supported." | 2 | ALTER ALL DNODES | Added | Modifies the configuration of all dnodes. -| 3 | ALTER DATABASE | Modified | Deprecated
  • QUORUM: Specified the required number of confirmations. TDengine 3.0 provides strict consistency by default and doesn't allow to change to weak consitency.
  • BLOCKS: Specified the memory blocks used by each vnode. BUFFER is now used to specify the size of the write cache pool for each vnode.
  • UPDATE: Specified whether update operations were supported. All databases now support updating data in certain columns.
  • CACHELAST: Specified how to cache the newest row of data. CACHEMODEL now replaces CACHELAST.
  • COMP: Cannot be modified.
    Added
  • CACHEMODEL: Specifies whether to cache the latest subtable data.
  • CACHESIZE: Specifies the size of the cache for the newest subtable data.
  • WAL_FSYNC_PERIOD: Replaces the FSYNC parameter.
  • WAL_LEVEL: Replaces the WAL parameter.
  • WAL_RETENTION_PERIOD: specifies the time after which WAL files are deleted. This parameter is used for data subscription.
  • WAL_RETENTION_SIZE: specifies the size at which WAL files are deleted. This parameter is used for data subscription.
    Modified
  • REPLICA: Cannot be modified.
  • KEEP: Now supports units.
+| 3 | ALTER DATABASE | Modified | Deprecated
  • QUORUM: Specified the required number of confirmations. TDengine 3.0 provides strict consistency by default and doesn't allow to change to weak consistency.
  • BLOCKS: Specified the memory blocks used by each vnode. BUFFER is now used to specify the size of the write cache pool for each vnode.
  • UPDATE: Specified whether update operations were supported. All databases now support updating data in certain columns.
  • CACHELAST: Specified how to cache the newest row of data. CACHEMODEL now replaces CACHELAST.
  • COMP: Cannot be modified.
    Added
  • CACHEMODEL: Specifies whether to cache the latest subtable data.
  • CACHESIZE: Specifies the size of the cache for the newest subtable data.
  • WAL_FSYNC_PERIOD: Replaces the FSYNC parameter.
  • WAL_LEVEL: Replaces the WAL parameter.
  • WAL_RETENTION_PERIOD: specifies the time after which WAL files are deleted. This parameter is used for data subscription.
  • WAL_RETENTION_SIZE: specifies the size at which WAL files are deleted. This parameter is used for data subscription.
    Modified
  • REPLICA: Cannot be modified.
  • KEEP: Now supports units.
| 4 | ALTER STABLE | Modified | Deprecated
  • CHANGE TAG: Modified the name of a tag. Replaced by RENAME TAG.
    Added
  • RENAME TAG: Replaces CHANGE TAG.
  • COMMENT: Specifies comments for a supertable.
| 5 | ALTER TABLE | Modified | Deprecated
  • CHANGE TAG: Modified the name of a tag. Replaced by RENAME TAG.
    Added
  • RENAME TAG: Replaces CHANGE TAG.
  • COMMENT: Specifies comments for a standard table.
  • TTL: Specifies the time-to-live for a standard table.
| 6 | ALTER USER | Modified | Deprecated
  • PRIVILEGE: Specified user permissions. Replaced by GRANT and REVOKE.
    Added
  • ENABLE: Enables or disables a user.
  • SYSINFO: Specifies whether a user can query system information.
diff --git a/docs/en/14-reference/03-connector/03-cpp.mdx b/docs/en/14-reference/03-connector/03-cpp.mdx index b543879b3c..014109b77e 100644 --- a/docs/en/14-reference/03-connector/03-cpp.mdx +++ b/docs/en/14-reference/03-connector/03-cpp.mdx @@ -423,6 +423,6 @@ In addition to writing data using the SQL method or the parameter binding API, w **Description** - The above seven interfaces are extension interfaces, which are mainly used to pass ttl and reqid parameters, and can be used as needed. - - Withing _raw interfaces represent data through the passed parameters lines and len. In order to solve the problem that the original interface data contains '\0' and is truncated. The totalRows pointer returns the number of parsed data rows. - - Withing _ttl interfaces can pass the ttl parameter to control the ttl expiration time of the table. - - Withing _reqid interfaces can track the entire call chain by passing the reqid parameter. + - Within _raw interfaces represent data through the passed parameters lines and len. In order to solve the problem that the original interface data contains '\0' and is truncated. The totalRows pointer returns the number of parsed data rows. + - Within _ttl interfaces can pass the ttl parameter to control the ttl expiration time of the table. + - Within _reqid interfaces can track the entire call chain by passing the reqid parameter. diff --git a/docs/en/14-reference/03-connector/04-java.mdx b/docs/en/14-reference/03-connector/04-java.mdx index fd4b4641d7..85d79ef0e0 100644 --- a/docs/en/14-reference/03-connector/04-java.mdx +++ b/docs/en/14-reference/03-connector/04-java.mdx @@ -82,7 +82,7 @@ Add following dependency in the `pom.xml` file of your Maven project: com.taosdata.jdbc taos-jdbcdriver - 3.0.0 + 3.1.0 ``` @@ -227,7 +227,7 @@ In addition to getting the connection from the specified URL, you can use Proper Note: - The client parameter set in the application is process-level. If you want to update the parameters of the client, you need to restart the application. This is because the client parameter is a global parameter that takes effect only the first time the application is set. -- The following sample code is based on taos-jdbcdriver-3.0.0. +- The following sample code is based on taos-jdbcdriver-3.1.0. ```java public Connection getConn() throws Exception{ @@ -364,7 +364,7 @@ TDengine has significantly improved the bind APIs to support data writing (INSER **Note:** - JDBC REST connections do not currently support bind interface -- The following sample code is based on taos-jdbcdriver-3.0.0 +- The following sample code is based on taos-jdbcdriver-3.1.0 - The setString method should be called for binary type data, and the setNString method should be called for nchar type data - both setString and setNString require the user to declare the width of the corresponding column in the size parameter of the table definition @@ -632,7 +632,7 @@ TDengine supports schemaless writing. It is compatible with InfluxDB's Line Prot Note: - JDBC REST connections do not currently support schemaless writes -- The following sample code is based on taos-jdbcdriver-3.0.0 +- The following sample code is based on taos-jdbcdriver-3.1.0 ```java public class SchemalessInsertTest { diff --git a/docs/zh/08-connector/14-java.mdx b/docs/zh/08-connector/14-java.mdx index d1c1258365..8a9bba7fe7 100644 --- a/docs/zh/08-connector/14-java.mdx +++ b/docs/zh/08-connector/14-java.mdx @@ -82,7 +82,7 @@ Maven 项目中，在 pom.xml 中添加以下依赖： com.taosdata.jdbc taos-jdbcdriver - 3.0.0 + 3.1.0 ``` @@ -230,7 +230,7 @@ INSERT INTO test.t1 USING test.weather (ts, temperature) TAGS('California.SanFra **注意**： - 应用中设置的 client parameter 为进程级别的，即如果要更新 client 的参数，需要重启应用。这是因为 client parameter 是全局参数，仅在应用程序的第一次设置生效。 -- 以下示例代码基于 taos-jdbcdriver-3.0.0。 +- 以下示例代码基于 taos-jdbcdriver-3.1.0。 ```java public Connection getConn() throws Exception{ @@ -367,7 +367,7 @@ TDengine 的 JDBC 原生连接实现大幅改进了参数绑定方式对数据 **注意**： - JDBC REST 连接目前不支持参数绑定 -- 以下示例代码基于 taos-jdbcdriver-3.0.0 +- 以下示例代码基于 taos-jdbcdriver-3.1.0 - binary 类型数据需要调用 setString 方法，nchar 类型数据需要调用 setNString 方法 - setString 和 setNString 都要求用户在 size 参数里声明表定义中对应列的列宽 @@ -635,7 +635,7 @@ TDengine 支持无模式写入功能。无模式写入兼容 InfluxDB 的 行协 **注意**： - JDBC REST 连接目前不支持无模式写入 -- 以下示例代码基于 taos-jdbcdriver-3.0.0 +- 以下示例代码基于 taos-jdbcdriver-3.1.0 ```java public class SchemalessInsertTest { From f7141fdebb7d7e63c27715d70da36d30423f4cb2 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 26 Apr 2023 16:53:52 +0800 Subject: [PATCH 58/61] Update 14-stream.md --- docs/en/12-taos-sql/14-stream.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/12-taos-sql/14-stream.md b/docs/en/12-taos-sql/14-stream.md index 9d1b63ce81..53d61ddde4 100644 --- a/docs/en/12-taos-sql/14-stream.md +++ b/docs/en/12-taos-sql/14-stream.md @@ -149,7 +149,7 @@ In both of these methods, configuring the watermark is essential for obtaining a All [scalar functions](../function/#scalar-functions) are available in stream processing. All System Information Functions are not available for stream processing. -All aggregate and selection functions are available in stream processing, except following functions +All [aggregate and selection functions](../function/#system-information-functions) are available in stream processing, except following functions - [leastsquares](../function/#leastsquares) - [percentile](../function/#percentile) From cf9567ac966fe57d57579d3066949e4c9e4c60f2 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 26 Apr 2023 17:04:37 +0800 Subject: [PATCH 59/61] Update 14-stream.md --- docs/en/12-taos-sql/14-stream.md | 43 +++++++++++++++----------------- 1 file changed, 20 insertions(+), 23 deletions(-) diff --git a/docs/en/12-taos-sql/14-stream.md b/docs/en/12-taos-sql/14-stream.md index 53d61ddde4..0e013ed846 100644 --- a/docs/en/12-taos-sql/14-stream.md +++ b/docs/en/12-taos-sql/14-stream.md @@ -147,27 +147,24 @@ In both of these methods, configuring the watermark is essential for obtaining a ## Supported functions -All [scalar functions](../function/#scalar-functions) are available in stream processing. -All System Information Functions are not available for stream processing. -All [aggregate and selection functions](../function/#system-information-functions) are available in stream processing, except following functions - -- [leastsquares](../function/#leastsquares) -- [percentile](../function/#percentile) -- [top](../function/#leastsquares) -- [bottom](../function/#top) -- [elapsed](../function/#leastsquares) -- [interp](../function/#elapsed) -- [derivative](../function/#derivative) -- [irate](../function/#irate) -- [twa](../function/#twa) -- [histogram](../function/#histogram) -- [diff](../function/#diff) -- [statecount](../function/#statecount) -- [stateduration](../function/#stateduration) -- [csum](../function/#csum) -- [mavg](../function/#mavg) -- [sample](../function/#sample) -- [tail](../function/#tail) -- [unique](../function/#unique) -- [mode](../function/#mode) +All [scalar functions](../function/#scalar-functions) are available in stream processing. All [System Information Functions](../function/#system-information-functions) are not allowed in stream processing. All [Aggregate functions](function/#aggregate-functions) and [Selection functions](../function/#selection-functions) are available in stream processing, except the followings: + - [leastsquares](../function/#leastsquares) + - [percentile](../function/#percentile) + - [top](../function/#leastsquares) + - [bottom](../function/#top) + - [elapsed](../function/#leastsquares) + - [interp](../function/#elapsed) + - [derivative](../function/#derivative) + - [irate](../function/#irate) + - [twa](../function/#twa) + - [histogram](../function/#histogram) + - [diff](../function/#diff) + - [statecount](../function/#statecount) + - [stateduration](../function/#stateduration) + - [csum](../function/#csum) + - [mavg](../function/#mavg) + - [sample](../function/#sample) + - [tail](../function/#tail) + - [unique](../function/#unique) + - [mode](../function/#mode) From e4d40489e77db0094bd66d24bd0a67633f2440fd Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 26 Apr 2023 17:13:20 +0800 Subject: [PATCH 60/61] Update 14-stream.md --- docs/en/12-taos-sql/14-stream.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/12-taos-sql/14-stream.md b/docs/en/12-taos-sql/14-stream.md index 0e013ed846..0fe2c506a6 100644 --- a/docs/en/12-taos-sql/14-stream.md +++ b/docs/en/12-taos-sql/14-stream.md @@ -147,7 +147,7 @@ In both of these methods, configuring the watermark is essential for obtaining a ## Supported functions -All [scalar functions](../function/#scalar-functions) are available in stream processing. All [System Information Functions](../function/#system-information-functions) are not allowed in stream processing. All [Aggregate functions](function/#aggregate-functions) and [Selection functions](../function/#selection-functions) are available in stream processing, except the followings: +All [scalar functions](../function/#scalar-functions) are available in stream processing. All [System Information Functions](../function/#system-information-functions) are not allowed in stream processing. All [Aggregate functions](../function/#aggregate-functions) and [Selection functions](../function/#selection-functions) are available in stream processing, except the followings: - [leastsquares](../function/#leastsquares) - [percentile](../function/#percentile) - [top](../function/#leastsquares) From 457ed296d3384f28d9a399f509b5ef87f03dc1af Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 26 Apr 2023 17:21:45 +0800 Subject: [PATCH 61/61] Update 14-stream.md --- docs/en/12-taos-sql/14-stream.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/12-taos-sql/14-stream.md b/docs/en/12-taos-sql/14-stream.md index 0fe2c506a6..de3fd1f4be 100644 --- a/docs/en/12-taos-sql/14-stream.md +++ b/docs/en/12-taos-sql/14-stream.md @@ -147,7 +147,7 @@ In both of these methods, configuring the watermark is essential for obtaining a ## Supported functions -All [scalar functions](../function/#scalar-functions) are available in stream processing. All [System Information Functions](../function/#system-information-functions) are not allowed in stream processing. All [Aggregate functions](../function/#aggregate-functions) and [Selection functions](../function/#selection-functions) are available in stream processing, except the followings: +All [scalar functions](../function/#scalar-functions) are available in stream processing. All [System information functions](../function/#system-information-functions) are not allowed in stream processing. All [Aggregate functions](../function/#aggregate-functions) and [Selection functions](../function/#selection-functions) are available in stream processing, except the followings: - [leastsquares](../function/#leastsquares) - [percentile](../function/#percentile) - [top](../function/#leastsquares)