Merge branch '3.0' into refactor/db_precision

This commit is contained in:
Ganlin Zhao 2022-06-29 20:50:06 +08:00
commit 4d59cfec76
55 changed files with 1335 additions and 797 deletions

View File

@ -164,6 +164,7 @@ DLL_EXPORT TAOS_RES *taos_query(TAOS *taos, const char *sql);
DLL_EXPORT TAOS_ROW taos_fetch_row(TAOS_RES *res); DLL_EXPORT TAOS_ROW taos_fetch_row(TAOS_RES *res);
DLL_EXPORT int taos_result_precision(TAOS_RES *res); // get the time precision of result DLL_EXPORT int taos_result_precision(TAOS_RES *res); // get the time precision of result
DLL_EXPORT void taos_free_result(TAOS_RES *res); DLL_EXPORT void taos_free_result(TAOS_RES *res);
DLL_EXPORT void taos_kill_query(TAOS *taos);
DLL_EXPORT int taos_field_count(TAOS_RES *res); DLL_EXPORT int taos_field_count(TAOS_RES *res);
DLL_EXPORT int taos_num_fields(TAOS_RES *res); DLL_EXPORT int taos_num_fields(TAOS_RES *res);
DLL_EXPORT int taos_affected_rows(TAOS_RES *res); DLL_EXPORT int taos_affected_rows(TAOS_RES *res);

View File

@ -36,7 +36,6 @@ typedef struct SReadHandle {
void* vnode; void* vnode;
void* mnd; void* mnd;
SMsgCb* pMsgCb; SMsgCb* pMsgCb;
// int8_t initTsdbReader;
} SReadHandle; } SReadHandle;
enum { enum {
@ -140,12 +139,6 @@ int32_t qKillTask(qTaskInfo_t tinfo);
*/ */
int32_t qAsyncKillTask(qTaskInfo_t tinfo); int32_t qAsyncKillTask(qTaskInfo_t tinfo);
/**
* return whether query is completed or not
* @param tinfo
* @return
*/
int32_t qIsTaskCompleted(qTaskInfo_t tinfo);
/** /**
* destroy query info structure * destroy query info structure
@ -176,6 +169,15 @@ int32_t qSerializeTaskStatus(qTaskInfo_t tinfo, char** pOutput, int32_t* len);
int32_t qDeserializeTaskStatus(qTaskInfo_t tinfo, const char* pInput, int32_t len); int32_t qDeserializeTaskStatus(qTaskInfo_t tinfo, const char* pInput, int32_t len);
/**
* return the scan info, in the form of tuple of two items, including table uid and current timestamp
* @param tinfo
* @param uid
* @param ts
* @return
*/
int32_t qGetStreamScanStatus(qTaskInfo_t tinfo, uint64_t* uid, int64_t* ts);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

View File

@ -69,18 +69,20 @@ typedef struct SSchdFetchParam {
int32_t* code; int32_t* code;
} SSchdFetchParam; } SSchdFetchParam;
typedef void (*schedulerExecCallback)(SQueryResult* pResult, void* param, int32_t code); typedef void (*schedulerExecFp)(SQueryResult* pResult, void* param, int32_t code);
typedef void (*schedulerFetchCallback)(void* pResult, void* param, int32_t code); typedef void (*schedulerFetchFp)(void* pResult, void* param, int32_t code);
typedef bool (*schedulerChkKillFp)(void* param);
typedef struct SSchedulerReq { typedef struct SSchedulerReq {
bool *reqKilled;
SRequestConnInfo *pConn; SRequestConnInfo *pConn;
SArray *pNodeList; SArray *pNodeList;
SQueryPlan *pDag; SQueryPlan *pDag;
const char *sql; const char *sql;
int64_t startTs; int64_t startTs;
schedulerExecCallback fp; schedulerExecFp execFp;
void* cbParam; void* execParam;
schedulerChkKillFp chkKillFp;
void* chkKillParam;
} SSchedulerReq; } SSchedulerReq;
@ -110,7 +112,7 @@ int32_t schedulerExecJob(SSchedulerReq *pReq, int64_t *pJob, SQueryResult *pRes)
*/ */
int32_t schedulerFetchRows(int64_t job, void **data); int32_t schedulerFetchRows(int64_t job, void **data);
void schedulerAsyncFetchRows(int64_t job, schedulerFetchCallback fp, void* param); void schedulerAsyncFetchRows(int64_t job, schedulerFetchFp fp, void* param);
int32_t schedulerGetTasksStatus(int64_t job, SArray *pSub); int32_t schedulerGetTasksStatus(int64_t job, SArray *pSub);

View File

@ -324,6 +324,23 @@ void syncAppendEntriesPrint2(char* s, const SyncAppendEntries* pMsg);
void syncAppendEntriesLog(const SyncAppendEntries* pMsg); void syncAppendEntriesLog(const SyncAppendEntries* pMsg);
void syncAppendEntriesLog2(char* s, const SyncAppendEntries* pMsg); void syncAppendEntriesLog2(char* s, const SyncAppendEntries* pMsg);
// ---------------------------------------------
typedef struct SyncAppendEntriesBatch {
uint32_t bytes;
int32_t vgId;
uint32_t msgType;
SRaftId srcId;
SRaftId destId;
// private data
SyncTerm term;
SyncIndex prevLogIndex;
SyncTerm prevLogTerm;
SyncIndex commitIndex;
SyncTerm privateTerm;
uint32_t dataLen;
char data[];
} SyncAppendEntriesBatch;
// --------------------------------------------- // ---------------------------------------------
typedef struct SyncAppendEntriesReply { typedef struct SyncAppendEntriesReply {
uint32_t bytes; uint32_t bytes;

View File

@ -45,7 +45,7 @@ typedef struct SRpcHandleInfo {
int32_t noResp; // has response or not(default 0, 0: resp, 1: no resp); int32_t noResp; // has response or not(default 0, 0: resp, 1: no resp);
int32_t persistHandle; // persist handle or not int32_t persistHandle; // persist handle or not
STraceId traceId; STraceId traceId;
// int64_t traceId; int8_t hasEpSet;
// app info // app info
void *ahandle; // app handle set by client void *ahandle; // app handle set by client
@ -123,7 +123,7 @@ void * rpcReallocCont(void *ptr, int32_t contLen);
void rpcSendRequest(void *thandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int64_t *rid); void rpcSendRequest(void *thandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int64_t *rid);
void rpcSendResponse(const SRpcMsg *pMsg); void rpcSendResponse(const SRpcMsg *pMsg);
void rpcRegisterBrokenLinkArg(SRpcMsg *msg); void rpcRegisterBrokenLinkArg(SRpcMsg *msg);
void rpcReleaseHandle(void *handle, int8_t type); // just release client conn to rpc instance, no close sock void rpcReleaseHandle(void *handle, int8_t type); // just release conn to rpc instance, no close sock
// These functions will not be called in the child process // These functions will not be called in the child process
void rpcSendRedirectRsp(void *pConn, const SEpSet *pEpSet); void rpcSendRedirectRsp(void *pConn, const SEpSet *pEpSet);

View File

@ -181,15 +181,9 @@ cd "${curr_dir}"
# 2. cmake executable file # 2. cmake executable file
compile_dir="${top_dir}/debug" compile_dir="${top_dir}/debug"
if [ -d ${compile_dir} ]; then ${csudo}rm -rf ${compile_dir}
${csudo}rm -rf ${compile_dir}
fi
if [ "$osType" != "Darwin" ]; then mkdir -p ${compile_dir}
${csudo}mkdir -p ${compile_dir}
else
mkdir -p ${compile_dir}
fi
cd ${compile_dir} cd ${compile_dir}
if [[ "$allocator" == "jemalloc" ]]; then if [[ "$allocator" == "jemalloc" ]]; then
@ -255,9 +249,9 @@ if [ "$osType" != "Darwin" ]; then
echo "====do deb package for the ubuntu system====" echo "====do deb package for the ubuntu system===="
output_dir="${top_dir}/debs" output_dir="${top_dir}/debs"
if [ -d ${output_dir} ]; then if [ -d ${output_dir} ]; then
${csudo}rm -rf ${output_dir} rm -rf ${output_dir}
fi fi
${csudo}mkdir -p ${output_dir} mkdir -p ${output_dir}
cd ${script_dir}/deb cd ${script_dir}/deb
${csudo}./makedeb.sh ${compile_dir} ${output_dir} ${verNumber} ${cpuType} ${osType} ${verMode} ${verType} ${csudo}./makedeb.sh ${compile_dir} ${output_dir} ${verNumber} ${cpuType} ${osType} ${verMode} ${verType}
@ -280,9 +274,9 @@ if [ "$osType" != "Darwin" ]; then
echo "====do rpm package for the centos system====" echo "====do rpm package for the centos system===="
output_dir="${top_dir}/rpms" output_dir="${top_dir}/rpms"
if [ -d ${output_dir} ]; then if [ -d ${output_dir} ]; then
${csudo}rm -rf ${output_dir} rm -rf ${output_dir}
fi fi
${csudo}mkdir -p ${output_dir} mkdir -p ${output_dir}
cd ${script_dir}/rpm cd ${script_dir}/rpm
${csudo}./makerpm.sh ${compile_dir} ${output_dir} ${verNumber} ${cpuType} ${osType} ${verMode} ${verType} ${csudo}./makerpm.sh ${compile_dir} ${output_dir} ${verNumber} ${cpuType} ${osType} ${verMode} ${verType}

0
packaging/rpm/makerpm.sh Normal file → Executable file
View File

View File

@ -118,6 +118,7 @@ struct SAppInstInfo {
uint64_t clusterId; uint64_t clusterId;
void* pTransporter; void* pTransporter;
SAppHbMgr* pAppHbMgr; SAppHbMgr* pAppHbMgr;
char* instKey;
}; };
typedef struct SAppInfo { typedef struct SAppInfo {
@ -139,7 +140,7 @@ typedef struct STscObj {
int8_t connType; int8_t connType;
int32_t acctId; int32_t acctId;
uint32_t connId; uint32_t connId;
TAOS* id; // ref ID returned by taosAddRef int64_t id; // ref ID returned by taosAddRef
TdThreadMutex mutex; // used to protect the operation on db TdThreadMutex mutex; // used to protect the operation on db
int32_t numOfReqs; // number of sqlObj bound to this connection int32_t numOfReqs; // number of sqlObj bound to this connection
SAppInstInfo* pAppInfo; SAppInstInfo* pAppInfo;
@ -183,7 +184,7 @@ typedef struct SRequestSendRecvBody {
void* param; void* param;
SDataBuf requestMsg; SDataBuf requestMsg;
int64_t queryJob; // query job, created according to sql query DAG. int64_t queryJob; // query job, created according to sql query DAG.
struct SQueryPlan* pDag; // the query dag, generated according to the sql statement. int32_t subplanNum;
SReqResultInfo resInfo; SReqResultInfo resInfo;
} SRequestSendRecvBody; } SRequestSendRecvBody;
@ -300,6 +301,8 @@ void* createRequest(STscObj* pObj, int32_t type);
void destroyRequest(SRequestObj* pRequest); void destroyRequest(SRequestObj* pRequest);
SRequestObj* acquireRequest(int64_t rid); SRequestObj* acquireRequest(int64_t rid);
int32_t releaseRequest(int64_t rid); int32_t releaseRequest(int64_t rid);
int32_t removeRequest(int64_t rid);
void doDestroyRequest(void *p);
char* getDbOfConnection(STscObj* pObj); char* getDbOfConnection(STscObj* pObj);
void setConnectionDB(STscObj* pTscObj, const char* db); void setConnectionDB(STscObj* pTscObj, const char* db);
@ -334,6 +337,8 @@ int hbHandleRsp(SClientHbBatchRsp* hbRsp);
// cluster level // cluster level
SAppHbMgr* appHbMgrInit(SAppInstInfo* pAppInstInfo, char* key); SAppHbMgr* appHbMgrInit(SAppInstInfo* pAppInstInfo, char* key);
void appHbMgrCleanup(void); void appHbMgrCleanup(void);
void hbRemoveAppHbMrg(SAppHbMgr **pAppHbMgr);
void closeAllRequests(SHashObj *pRequests);
// conn level // conn level
int hbRegisterConn(SAppHbMgr* pAppHbMgr, int64_t tscRefId, int64_t clusterId, int8_t connType); int hbRegisterConn(SAppHbMgr* pAppHbMgr, int64_t tscRefId, int64_t clusterId, int8_t connType);

View File

@ -37,10 +37,12 @@ int32_t clientConnRefPool = -1;
static TdThreadOnce tscinit = PTHREAD_ONCE_INIT; static TdThreadOnce tscinit = PTHREAD_ONCE_INIT;
volatile int32_t tscInitRes = 0; volatile int32_t tscInitRes = 0;
static void registerRequest(SRequestObj *pRequest) { static int32_t registerRequest(SRequestObj *pRequest) {
STscObj *pTscObj = acquireTscObj(*(int64_t *)pRequest->pTscObj->id); STscObj *pTscObj = acquireTscObj(pRequest->pTscObj->id);
if (NULL == pTscObj) {
assert(pTscObj != NULL); terrno = TSDB_CODE_TSC_DISCONNECTED;
return terrno;
}
// connection has been released already, abort creating request. // connection has been released already, abort creating request.
pRequest->self = taosAddRef(clientReqRefPool, pRequest); pRequest->self = taosAddRef(clientReqRefPool, pRequest);
@ -54,8 +56,10 @@ static void registerRequest(SRequestObj *pRequest) {
int32_t currentInst = atomic_add_fetch_64((int64_t *)&pSummary->currentRequests, 1); int32_t currentInst = atomic_add_fetch_64((int64_t *)&pSummary->currentRequests, 1);
tscDebug("0x%" PRIx64 " new Request from connObj:0x%" PRIx64 tscDebug("0x%" PRIx64 " new Request from connObj:0x%" PRIx64
", current:%d, app current:%d, total:%d, reqId:0x%" PRIx64, ", current:%d, app current:%d, total:%d, reqId:0x%" PRIx64,
pRequest->self, *(int64_t *)pRequest->pTscObj->id, num, currentInst, total, pRequest->requestId); pRequest->self, pRequest->pTscObj->id, num, currentInst, total, pRequest->requestId);
} }
return TSDB_CODE_SUCCESS;
} }
static void deregisterRequest(SRequestObj *pRequest) { static void deregisterRequest(SRequestObj *pRequest) {
@ -70,26 +74,23 @@ static void deregisterRequest(SRequestObj *pRequest) {
int64_t duration = taosGetTimestampUs() - pRequest->metric.start; int64_t duration = taosGetTimestampUs() - pRequest->metric.start;
tscDebug("0x%" PRIx64 " free Request from connObj: 0x%" PRIx64 ", reqId:0x%" PRIx64 " elapsed:%" PRIu64 tscDebug("0x%" PRIx64 " free Request from connObj: 0x%" PRIx64 ", reqId:0x%" PRIx64 " elapsed:%" PRIu64
" ms, current:%d, app current:%d", " ms, current:%d, app current:%d",
pRequest->self, *(int64_t *)pTscObj->id, pRequest->requestId, duration / 1000, num, currentInst); pRequest->self, pTscObj->id, pRequest->requestId, duration / 1000, num, currentInst);
releaseTscObj(*(int64_t *)pTscObj->id); releaseTscObj(pTscObj->id);
} }
// todo close the transporter properly // todo close the transporter properly
void closeTransporter(STscObj *pTscObj) { void closeTransporter(SAppInstInfo *pAppInfo) {
if (pTscObj == NULL || pTscObj->pAppInfo->pTransporter == NULL) { if (pAppInfo == NULL || pAppInfo->pTransporter == NULL) {
return; return;
} }
tscDebug("free transporter:%p in connObj: 0x%" PRIx64, pTscObj->pAppInfo->pTransporter, *(int64_t *)pTscObj->id); tscDebug("free transporter:%p in app inst %p", pAppInfo->pTransporter, pAppInfo);
rpcClose(pTscObj->pAppInfo->pTransporter); rpcClose(pAppInfo->pTransporter);
} }
static bool clientRpcRfp(int32_t code, tmsg_t msgType) { static bool clientRpcRfp(int32_t code, tmsg_t msgType) {
if (code == TSDB_CODE_RPC_REDIRECT || code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_NODE_NOT_DEPLOYED || if (code == TSDB_CODE_RPC_REDIRECT || code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_NODE_NOT_DEPLOYED ||
code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_APP_NOT_READY) { code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_APP_NOT_READY) {
if (msgType == TDMT_VND_QUERY || msgType == TDMT_VND_FETCH) {
return false;
}
return true; return true;
} else { } else {
return false; return false;
@ -123,26 +124,46 @@ void closeAllRequests(SHashObj *pRequests) {
while (pIter != NULL) { while (pIter != NULL) {
int64_t *rid = pIter; int64_t *rid = pIter;
releaseRequest(*rid); removeRequest(*rid);
pIter = taosHashIterate(pRequests, pIter); pIter = taosHashIterate(pRequests, pIter);
} }
} }
void destroyAppInst(SAppInstInfo *pAppInfo) {
tscDebug("destroy app inst mgr %p", pAppInfo);
taosThreadMutexLock(&appInfo.mutex);
hbRemoveAppHbMrg(&pAppInfo->pAppHbMgr);
taosHashRemove(appInfo.pInstMap, pAppInfo->instKey, strlen(pAppInfo->instKey));
taosThreadMutexUnlock(&appInfo.mutex);
taosMemoryFreeClear(pAppInfo->instKey);
closeTransporter(pAppInfo);
taosThreadMutexLock(&pAppInfo->qnodeMutex);
taosArrayDestroy(pAppInfo->pQnodeList);
taosThreadMutexUnlock(&pAppInfo->qnodeMutex);
taosMemoryFree(pAppInfo);
}
void destroyTscObj(void *pObj) { void destroyTscObj(void *pObj) {
STscObj *pTscObj = pObj; STscObj *pTscObj = pObj;
SClientHbKey connKey = {.tscRid = *(int64_t *)pTscObj->id, .connType = pTscObj->connType}; SClientHbKey connKey = {.tscRid = pTscObj->id, .connType = pTscObj->connType};
hbDeregisterConn(pTscObj->pAppInfo->pAppHbMgr, connKey); hbDeregisterConn(pTscObj->pAppInfo->pAppHbMgr, connKey);
int64_t connNum = atomic_sub_fetch_64(&pTscObj->pAppInfo->numOfConns, 1); int64_t connNum = atomic_sub_fetch_64(&pTscObj->pAppInfo->numOfConns, 1);
closeAllRequests(pTscObj->pRequests); closeAllRequests(pTscObj->pRequests);
schedulerStopQueryHb(pTscObj->pAppInfo->pTransporter); schedulerStopQueryHb(pTscObj->pAppInfo->pTransporter);
if (0 == connNum) { tscDebug("connObj 0x%" PRIx64 " p:%p destroyed, remain inst totalConn:%" PRId64, pTscObj->id, pTscObj,
// TODO
// closeTransporter(pTscObj);
}
tscDebug("connObj 0x%" PRIx64 " destroyed, totalConn:%" PRId64, *(int64_t *)pTscObj->id,
pTscObj->pAppInfo->numOfConns); pTscObj->pAppInfo->numOfConns);
if (0 == connNum) {
destroyAppInst(pTscObj->pAppInfo);
}
taosThreadMutexDestroy(&pTscObj->mutex); taosThreadMutexDestroy(&pTscObj->mutex);
taosMemoryFreeClear(pTscObj); taosMemoryFreeClear(pTscObj);
} }
@ -171,11 +192,12 @@ void *createTscObj(const char *user, const char *auth, const char *db, int32_t c
} }
taosThreadMutexInit(&pObj->mutex, NULL); taosThreadMutexInit(&pObj->mutex, NULL);
pObj->id = taosMemoryMalloc(sizeof(int64_t)); pObj->id = taosAddRef(clientConnRefPool, pObj);
*(int64_t *)pObj->id = taosAddRef(clientConnRefPool, pObj);
pObj->schemalessType = 1; pObj->schemalessType = 1;
tscDebug("connObj created, 0x%" PRIx64, *(int64_t *)pObj->id); atomic_add_fetch_64(&pObj->pAppInfo->numOfConns, 1);
tscDebug("connObj created, 0x%" PRIx64 ",p:%p", pObj->id, pObj);
return pObj; return pObj;
} }
@ -205,7 +227,10 @@ void *createRequest(STscObj *pObj, int32_t type) {
pRequest->msgBufLen = ERROR_MSG_BUF_DEFAULT_SIZE; pRequest->msgBufLen = ERROR_MSG_BUF_DEFAULT_SIZE;
tsem_init(&pRequest->body.rspSem, 0, 0); tsem_init(&pRequest->body.rspSem, 0, 0);
registerRequest(pRequest); if (registerRequest(pRequest)) {
doDestroyRequest(pRequest);
return NULL;
}
return pRequest; return pRequest;
} }
@ -227,12 +252,16 @@ void doFreeReqResultInfo(SReqResultInfo *pResInfo) {
} }
} }
static void doDestroyRequest(void *p) { SRequestObj *acquireRequest(int64_t rid) { return (SRequestObj *)taosAcquireRef(clientReqRefPool, rid); }
int32_t releaseRequest(int64_t rid) { return taosReleaseRef(clientReqRefPool, rid); }
int32_t removeRequest(int64_t rid) { return taosRemoveRef(clientReqRefPool, rid); }
void doDestroyRequest(void *p) {
assert(p != NULL); assert(p != NULL);
SRequestObj *pRequest = (SRequestObj *)p; SRequestObj *pRequest = (SRequestObj *)p;
assert(RID_VALID(pRequest->self));
taosHashRemove(pRequest->pTscObj->pRequests, &pRequest->self, sizeof(pRequest->self)); taosHashRemove(pRequest->pTscObj->pRequests, &pRequest->self, sizeof(pRequest->self));
if (pRequest->body.queryJob != 0) { if (pRequest->body.queryJob != 0) {
@ -244,14 +273,15 @@ static void doDestroyRequest(void *p) {
taosMemoryFreeClear(pRequest->pDb); taosMemoryFreeClear(pRequest->pDb);
doFreeReqResultInfo(&pRequest->body.resInfo); doFreeReqResultInfo(&pRequest->body.resInfo);
qDestroyQueryPlan(pRequest->body.pDag);
taosArrayDestroy(pRequest->tableList); taosArrayDestroy(pRequest->tableList);
taosArrayDestroy(pRequest->dbList); taosArrayDestroy(pRequest->dbList);
destroyQueryExecRes(&pRequest->body.resInfo.execRes); destroyQueryExecRes(&pRequest->body.resInfo.execRes);
if (pRequest->self) {
deregisterRequest(pRequest); deregisterRequest(pRequest);
}
taosMemoryFreeClear(pRequest); taosMemoryFreeClear(pRequest);
} }
@ -260,13 +290,9 @@ void destroyRequest(SRequestObj *pRequest) {
return; return;
} }
taosRemoveRef(clientReqRefPool, pRequest->self); removeRequest(pRequest->self);
} }
SRequestObj *acquireRequest(int64_t rid) { return (SRequestObj *)taosAcquireRef(clientReqRefPool, rid); }
int32_t releaseRequest(int64_t rid) { return taosReleaseRef(clientReqRefPool, rid); }
void taos_init_imp(void) { void taos_init_imp(void) {
// In the APIs of other program language, taos_cleanup is not available yet. // In the APIs of other program language, taos_cleanup is not available yet.
// So, to make sure taos_cleanup will be invoked to clean up the allocated resource to suppress the valgrind warning. // So, to make sure taos_cleanup will be invoked to clean up the allocated resource to suppress the valgrind warning.

View File

@ -66,25 +66,31 @@ static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog
if (rsp->vgVersion < 0) { if (rsp->vgVersion < 0) {
code = catalogRemoveDB(pCatalog, rsp->db, rsp->uid); code = catalogRemoveDB(pCatalog, rsp->db, rsp->uid);
} else { } else {
SDBVgInfo vgInfo = {0}; SDBVgInfo *vgInfo = taosMemoryCalloc(1, sizeof(SDBVgInfo));
vgInfo.vgVersion = rsp->vgVersion; if (NULL == vgInfo) {
vgInfo.hashMethod = rsp->hashMethod; return TSDB_CODE_TSC_OUT_OF_MEMORY;
vgInfo.vgHash = taosHashInit(rsp->vgNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); }
if (NULL == vgInfo.vgHash) {
vgInfo->vgVersion = rsp->vgVersion;
vgInfo->hashMethod = rsp->hashMethod;
vgInfo->vgHash = taosHashInit(rsp->vgNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK);
if (NULL == vgInfo->vgHash) {
taosMemoryFree(vgInfo);
tscError("hash init[%d] failed", rsp->vgNum); tscError("hash init[%d] failed", rsp->vgNum);
return TSDB_CODE_TSC_OUT_OF_MEMORY; return TSDB_CODE_TSC_OUT_OF_MEMORY;
} }
for (int32_t j = 0; j < rsp->vgNum; ++j) { for (int32_t j = 0; j < rsp->vgNum; ++j) {
SVgroupInfo *pInfo = taosArrayGet(rsp->pVgroupInfos, j); SVgroupInfo *pInfo = taosArrayGet(rsp->pVgroupInfos, j);
if (taosHashPut(vgInfo.vgHash, &pInfo->vgId, sizeof(int32_t), pInfo, sizeof(SVgroupInfo)) != 0) { if (taosHashPut(vgInfo->vgHash, &pInfo->vgId, sizeof(int32_t), pInfo, sizeof(SVgroupInfo)) != 0) {
tscError("hash push failed, errno:%d", errno); tscError("hash push failed, errno:%d", errno);
taosHashCleanup(vgInfo.vgHash); taosHashCleanup(vgInfo->vgHash);
taosMemoryFree(vgInfo);
return TSDB_CODE_TSC_OUT_OF_MEMORY; return TSDB_CODE_TSC_OUT_OF_MEMORY;
} }
} }
catalogUpdateDBVgInfo(pCatalog, rsp->db, rsp->uid, &vgInfo); catalogUpdateDBVgInfo(pCatalog, rsp->db, rsp->uid, vgInfo);
} }
if (code) { if (code) {
@ -269,8 +275,11 @@ static int32_t hbAsyncCallBack(void *param, const SDataBuf *pMsg, int32_t code)
int32_t rspNum = taosArrayGetSize(pRsp.rsps); int32_t rspNum = taosArrayGetSize(pRsp.rsps);
taosThreadMutexLock(&appInfo.mutex);
SAppInstInfo **pInst = taosHashGet(appInfo.pInstMap, key, strlen(key)); SAppInstInfo **pInst = taosHashGet(appInfo.pInstMap, key, strlen(key));
if (pInst == NULL || NULL == *pInst) { if (pInst == NULL || NULL == *pInst) {
taosThreadMutexUnlock(&appInfo.mutex);
tscError("cluster not exist, key:%s", key); tscError("cluster not exist, key:%s", key);
taosMemoryFreeClear(param); taosMemoryFreeClear(param);
tFreeClientHbBatchRsp(&pRsp); tFreeClientHbBatchRsp(&pRsp);
@ -294,6 +303,8 @@ static int32_t hbAsyncCallBack(void *param, const SDataBuf *pMsg, int32_t code)
} }
} }
taosThreadMutexUnlock(&appInfo.mutex);
tFreeClientHbBatchRsp(&pRsp); tFreeClientHbBatchRsp(&pRsp);
return code; return code;
@ -320,7 +331,7 @@ int32_t hbBuildQueryDesc(SQueryHbReqBasic *hbBasic, STscObj *pObj) {
desc.reqRid = pRequest->self; desc.reqRid = pRequest->self;
desc.stableQuery = pRequest->stableQuery; desc.stableQuery = pRequest->stableQuery;
taosGetFqdn(desc.fqdn); taosGetFqdn(desc.fqdn);
desc.subPlanNum = pRequest->body.pDag ? pRequest->body.pDag->numOfSubplans : 0; desc.subPlanNum = pRequest->body.subplanNum;
if (desc.subPlanNum) { if (desc.subPlanNum) {
desc.subDesc = taosArrayInit(desc.subPlanNum, sizeof(SQuerySubDesc)); desc.subDesc = taosArrayInit(desc.subPlanNum, sizeof(SQuerySubDesc));
@ -790,11 +801,7 @@ SAppHbMgr *appHbMgrInit(SAppInstInfo *pAppInstInfo, char *key) {
return pAppHbMgr; return pAppHbMgr;
} }
void appHbMgrCleanup(void) { void hbFreeAppHbMgr(SAppHbMgr *pTarget) {
int sz = taosArrayGetSize(clientHbMgr.appHbMgrs);
for (int i = 0; i < sz; i++) {
SAppHbMgr *pTarget = taosArrayGetP(clientHbMgr.appHbMgrs, i);
void *pIter = taosHashIterate(pTarget->activeInfo, NULL); void *pIter = taosHashIterate(pTarget->activeInfo, NULL);
while (pIter != NULL) { while (pIter != NULL) {
SClientHbReq *pOneReq = pIter; SClientHbReq *pOneReq = pIter;
@ -806,6 +813,28 @@ void appHbMgrCleanup(void) {
taosMemoryFree(pTarget->key); taosMemoryFree(pTarget->key);
taosMemoryFree(pTarget); taosMemoryFree(pTarget);
}
void hbRemoveAppHbMrg(SAppHbMgr **pAppHbMgr) {
taosThreadMutexLock(&clientHbMgr.lock);
int32_t mgrSize = taosArrayGetSize(clientHbMgr.appHbMgrs);
for (int32_t i = 0; i < mgrSize; ++i) {
SAppHbMgr *pItem = taosArrayGetP(clientHbMgr.appHbMgrs, i);
if (pItem == *pAppHbMgr) {
hbFreeAppHbMgr(*pAppHbMgr);
*pAppHbMgr = NULL;
taosArrayRemove(clientHbMgr.appHbMgrs, i);
break;
}
}
taosThreadMutexUnlock(&clientHbMgr.lock);
}
void appHbMgrCleanup(void) {
int sz = taosArrayGetSize(clientHbMgr.appHbMgrs);
for (int i = 0; i < sz; i++) {
SAppHbMgr *pTarget = taosArrayGetP(clientHbMgr.appHbMgrs, i);
hbFreeAppHbMgr(pTarget);
} }
} }

View File

@ -55,6 +55,18 @@ static char* getClusterKey(const char* user, const char* auth, const char* ip, i
return strdup(key); return strdup(key);
} }
bool chkRequestKilled(void* param) {
bool killed = false;
SRequestObj* pRequest = acquireRequest((int64_t)param);
if (NULL == pRequest || pRequest->killed) {
killed = true;
}
releaseRequest((int64_t)param);
return killed;
}
static STscObj* taosConnectImpl(const char* user, const char* auth, const char* db, __taos_async_fn_t fp, void* param, static STscObj* taosConnectImpl(const char* user, const char* auth, const char* db, __taos_async_fn_t fp, void* param,
SAppInstInfo* pAppInfo, int connType); SAppInstInfo* pAppInfo, int connType);
@ -122,6 +134,9 @@ STscObj* taos_connect_internal(const char* ip, const char* user, const char* pas
p->pTransporter = openTransporter(user, secretEncrypt, tsNumOfCores); p->pTransporter = openTransporter(user, secretEncrypt, tsNumOfCores);
p->pAppHbMgr = appHbMgrInit(p, key); p->pAppHbMgr = appHbMgrInit(p, key);
taosHashPut(appInfo.pInstMap, key, strlen(key), &p, POINTER_BYTES); taosHashPut(appInfo.pInstMap, key, strlen(key), &p, POINTER_BYTES);
p->instKey = key;
key = NULL;
tscDebug("new app inst mgr %p, user:%s, ip:%s, port:%d", p, user, ip, port);
pInst = &p; pInst = &p;
} }
@ -609,58 +624,6 @@ _return:
return code; return code;
} }
int32_t scheduleAsyncQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList) {
tsem_init(&schdRspSem, 0, 0);
SQueryResult res = {.code = 0, .numOfRows = 0};
SRequestConnInfo conn = {.pTrans = pRequest->pTscObj->pAppInfo->pTransporter,
.requestId = pRequest->requestId,
.requestObjRefId = pRequest->self};
SSchedulerReq req = {.pConn = &conn,
.pNodeList = pNodeList,
.pDag = pDag,
.sql = pRequest->sqlstr,
.startTs = pRequest->metric.start,
.fp = schdExecCallback,
.cbParam = &res};
int32_t code = schedulerAsyncExecJob(&req, &pRequest->body.queryJob);
pRequest->body.resInfo.execRes = res.res;
while (true) {
if (code != TSDB_CODE_SUCCESS) {
if (pRequest->body.queryJob != 0) {
schedulerFreeJob(pRequest->body.queryJob, 0);
}
pRequest->code = code;
terrno = code;
return pRequest->code;
} else {
tsem_wait(&schdRspSem);
if (res.code) {
code = res.code;
} else {
break;
}
}
}
if (TDMT_VND_SUBMIT == pRequest->type || TDMT_VND_CREATE_TABLE == pRequest->type) {
pRequest->body.resInfo.numOfRows = res.numOfRows;
if (pRequest->body.queryJob != 0) {
schedulerFreeJob(pRequest->body.queryJob, 0);
}
}
pRequest->code = res.code;
terrno = res.code;
return pRequest->code;
}
int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList) { int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList) {
void* pTransporter = pRequest->pTscObj->pAppInfo->pTransporter; void* pTransporter = pRequest->pTscObj->pAppInfo->pTransporter;
@ -673,9 +636,10 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList
.pDag = pDag, .pDag = pDag,
.sql = pRequest->sqlstr, .sql = pRequest->sqlstr,
.startTs = pRequest->metric.start, .startTs = pRequest->metric.start,
.fp = NULL, .execFp = NULL,
.cbParam = NULL, .execParam = NULL,
.reqKilled = &pRequest->killed}; .chkKillFp = chkRequestKilled,
.chkKillParam = (void*)pRequest->self};
int32_t code = schedulerExecJob(&req, &pRequest->body.queryJob, &res); int32_t code = schedulerExecJob(&req, &pRequest->body.queryJob, &res);
pRequest->body.resInfo.execRes = res.res; pRequest->body.resInfo.execRes = res.res;
@ -875,14 +839,18 @@ SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, bool keepQue
break; break;
case QUERY_EXEC_MODE_SCHEDULE: { case QUERY_EXEC_MODE_SCHEDULE: {
SArray* pMnodeList = taosArrayInit(4, sizeof(SQueryNodeLoad)); SArray* pMnodeList = taosArrayInit(4, sizeof(SQueryNodeLoad));
code = getPlan(pRequest, pQuery, &pRequest->body.pDag, pMnodeList); SQueryPlan* pDag = NULL;
if (TSDB_CODE_SUCCESS == code && !pRequest->validateOnly) { code = getPlan(pRequest, pQuery, &pDag, pMnodeList);
if (TSDB_CODE_SUCCESS == code) {
pRequest->body.subplanNum = pDag->numOfSubplans;
if (!pRequest->validateOnly) {
SArray* pNodeList = NULL; SArray* pNodeList = NULL;
buildSyncExecNodeList(pRequest, &pNodeList, pMnodeList); buildSyncExecNodeList(pRequest, &pNodeList, pMnodeList);
code = scheduleQuery(pRequest, pRequest->body.pDag, pNodeList); code = scheduleQuery(pRequest, pDag, pNodeList);
taosArrayDestroy(pNodeList); taosArrayDestroy(pNodeList);
} }
}
taosArrayDestroy(pMnodeList); taosArrayDestroy(pMnodeList);
break; break;
} }
@ -959,10 +927,13 @@ void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData* pResultM
.pUser = pRequest->pTscObj->user}; .pUser = pRequest->pTscObj->user};
SAppInstInfo* pAppInfo = getAppInfo(pRequest); SAppInstInfo* pAppInfo = getAppInfo(pRequest);
code = qCreateQueryPlan(&cxt, &pRequest->body.pDag, pMnodeList); SQueryPlan* pDag = NULL;
code = qCreateQueryPlan(&cxt, &pDag, pMnodeList);
if (code) { if (code) {
tscError("0x%" PRIx64 " failed to create query plan, code:%s 0x%" PRIx64, pRequest->self, tstrerror(code), tscError("0x%" PRIx64 " failed to create query plan, code:%s 0x%" PRIx64, pRequest->self, tstrerror(code),
pRequest->requestId); pRequest->requestId);
} else {
pRequest->body.subplanNum = pDag->numOfSubplans;
} }
if (TSDB_CODE_SUCCESS == code && !pRequest->validateOnly) { if (TSDB_CODE_SUCCESS == code && !pRequest->validateOnly) {
@ -973,12 +944,13 @@ void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData* pResultM
.pTrans = pAppInfo->pTransporter, .requestId = pRequest->requestId, .requestObjRefId = pRequest->self}; .pTrans = pAppInfo->pTransporter, .requestId = pRequest->requestId, .requestObjRefId = pRequest->self};
SSchedulerReq req = {.pConn = &conn, SSchedulerReq req = {.pConn = &conn,
.pNodeList = pNodeList, .pNodeList = pNodeList,
.pDag = pRequest->body.pDag, .pDag = pDag,
.sql = pRequest->sqlstr, .sql = pRequest->sqlstr,
.startTs = pRequest->metric.start, .startTs = pRequest->metric.start,
.fp = schedulerExecCb, .execFp = schedulerExecCb,
.cbParam = pRequest, .execParam = pRequest,
.reqKilled = &pRequest->killed}; .chkKillFp = chkRequestKilled,
.chkKillParam = (void*)pRequest->self};
code = schedulerAsyncExecJob(&req, &pRequest->body.queryJob); code = schedulerAsyncExecJob(&req, &pRequest->body.queryJob);
taosArrayDestroy(pNodeList); taosArrayDestroy(pNodeList);
} else { } else {
@ -1163,7 +1135,7 @@ STscObj* taosConnectImpl(const char* user, const char* auth, const char* db, __t
taos_close_internal(pTscObj); taos_close_internal(pTscObj);
pTscObj = NULL; pTscObj = NULL;
} else { } else {
tscDebug("0x%" PRIx64 " connection is opening, connId:%u, dnodeConn:%p, reqId:0x%" PRIx64, *(int64_t*)pTscObj->id, tscDebug("0x%" PRIx64 " connection is opening, connId:%u, dnodeConn:%p, reqId:0x%" PRIx64, pTscObj->id,
pTscObj->connId, pTscObj->pAppInfo->pTransporter, pRequest->requestId); pTscObj->connId, pTscObj->pAppInfo->pTransporter, pRequest->requestId);
destroyRequest(pRequest); destroyRequest(pRequest);
} }
@ -1326,7 +1298,9 @@ TAOS* taos_connect_auth(const char* ip, const char* user, const char* auth, cons
STscObj* pObj = taos_connect_internal(ip, user, NULL, auth, db, port, CONN_TYPE__QUERY); STscObj* pObj = taos_connect_internal(ip, user, NULL, auth, db, port, CONN_TYPE__QUERY);
if (pObj) { if (pObj) {
return pObj->id; int64_t* rid = taosMemoryCalloc(1, sizeof(int64_t));
*rid = pObj->id;
return (TAOS*)rid;
} }
return NULL; return NULL;
@ -2001,17 +1975,26 @@ void syncCatalogFn(SMetaData* pResult, void* param, int32_t code) {
void syncQueryFn(void* param, void* res, int32_t code) { void syncQueryFn(void* param, void* res, int32_t code) {
SSyncQueryParam* pParam = param; SSyncQueryParam* pParam = param;
pParam->pRequest = res; pParam->pRequest = res;
if (pParam->pRequest) {
pParam->pRequest->code = code; pParam->pRequest->code = code;
}
tsem_post(&pParam->sem); tsem_post(&pParam->sem);
} }
void taosAsyncQueryImpl(TAOS* taos, const char* sql, __taos_async_fn_t fp, void* param, bool validateOnly) { void taosAsyncQueryImpl(TAOS* taos, const char* sql, __taos_async_fn_t fp, void* param, bool validateOnly) {
STscObj* pTscObj = acquireTscObj(*(int64_t*)taos); if (NULL == taos) {
terrno = TSDB_CODE_TSC_DISCONNECTED;
fp(param, NULL, terrno);
return;
}
int64_t rid = *(int64_t*)taos;
STscObj* pTscObj = acquireTscObj(rid);
if (pTscObj == NULL || sql == NULL || NULL == fp) { if (pTscObj == NULL || sql == NULL || NULL == fp) {
terrno = TSDB_CODE_INVALID_PARA; terrno = TSDB_CODE_INVALID_PARA;
if (pTscObj) { if (pTscObj) {
releaseTscObj(*(int64_t*)taos); releaseTscObj(rid);
} else { } else {
terrno = TSDB_CODE_TSC_DISCONNECTED; terrno = TSDB_CODE_TSC_DISCONNECTED;
} }
@ -2023,6 +2006,7 @@ void taosAsyncQueryImpl(TAOS* taos, const char* sql, __taos_async_fn_t fp, void*
if (sqlLen > (size_t)TSDB_MAX_ALLOWED_SQL_LEN) { if (sqlLen > (size_t)TSDB_MAX_ALLOWED_SQL_LEN) {
tscError("sql string exceeds max length:%d", TSDB_MAX_ALLOWED_SQL_LEN); tscError("sql string exceeds max length:%d", TSDB_MAX_ALLOWED_SQL_LEN);
terrno = TSDB_CODE_TSC_EXCEED_SQL_LIMIT; terrno = TSDB_CODE_TSC_EXCEED_SQL_LIMIT;
releaseTscObj(rid);
fp(param, NULL, terrno); fp(param, NULL, terrno);
return; return;
@ -2032,6 +2016,7 @@ void taosAsyncQueryImpl(TAOS* taos, const char* sql, __taos_async_fn_t fp, void*
int32_t code = buildRequest(pTscObj, sql, sqlLen, &pRequest); int32_t code = buildRequest(pTscObj, sql, sqlLen, &pRequest);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
terrno = code; terrno = code;
releaseTscObj(rid);
fp(param, NULL, terrno); fp(param, NULL, terrno);
return; return;
} }
@ -2040,6 +2025,7 @@ void taosAsyncQueryImpl(TAOS* taos, const char* sql, __taos_async_fn_t fp, void*
pRequest->body.queryFp = fp; pRequest->body.queryFp = fp;
pRequest->body.param = param; pRequest->body.param = param;
doAsyncQuery(pRequest, false); doAsyncQuery(pRequest, false);
releaseTscObj(rid);
} }
TAOS_RES* taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly) { TAOS_RES* taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly) {
@ -2048,7 +2034,8 @@ TAOS_RES* taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly) {
return NULL; return NULL;
} }
STscObj* pTscObj = acquireTscObj(*(int64_t*)taos); int64_t rid = *(int64_t*)taos;
STscObj* pTscObj = acquireTscObj(rid);
if (pTscObj == NULL || sql == NULL) { if (pTscObj == NULL || sql == NULL) {
terrno = TSDB_CODE_TSC_DISCONNECTED; terrno = TSDB_CODE_TSC_DISCONNECTED;
return NULL; return NULL;
@ -2058,16 +2045,16 @@ TAOS_RES* taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly) {
SSyncQueryParam* param = taosMemoryCalloc(1, sizeof(SSyncQueryParam)); SSyncQueryParam* param = taosMemoryCalloc(1, sizeof(SSyncQueryParam));
tsem_init(&param->sem, 0, 0); tsem_init(&param->sem, 0, 0);
taosAsyncQueryImpl(taos, sql, syncQueryFn, param, validateOnly); taosAsyncQueryImpl((TAOS*)&rid, sql, syncQueryFn, param, validateOnly);
tsem_wait(&param->sem); tsem_wait(&param->sem);
releaseTscObj(*(int64_t*)taos); releaseTscObj(rid);
return param->pRequest; return param->pRequest;
#else #else
size_t sqlLen = strlen(sql); size_t sqlLen = strlen(sql);
if (sqlLen > (size_t)TSDB_MAX_ALLOWED_SQL_LEN) { if (sqlLen > (size_t)TSDB_MAX_ALLOWED_SQL_LEN) {
releaseTscObj(*(int64_t*)taos); releaseTscObj(rid);
tscError("sql string exceeds max length:%d", TSDB_MAX_ALLOWED_SQL_LEN); tscError("sql string exceeds max length:%d", TSDB_MAX_ALLOWED_SQL_LEN);
terrno = TSDB_CODE_TSC_EXCEED_SQL_LIMIT; terrno = TSDB_CODE_TSC_EXCEED_SQL_LIMIT;
return NULL; return NULL;
@ -2075,7 +2062,7 @@ TAOS_RES* taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly) {
TAOS_RES* pRes = execQuery(pTscObj, sql, sqlLen, validateOnly); TAOS_RES* pRes = execQuery(pTscObj, sql, sqlLen, validateOnly);
releaseTscObj(*(int64_t*)taos); releaseTscObj(rid);
return pRes; return pRes;
#endif #endif

View File

@ -106,7 +106,9 @@ TAOS *taos_connect(const char *ip, const char *user, const char *pass, const cha
STscObj *pObj = taos_connect_internal(ip, user, pass, NULL, db, port, CONN_TYPE__QUERY); STscObj *pObj = taos_connect_internal(ip, user, pass, NULL, db, port, CONN_TYPE__QUERY);
if (pObj) { if (pObj) {
return pObj->id; int64_t *rid = taosMemoryCalloc(1, sizeof(int64_t));
*rid = pObj->id;
return (TAOS*)rid;
} }
return NULL; return NULL;
@ -118,9 +120,9 @@ void taos_close_internal(void *taos) {
} }
STscObj *pTscObj = (STscObj *)taos; STscObj *pTscObj = (STscObj *)taos;
tscDebug("0x%" PRIx64 " try to close connection, numOfReq:%d", *(int64_t *)pTscObj->id, pTscObj->numOfReqs); tscDebug("0x%" PRIx64 " try to close connection, numOfReq:%d", pTscObj->id, pTscObj->numOfReqs);
taosRemoveRef(clientConnRefPool, *(int64_t *)pTscObj->id); taosRemoveRef(clientConnRefPool, pTscObj->id);
} }
void taos_close(TAOS *taos) { void taos_close(TAOS *taos) {
@ -192,6 +194,17 @@ void taos_free_result(TAOS_RES *res) {
} }
} }
void taos_kill_query(TAOS *taos) {
if (NULL == taos) {
return;
}
int64_t rid = *(int64_t*)taos;
STscObj* pTscObj = acquireTscObj(rid);
closeAllRequests(pTscObj->pRequests);
releaseTscObj(rid);
}
int taos_field_count(TAOS_RES *res) { int taos_field_count(TAOS_RES *res) {
if (res == NULL || TD_RES_TMQ_META(res)) { if (res == NULL || TD_RES_TMQ_META(res)) {
return 0; return 0;
@ -895,6 +908,12 @@ void taos_unsubscribe(TAOS_SUB *tsub, int keepProgress) {
} }
int taos_load_table_info(TAOS *taos, const char *tableNameList) { int taos_load_table_info(TAOS *taos, const char *tableNameList) {
if (NULL == taos) {
terrno = TSDB_CODE_TSC_DISCONNECTED;
return terrno;
}
int64_t rid = *(int64_t*)taos;
const int32_t MAX_TABLE_NAME_LENGTH = 12 * 1024 * 1024; // 12MB list const int32_t MAX_TABLE_NAME_LENGTH = 12 * 1024 * 1024; // 12MB list
int32_t code = 0; int32_t code = 0;
SRequestObj *pRequest = NULL; SRequestObj *pRequest = NULL;
@ -912,7 +931,7 @@ int taos_load_table_info(TAOS *taos, const char *tableNameList) {
return TSDB_CODE_TSC_INVALID_OPERATION; return TSDB_CODE_TSC_INVALID_OPERATION;
} }
STscObj *pTscObj = acquireTscObj(*(int64_t *)taos); STscObj *pTscObj = acquireTscObj(rid);
if (pTscObj == NULL) { if (pTscObj == NULL) {
terrno = TSDB_CODE_TSC_DISCONNECTED; terrno = TSDB_CODE_TSC_DISCONNECTED;
return terrno; return terrno;
@ -957,7 +976,7 @@ _return:
taosArrayDestroy(catalogReq.pTableMeta); taosArrayDestroy(catalogReq.pTableMeta);
destroyRequest(pRequest); destroyRequest(pRequest);
releaseTscObj(*(int64_t *)taos); releaseTscObj(rid);
return code; return code;
} }

View File

@ -77,7 +77,7 @@ int32_t processConnectRsp(void* param, const SDataBuf* pMsg, int32_t code) {
for (int32_t i = 0; i < connectRsp.epSet.numOfEps; ++i) { for (int32_t i = 0; i < connectRsp.epSet.numOfEps; ++i) {
tscDebug("0x%" PRIx64 " epSet.fqdn[%d]:%s port:%d, connObj:0x%" PRIx64, pRequest->requestId, i, tscDebug("0x%" PRIx64 " epSet.fqdn[%d]:%s port:%d, connObj:0x%" PRIx64, pRequest->requestId, i,
connectRsp.epSet.eps[i].fqdn, connectRsp.epSet.eps[i].port, *(int64_t*)pTscObj->id); connectRsp.epSet.eps[i].fqdn, connectRsp.epSet.eps[i].port, pTscObj->id);
} }
pTscObj->connId = connectRsp.connId; pTscObj->connId = connectRsp.connId;
@ -87,11 +87,10 @@ int32_t processConnectRsp(void* param, const SDataBuf* pMsg, int32_t code) {
// update the appInstInfo // update the appInstInfo
pTscObj->pAppInfo->clusterId = connectRsp.clusterId; pTscObj->pAppInfo->clusterId = connectRsp.clusterId;
atomic_add_fetch_64(&pTscObj->pAppInfo->numOfConns, 1);
pTscObj->connType = connectRsp.connType; pTscObj->connType = connectRsp.connType;
hbRegisterConn(pTscObj->pAppInfo->pAppHbMgr, *(int64_t*)pTscObj->id, connectRsp.clusterId, connectRsp.connType); hbRegisterConn(pTscObj->pAppInfo->pAppHbMgr, pTscObj->id, connectRsp.clusterId, connectRsp.connType);
// pRequest->body.resInfo.pRspMsg = pMsg->pData; // pRequest->body.resInfo.pRspMsg = pMsg->pData;
tscDebug("0x%" PRIx64 " clusterId:%" PRId64 ", totalConn:%" PRId64, pRequest->requestId, connectRsp.clusterId, tscDebug("0x%" PRIx64 " clusterId:%" PRId64 ", totalConn:%" PRId64, pRequest->requestId, connectRsp.clusterId,

View File

@ -309,7 +309,7 @@ static int32_t smlApplySchemaAction(SSmlHandle *info, SSchemaAction *action) {
case SCHEMA_ACTION_ADD_COLUMN: { case SCHEMA_ACTION_ADD_COLUMN: {
int n = sprintf(result, "alter stable `%s` add column ", action->alterSTable.sTableName); int n = sprintf(result, "alter stable `%s` add column ", action->alterSTable.sTableName);
smlBuildColumnDescription(action->alterSTable.field, result + n, capacity - n, &outBytes); smlBuildColumnDescription(action->alterSTable.field, result + n, capacity - n, &outBytes);
TAOS_RES *res = taos_query(info->taos->id, result); // TODO async doAsyncQuery TAOS_RES *res = taos_query((TAOS*)&info->taos->id, result); // TODO async doAsyncQuery
code = taos_errno(res); code = taos_errno(res);
const char *errStr = taos_errstr(res); const char *errStr = taos_errstr(res);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
@ -323,7 +323,7 @@ static int32_t smlApplySchemaAction(SSmlHandle *info, SSchemaAction *action) {
case SCHEMA_ACTION_ADD_TAG: { case SCHEMA_ACTION_ADD_TAG: {
int n = sprintf(result, "alter stable `%s` add tag ", action->alterSTable.sTableName); int n = sprintf(result, "alter stable `%s` add tag ", action->alterSTable.sTableName);
smlBuildColumnDescription(action->alterSTable.field, result + n, capacity - n, &outBytes); smlBuildColumnDescription(action->alterSTable.field, result + n, capacity - n, &outBytes);
TAOS_RES *res = taos_query(info->taos->id, result); // TODO async doAsyncQuery TAOS_RES *res = taos_query((TAOS*)&info->taos->id, result); // TODO async doAsyncQuery
code = taos_errno(res); code = taos_errno(res);
const char *errStr = taos_errstr(res); const char *errStr = taos_errstr(res);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
@ -337,7 +337,7 @@ static int32_t smlApplySchemaAction(SSmlHandle *info, SSchemaAction *action) {
case SCHEMA_ACTION_CHANGE_COLUMN_SIZE: { case SCHEMA_ACTION_CHANGE_COLUMN_SIZE: {
int n = sprintf(result, "alter stable `%s` modify column ", action->alterSTable.sTableName); int n = sprintf(result, "alter stable `%s` modify column ", action->alterSTable.sTableName);
smlBuildColumnDescription(action->alterSTable.field, result + n, capacity - n, &outBytes); smlBuildColumnDescription(action->alterSTable.field, result + n, capacity - n, &outBytes);
TAOS_RES *res = taos_query(info->taos->id, result); // TODO async doAsyncQuery TAOS_RES *res = taos_query((TAOS*)&info->taos->id, result); // TODO async doAsyncQuery
code = taos_errno(res); code = taos_errno(res);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
uError("SML:0x%" PRIx64 " apply schema action. error : %s", info->id, taos_errstr(res)); uError("SML:0x%" PRIx64 " apply schema action. error : %s", info->id, taos_errstr(res));
@ -350,7 +350,7 @@ static int32_t smlApplySchemaAction(SSmlHandle *info, SSchemaAction *action) {
case SCHEMA_ACTION_CHANGE_TAG_SIZE: { case SCHEMA_ACTION_CHANGE_TAG_SIZE: {
int n = sprintf(result, "alter stable `%s` modify tag ", action->alterSTable.sTableName); int n = sprintf(result, "alter stable `%s` modify tag ", action->alterSTable.sTableName);
smlBuildColumnDescription(action->alterSTable.field, result + n, capacity - n, &outBytes); smlBuildColumnDescription(action->alterSTable.field, result + n, capacity - n, &outBytes);
TAOS_RES *res = taos_query(info->taos->id, result); // TODO async doAsyncQuery TAOS_RES *res = taos_query((TAOS*)&info->taos->id, result); // TODO async doAsyncQuery
code = taos_errno(res); code = taos_errno(res);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
uError("SML:0x%" PRIx64 " apply schema action. error : %s", info->id, taos_errstr(res)); uError("SML:0x%" PRIx64 " apply schema action. error : %s", info->id, taos_errstr(res));
@ -405,7 +405,7 @@ static int32_t smlApplySchemaAction(SSmlHandle *info, SSchemaAction *action) {
pos--; pos--;
++freeBytes; ++freeBytes;
outBytes = snprintf(pos, freeBytes, ")"); outBytes = snprintf(pos, freeBytes, ")");
TAOS_RES *res = taos_query(info->taos->id, result); TAOS_RES *res = taos_query((TAOS*)&info->taos->id, result);
code = taos_errno(res); code = taos_errno(res);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
uError("SML:0x%" PRIx64 " apply schema action. error : %s", info->id, taos_errstr(res)); uError("SML:0x%" PRIx64 " apply schema action. error : %s", info->id, taos_errstr(res));
@ -2436,7 +2436,13 @@ static void smlInsertCallback(void *param, void *res, int32_t code) {
*/ */
TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int protocol, int precision) { TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int protocol, int precision) {
STscObj* pTscObj = acquireTscObj(*(int64_t*)taos); if (NULL == taos) {
terrno = TSDB_CODE_TSC_DISCONNECTED;
return NULL;
}
int64_t rid = *(int64_t*)taos;
STscObj* pTscObj = acquireTscObj(rid);
if (NULL == pTscObj) { if (NULL == pTscObj) {
terrno = TSDB_CODE_TSC_DISCONNECTED; terrno = TSDB_CODE_TSC_DISCONNECTED;
uError("SML:taos_schemaless_insert invalid taos"); uError("SML:taos_schemaless_insert invalid taos");
@ -2445,7 +2451,7 @@ TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int pr
SRequestObj* request = (SRequestObj*)createRequest(pTscObj, TSDB_SQL_INSERT); SRequestObj* request = (SRequestObj*)createRequest(pTscObj, TSDB_SQL_INSERT);
if(!request){ if(!request){
releaseTscObj(*(int64_t*)taos); releaseTscObj(rid);
uError("SML:taos_schemaless_insert error request is null"); uError("SML:taos_schemaless_insert error request is null");
return NULL; return NULL;
} }
@ -2533,6 +2539,6 @@ end:
// ((STscObj *)taos)->schemalessType = 0; // ((STscObj *)taos)->schemalessType = 0;
pTscObj->schemalessType = 1; pTscObj->schemalessType = 1;
uDebug("resultend:%s", request->msgBuf); uDebug("resultend:%s", request->msgBuf);
releaseTscObj(*(int64_t*)taos); releaseTscObj(rid);
return (TAOS_RES*)request; return (TAOS_RES*)request;
} }

View File

@ -1662,8 +1662,8 @@ char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf)
int32_t colNum = taosArrayGetSize(pDataBlock->pDataBlock); int32_t colNum = taosArrayGetSize(pDataBlock->pDataBlock);
int32_t rows = pDataBlock->info.rows; int32_t rows = pDataBlock->info.rows;
int32_t len = 0; int32_t len = 0;
len += snprintf(dumpBuf + len, size - len, "\n%s |block type %d |child id %d|\n", flag, len += snprintf(dumpBuf + len, size - len, "\n%s |block type %d |child id %d|group id %lu|\n", flag,
(int32_t)pDataBlock->info.type, pDataBlock->info.childId); (int32_t)pDataBlock->info.type, pDataBlock->info.childId, pDataBlock->info.groupId);
for (int32_t j = 0; j < rows; j++) { for (int32_t j = 0; j < rows; j++) {
len += snprintf(dumpBuf + len, size - len, "%s |", flag); len += snprintf(dumpBuf + len, size - len, "%s |", flag);
for (int32_t k = 0; k < colNum; k++) { for (int32_t k = 0; k < colNum; k++) {

View File

@ -107,13 +107,7 @@ static void vmProcessSyncQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOf
const STraceId *trace = &pMsg->info.traceId; const STraceId *trace = &pMsg->info.traceId;
dGTrace("vgId:%d, msg:%p get from vnode-sync queue", pVnode->vgId, pMsg); dGTrace("vgId:%d, msg:%p get from vnode-sync queue", pVnode->vgId, pMsg);
int32_t code = vnodeProcessSyncReq(pVnode->pImpl, pMsg, NULL); int32_t code = vnodeProcessSyncReq(pVnode->pImpl, pMsg, NULL); // no response here
if (code != 0) {
if (terrno != 0) code = terrno;
dGError("vgId:%d, msg:%p failed to sync since %s", pVnode->vgId, pMsg, terrstr());
vmSendRsp(pMsg, code);
}
dGTrace("vgId:%d, msg:%p is freed, code:0x%x", pVnode->vgId, pMsg, code); dGTrace("vgId:%d, msg:%p is freed, code:0x%x", pVnode->vgId, pMsg, code);
rpcFreeCont(pMsg->pCont); rpcFreeCont(pMsg->pCont);
taosFreeQitem(pMsg); taosFreeQitem(pMsg);

View File

@ -251,9 +251,6 @@ static inline void dmReleaseHandle(SRpcHandleInfo *pHandle, int8_t type) {
static bool rpcRfp(int32_t code, tmsg_t msgType) { static bool rpcRfp(int32_t code, tmsg_t msgType) {
if (code == TSDB_CODE_RPC_REDIRECT || code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_NODE_NOT_DEPLOYED || if (code == TSDB_CODE_RPC_REDIRECT || code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_NODE_NOT_DEPLOYED ||
code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_APP_NOT_READY) { code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_APP_NOT_READY) {
if (msgType == TDMT_VND_QUERY || msgType == TDMT_VND_FETCH) {
return false;
}
return true; return true;
} else { } else {
return false; return false;
@ -264,7 +261,7 @@ int32_t dmInitClient(SDnode *pDnode) {
SDnodeTrans *pTrans = &pDnode->trans; SDnodeTrans *pTrans = &pDnode->trans;
SRpcInit rpcInit = {0}; SRpcInit rpcInit = {0};
rpcInit.label = "DND"; rpcInit.label = "DND-C";
rpcInit.numOfThreads = 1; rpcInit.numOfThreads = 1;
rpcInit.cfp = (RpcCfp)dmProcessRpcMsg; rpcInit.cfp = (RpcCfp)dmProcessRpcMsg;
rpcInit.sessions = 1024; rpcInit.sessions = 1024;
@ -298,7 +295,7 @@ int32_t dmInitServer(SDnode *pDnode) {
SRpcInit rpcInit = {0}; SRpcInit rpcInit = {0};
strncpy(rpcInit.localFqdn, tsLocalFqdn, strlen(tsLocalFqdn)); strncpy(rpcInit.localFqdn, tsLocalFqdn, strlen(tsLocalFqdn));
rpcInit.localPort = tsServerPort; rpcInit.localPort = tsServerPort;
rpcInit.label = "DND"; rpcInit.label = "DND-S";
rpcInit.numOfThreads = tsNumOfRpcThreads; rpcInit.numOfThreads = tsNumOfRpcThreads;
rpcInit.cfp = (RpcCfp)dmProcessRpcMsg; rpcInit.cfp = (RpcCfp)dmProcessRpcMsg;
rpcInit.sessions = tsMaxShellConns; rpcInit.sessions = tsMaxShellConns;

View File

@ -527,6 +527,11 @@ int32_t mndProcessSyncMsg(SRpcMsg *pMsg) {
static int32_t mndCheckMnodeState(SRpcMsg *pMsg) { static int32_t mndCheckMnodeState(SRpcMsg *pMsg) {
if (!IsReq(pMsg)) return 0; if (!IsReq(pMsg)) return 0;
if (pMsg->msgType == TDMT_VND_QUERY || pMsg->msgType == TDMT_VND_QUERY_CONTINUE ||
pMsg->msgType == TDMT_VND_QUERY_HEARTBEAT || pMsg->msgType == TDMT_VND_FETCH ||
pMsg->msgType == TDMT_VND_DROP_TASK) {
return 0;
}
if (mndAcquireRpcRef(pMsg->info.node) == 0) return 0; if (mndAcquireRpcRef(pMsg->info.node) == 0) return 0;
if (pMsg->msgType == TDMT_MND_MQ_TIMER || pMsg->msgType == TDMT_MND_TELEM_TIMER || if (pMsg->msgType == TDMT_MND_MQ_TIMER || pMsg->msgType == TDMT_MND_TELEM_TIMER ||
pMsg->msgType == TDMT_MND_TRANS_TIMER || pMsg->msgType == TDMT_MND_TTL_TIMER) { pMsg->msgType == TDMT_MND_TRANS_TIMER || pMsg->msgType == TDMT_MND_TTL_TIMER) {

View File

@ -136,7 +136,7 @@ int32_t vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp
if (vnodeProcessDropTbReq(pVnode, version, pReq, len, pRsp) < 0) goto _err; if (vnodeProcessDropTbReq(pVnode, version, pReq, len, pRsp) < 0) goto _err;
break; break;
case TDMT_VND_DROP_TTL_TABLE: case TDMT_VND_DROP_TTL_TABLE:
if (vnodeProcessDropTtlTbReq(pVnode, version, pReq, len, pRsp) < 0) goto _err; //if (vnodeProcessDropTtlTbReq(pVnode, version, pReq, len, pRsp) < 0) goto _err;
break; break;
case TDMT_VND_CREATE_SMA: { case TDMT_VND_CREATE_SMA: {
if (vnodeProcessCreateTSmaReq(pVnode, version, pReq, len, pRsp) < 0) goto _err; if (vnodeProcessCreateTSmaReq(pVnode, version, pReq, len, pRsp) < 0) goto _err;

View File

@ -73,7 +73,7 @@ static int32_t vnodeSetStandBy(SVnode *pVnode) {
vInfo("vgId:%d, set standby success", TD_VID(pVnode)); vInfo("vgId:%d, set standby success", TD_VID(pVnode));
return 0; return 0;
} else { } else {
vError("vgId:%d, failed to set standby since %s", TD_VID(pVnode), terrstr()); vError("vgId:%d, failed to set standby after leader transfer since %s", TD_VID(pVnode), terrstr());
return -1; return -1;
} }
} }

View File

@ -1127,13 +1127,15 @@ int32_t catalogGetExpiredUsers(SCatalog* pCtg, SUserAuthVersion** users, uint32_
} }
*num = taosHashGetSize(pCtg->userCache); *num = taosHashGetSize(pCtg->userCache);
if (*num > 0) { if (*num <= 0) {
CTG_API_LEAVE(TSDB_CODE_SUCCESS);
}
*users = taosMemoryCalloc(*num, sizeof(SUserAuthVersion)); *users = taosMemoryCalloc(*num, sizeof(SUserAuthVersion));
if (NULL == *users) { if (NULL == *users) {
ctgError("calloc %d userAuthVersion failed", *num); ctgError("calloc %d userAuthVersion failed", *num);
CTG_API_LEAVE(TSDB_CODE_OUT_OF_MEMORY); CTG_API_LEAVE(TSDB_CODE_OUT_OF_MEMORY);
} }
}
uint32_t i = 0; uint32_t i = 0;
SCtgUserAuth* pAuth = taosHashIterate(pCtg->userCache, NULL); SCtgUserAuth* pAuth = taosHashIterate(pCtg->userCache, NULL);
@ -1144,6 +1146,11 @@ int32_t catalogGetExpiredUsers(SCatalog* pCtg, SUserAuthVersion** users, uint32_
(*users)[i].user[len] = 0; (*users)[i].user[len] = 0;
(*users)[i].version = pAuth->version; (*users)[i].version = pAuth->version;
++i; ++i;
if (i >= *num) {
taosHashCancelIterate(pCtg->userCache, pAuth);
break;
}
pAuth = taosHashIterate(pCtg->userCache, pAuth); pAuth = taosHashIterate(pCtg->userCache, pAuth);
} }

View File

@ -826,6 +826,9 @@ _return:
pJob->jobResCode = code; pJob->jobResCode = code;
//taosSsleep(2);
//qDebug("QID:0x%" PRIx64 " ctg after sleep", pJob->queryId);
taosAsyncExec(ctgCallUserCb, pJob, NULL); taosAsyncExec(ctgCallUserCb, pJob, NULL);
CTG_RET(code); CTG_RET(code);

View File

@ -1083,7 +1083,7 @@ int32_t ctgMetaRentUpdate(SCtgRentMgmt *mgmt, void *meta, int64_t id, int32_t si
CTG_LOCK(CTG_WRITE, &slot->lock); CTG_LOCK(CTG_WRITE, &slot->lock);
if (NULL == slot->meta) { if (NULL == slot->meta) {
qError("empty meta slot, id:0x%"PRIx64", slot idx:%d, type:%d", id, widx, mgmt->type); qDebug("empty meta slot, id:0x%"PRIx64", slot idx:%d, type:%d", id, widx, mgmt->type);
CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
} }
@ -1536,8 +1536,6 @@ void ctgClearAllInstance(void) {
pIter = taosHashIterate(gCtgMgmt.pCluster, pIter); pIter = taosHashIterate(gCtgMgmt.pCluster, pIter);
} }
taosHashClear(gCtgMgmt.pCluster);
} }
void ctgFreeAllInstance(void) { void ctgFreeAllInstance(void) {
@ -1566,27 +1564,27 @@ int32_t ctgOpUpdateVgroup(SCtgCacheOperation *operation) {
SCatalog* pCtg = msg->pCtg; SCatalog* pCtg = msg->pCtg;
if (NULL == dbInfo->vgHash) { if (NULL == dbInfo->vgHash) {
return TSDB_CODE_SUCCESS; goto _return;
} }
if (dbInfo->vgVersion < 0 || taosHashGetSize(dbInfo->vgHash) <= 0) { if (dbInfo->vgVersion < 0 || taosHashGetSize(dbInfo->vgHash) <= 0) {
ctgError("invalid db vgInfo, dbFName:%s, vgHash:%p, vgVersion:%d, vgHashSize:%d", ctgError("invalid db vgInfo, dbFName:%s, vgHash:%p, vgVersion:%d, vgHashSize:%d",
dbFName, dbInfo->vgHash, dbInfo->vgVersion, taosHashGetSize(dbInfo->vgHash)); dbFName, dbInfo->vgHash, dbInfo->vgVersion, taosHashGetSize(dbInfo->vgHash));
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); CTG_ERR_JRET(TSDB_CODE_APP_ERROR);
} }
bool newAdded = false; bool newAdded = false;
SDbVgVersion vgVersion = {.dbId = msg->dbId, .vgVersion = dbInfo->vgVersion, .numOfTable = dbInfo->numOfTable}; SDbVgVersion vgVersion = {.dbId = msg->dbId, .vgVersion = dbInfo->vgVersion, .numOfTable = dbInfo->numOfTable};
SCtgDBCache *dbCache = NULL; SCtgDBCache *dbCache = NULL;
CTG_ERR_RET(ctgGetAddDBCache(msg->pCtg, dbFName, msg->dbId, &dbCache)); CTG_ERR_JRET(ctgGetAddDBCache(msg->pCtg, dbFName, msg->dbId, &dbCache));
if (NULL == dbCache) { if (NULL == dbCache) {
ctgInfo("conflict db update, ignore this update, dbFName:%s, dbId:0x%"PRIx64, dbFName, msg->dbId); ctgInfo("conflict db update, ignore this update, dbFName:%s, dbId:0x%"PRIx64, dbFName, msg->dbId);
CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
} }
SCtgVgCache *vgCache = &dbCache->vgCache; SCtgVgCache *vgCache = &dbCache->vgCache;
CTG_ERR_RET(ctgWLockVgInfo(msg->pCtg, dbCache)); CTG_ERR_JRET(ctgWLockVgInfo(msg->pCtg, dbCache));
if (vgCache->vgInfo) { if (vgCache->vgInfo) {
SDBVgInfo *vgInfo = vgCache->vgInfo; SDBVgInfo *vgInfo = vgCache->vgInfo;
@ -1595,14 +1593,14 @@ int32_t ctgOpUpdateVgroup(SCtgCacheOperation *operation) {
ctgDebug("db vgVer is old, dbFName:%s, vgVer:%d, curVer:%d", dbFName, dbInfo->vgVersion, vgInfo->vgVersion); ctgDebug("db vgVer is old, dbFName:%s, vgVer:%d, curVer:%d", dbFName, dbInfo->vgVersion, vgInfo->vgVersion);
ctgWUnlockVgInfo(dbCache); ctgWUnlockVgInfo(dbCache);
return TSDB_CODE_SUCCESS; goto _return;
} }
if (dbInfo->vgVersion == vgInfo->vgVersion && dbInfo->numOfTable == vgInfo->numOfTable) { if (dbInfo->vgVersion == vgInfo->vgVersion && dbInfo->numOfTable == vgInfo->numOfTable) {
ctgDebug("no new db vgVer or numOfTable, dbFName:%s, vgVer:%d, numOfTable:%d", dbFName, dbInfo->vgVersion, dbInfo->numOfTable); ctgDebug("no new db vgVer or numOfTable, dbFName:%s, vgVer:%d, numOfTable:%d", dbFName, dbInfo->vgVersion, dbInfo->numOfTable);
ctgWUnlockVgInfo(dbCache); ctgWUnlockVgInfo(dbCache);
return TSDB_CODE_SUCCESS; goto _return;
} }
ctgFreeVgInfo(vgInfo); ctgFreeVgInfo(vgInfo);

View File

@ -19,7 +19,7 @@
#include "catalogInt.h" #include "catalogInt.h"
extern SCatalogMgmt gCtgMgmt; extern SCatalogMgmt gCtgMgmt;
SCtgDebug gCTGDebug = {.lockEnable = true, .apiEnable = true}; SCtgDebug gCTGDebug = {0};
void ctgdUserCallback(SMetaData* pResult, void* param, int32_t code) { void ctgdUserCallback(SMetaData* pResult, void* param, int32_t code) {
ASSERT(*(int32_t*)param == 1); ASSERT(*(int32_t*)param == 1);

View File

@ -253,18 +253,15 @@ typedef struct STableScanInfo {
SReadHandle readHandle; SReadHandle readHandle;
SFileBlockLoadRecorder readRecorder; SFileBlockLoadRecorder readRecorder;
int64_t numOfRows;
SScanInfo scanInfo; SScanInfo scanInfo;
int32_t scanTimes; int32_t scanTimes;
SNode* pFilterNode; // filter info, which is push down by optimizer SNode* pFilterNode; // filter info, which is push down by optimizer
SqlFunctionCtx* pCtx; // which belongs to the direct upstream operator operator query context SqlFunctionCtx* pCtx; // which belongs to the direct upstream operator operator query context,todo: remove this by using SExprSup
SResultRowInfo* pResultRowInfo; int32_t* rowEntryInfoOffset; // todo: remove this by using SExprSup
int32_t* rowEntryInfoOffset; SExprInfo* pExpr;// todo: remove this by using SExprSup
SExprInfo* pExpr;
SSDataBlock* pResBlock; SSDataBlock* pResBlock;
SArray* pColMatchInfo; SArray* pColMatchInfo;
int32_t numOfOutput;
SExprSupp pseudoSup; SExprSupp pseudoSup;
SQueryTableDataCond cond; SQueryTableDataCond cond;
int32_t scanFlag; // table scan flag to denote if it is a repeat/reverse/main scan int32_t scanFlag; // table scan flag to denote if it is a repeat/reverse/main scan
@ -275,8 +272,13 @@ typedef struct STableScanInfo {
int32_t curTWinIdx; int32_t curTWinIdx;
int32_t currentGroupId; int32_t currentGroupId;
uint64_t queryId; uint64_t queryId; // todo remove it
uint64_t taskId; uint64_t taskId; // todo remove it
struct {
uint64_t uid;
int64_t t;
} scanStatus;
} STableScanInfo; } STableScanInfo;
typedef struct STagScanInfo { typedef struct STagScanInfo {
@ -321,31 +323,31 @@ typedef struct SessionWindowSupporter {
} SessionWindowSupporter; } SessionWindowSupporter;
typedef struct SStreamBlockScanInfo { typedef struct SStreamBlockScanInfo {
uint64_t tableUid; // queried super table uid
SExprInfo* pPseudoExpr;
int32_t numOfPseudoExpr;
int32_t primaryTsIndex; // primary time stamp slot id
SReadHandle readHandle;
SInterval interval; // if the upstream is an interval operator, the interval info is also kept here.
SArray* pColMatchInfo; //
SNode* pCondition;
SArray* pBlockLists; // multiple SSDatablock. SArray* pBlockLists; // multiple SSDatablock.
SSDataBlock* pRes; // result SSDataBlock SSDataBlock* pRes; // result SSDataBlock
SSDataBlock* pUpdateRes; // update SSDataBlock SSDataBlock* pUpdateRes; // update SSDataBlock
int32_t updateResIndex; int32_t updateResIndex;
int32_t blockType; // current block type int32_t blockType; // current block type
int32_t validBlockIndex; // Is current data has returned? int32_t validBlockIndex; // Is current data has returned?
SColumnInfo* pCols; // the output column info
uint64_t numOfExec; // execution times uint64_t numOfExec; // execution times
void* streamBlockReader;// stream block reader handle void* streamBlockReader;// stream block reader handle
SArray* pColMatchInfo; //
SNode* pCondition;
int32_t tsArrayIndex; int32_t tsArrayIndex;
SArray* tsArray; SArray* tsArray;
uint64_t groupId; uint64_t groupId;
SUpdateInfo* pUpdateInfo; SUpdateInfo* pUpdateInfo;
SExprInfo* pPseudoExpr;
int32_t numOfPseudoExpr;
int32_t primaryTsIndex; // primary time stamp slot id
SReadHandle readHandle;
uint64_t tableUid; // queried super table uid
EStreamScanMode scanMode; EStreamScanMode scanMode;
SOperatorInfo* pSnapshotReadOp; SOperatorInfo* pSnapshotReadOp;
SInterval interval; // if the upstream is an interval operator, the interval info is also kept here.
SArray* childIds; SArray* childIds;
SessionWindowSupporter sessionSup; SessionWindowSupporter sessionSup;
bool assignBlockUid; // assign block uid to groupId, temporarily used for generating rollup SMA. bool assignBlockUid; // assign block uid to groupId, temporarily used for generating rollup SMA.
@ -416,6 +418,7 @@ typedef struct SIntervalAggOperatorInfo {
STimeWindowAggSupp twAggSup; STimeWindowAggSupp twAggSup;
bool invertible; bool invertible;
SArray* pPrevValues; // SArray<SGroupKeys> used to keep the previous not null value for interpolation. SArray* pPrevValues; // SArray<SGroupKeys> used to keep the previous not null value for interpolation.
bool ignoreCloseWindow;
} SIntervalAggOperatorInfo; } SIntervalAggOperatorInfo;
typedef struct SStreamFinalIntervalOperatorInfo { typedef struct SStreamFinalIntervalOperatorInfo {
@ -437,6 +440,7 @@ typedef struct SStreamFinalIntervalOperatorInfo {
SArray* pPullWins; // SPullWindowInfo SArray* pPullWins; // SPullWindowInfo
int32_t pullIndex; int32_t pullIndex;
SSDataBlock* pPullDataRes; SSDataBlock* pPullDataRes;
bool ignoreCloseWindow;
} SStreamFinalIntervalOperatorInfo; } SStreamFinalIntervalOperatorInfo;
typedef struct SAggOperatorInfo { typedef struct SAggOperatorInfo {
@ -574,6 +578,7 @@ typedef struct SStreamSessionAggOperatorInfo {
SArray* pChildren; // cache for children's result; final stream operator SArray* pChildren; // cache for children's result; final stream operator
SPhysiNode* pPhyNode; // create new child SPhysiNode* pPhyNode; // create new child
bool isFinal; bool isFinal;
bool ignoreCloseWindow;
} SStreamSessionAggOperatorInfo; } SStreamSessionAggOperatorInfo;
typedef struct STimeSliceOperatorInfo { typedef struct STimeSliceOperatorInfo {
@ -617,6 +622,7 @@ typedef struct SStreamStateAggOperatorInfo {
void* pDelIterator; void* pDelIterator;
SArray* pScanWindow; SArray* pScanWindow;
SArray* pChildren; // cache for children's result; SArray* pChildren; // cache for children's result;
bool ignoreCloseWindow;
} SStreamStateAggOperatorInfo; } SStreamStateAggOperatorInfo;
typedef struct SSortedMergeOperatorInfo { typedef struct SSortedMergeOperatorInfo {
@ -683,7 +689,7 @@ int32_t appendDownstream(SOperatorInfo* p, SOperatorInfo** pDownstream, int32_t
void initBasicInfo(SOptrBasicInfo* pInfo, SSDataBlock* pBlock); void initBasicInfo(SOptrBasicInfo* pInfo, SSDataBlock* pBlock);
void cleanupBasicInfo(SOptrBasicInfo* pInfo); void cleanupBasicInfo(SOptrBasicInfo* pInfo);
int32_t initExprSupp(SExprSupp* pSup, SExprInfo* pExprInfo, int32_t numOfExpr); int32_t initExprSupp(SExprSupp* pSup, SExprInfo* pExprInfo, int32_t numOfExpr);
void cleanupExprSup(SExprSupp* pSup); void cleanupExprSupp(SExprSupp* pSup);
int32_t initAggInfo(SExprSupp *pSup, SAggSupporter* pAggSup, SExprInfo* pExprInfo, int32_t numOfCols, size_t keyBufSize, int32_t initAggInfo(SExprSupp *pSup, SAggSupporter* pAggSup, SExprInfo* pExprInfo, int32_t numOfCols, size_t keyBufSize,
const char* pkey); const char* pkey);
void initResultSizeInfo(SOperatorInfo* pOperator, int32_t numOfRows); void initResultSizeInfo(SOperatorInfo* pOperator, int32_t numOfRows);
@ -707,7 +713,7 @@ void destroyBasicOperatorInfo(void* param, int32_t numOfOutput);
void appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle); void appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle);
void setTbNameColData(void* pMeta, const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, int32_t functionId); void setTbNameColData(void* pMeta, const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, int32_t functionId);
void cleanupExecSupp(SExprSupp* pSupp); int32_t doGetScanStatus(SOperatorInfo* pOperator, uint64_t* uid, int64_t* ts);
SSDataBlock* loadNextDataBlock(void* param); SSDataBlock* loadNextDataBlock(void* param);

View File

@ -191,16 +191,6 @@ int32_t qAsyncKillTask(qTaskInfo_t qinfo) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t qIsTaskCompleted(qTaskInfo_t qinfo) {
SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)qinfo;
if (pTaskInfo == NULL) {
return TSDB_CODE_QRY_INVALID_QHANDLE;
}
return isTaskKilled(pTaskInfo);
}
void qDestroyTask(qTaskInfo_t qTaskHandle) { void qDestroyTask(qTaskInfo_t qTaskHandle) {
SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)qTaskHandle; SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)qTaskHandle;
qDebug("%s execTask completed, numOfRows:%" PRId64, GET_TASKID(pTaskInfo), pTaskInfo->pRoot->resultInfo.totalRows); qDebug("%s execTask completed, numOfRows:%" PRId64, GET_TASKID(pTaskInfo), pTaskInfo->pRoot->resultInfo.totalRows);
@ -242,3 +232,10 @@ int32_t qDeserializeTaskStatus(qTaskInfo_t tinfo, const char* pInput, int32_t le
} }
int32_t qGetStreamScanStatus(qTaskInfo_t tinfo, uint64_t* uid, int64_t* ts) {
SExecTaskInfo* pTaskInfo = (SExecTaskInfo*) tinfo;
return TSDB_CODE_SUCCESS;
}

View File

@ -1033,7 +1033,7 @@ static uint32_t doFilterByBlockTimeWindow(STableScanInfo* pTableScanInfo, SSData
SqlFunctionCtx* pCtx = pTableScanInfo->pCtx; SqlFunctionCtx* pCtx = pTableScanInfo->pCtx;
uint32_t status = BLK_DATA_NOT_LOAD; uint32_t status = BLK_DATA_NOT_LOAD;
int32_t numOfOutput = pTableScanInfo->numOfOutput; int32_t numOfOutput = 0;//pTableScanInfo->numOfOutput;
for (int32_t i = 0; i < numOfOutput; ++i) { for (int32_t i = 0; i < numOfOutput; ++i) {
int32_t functionId = pCtx[i].functionId; int32_t functionId = pCtx[i].functionId;
int32_t colId = pTableScanInfo->pExpr[i].base.pParam[0].pCol->colId; int32_t colId = pTableScanInfo->pExpr[i].base.pParam[0].pCol->colId;
@ -2821,6 +2821,24 @@ int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scan
} }
} }
int32_t doGetScanStatus(SOperatorInfo* pOperator, uint64_t* uid, int64_t* ts) {
int32_t type = pOperator->operatorType;
if (type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
SStreamBlockScanInfo* pScanInfo = pOperator->info;
STableScanInfo* pSnapShotScanInfo = pScanInfo->pSnapshotReadOp->info;
*uid = pSnapShotScanInfo->scanStatus.uid;
*ts = pSnapShotScanInfo->scanStatus.t;
} else {
if (pOperator->pDownstream[0] == NULL) {
return TSDB_CODE_INVALID_PARA;
} else {
doGetScanStatus(pOperator->pDownstream[0], uid, ts);
}
}
return TSDB_CODE_SUCCESS;
}
// this is a blocking operator // this is a blocking operator
static int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) { static int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) {
if (OPTR_IS_OPENED(pOperator)) { if (OPTR_IS_OPENED(pOperator)) {
@ -3543,7 +3561,7 @@ static void destroyProjectOperatorInfo(void* param, int32_t numOfOutput) {
taosArrayDestroy(pInfo->pPseudoColInfo); taosArrayDestroy(pInfo->pPseudoColInfo);
} }
void cleanupExecSupp(SExprSupp* pSupp) { void cleanupExprSupp(SExprSupp* pSupp) {
destroySqlFunctionCtx(pSupp->pCtx, pSupp->numOfExprs); destroySqlFunctionCtx(pSupp->pCtx, pSupp->numOfExprs);
destroyExprInfo(pSupp->pExprInfo, pSupp->numOfExprs); destroyExprInfo(pSupp->pExprInfo, pSupp->numOfExprs);
@ -3556,7 +3574,7 @@ static void destroyIndefinitOperatorInfo(void* param, int32_t numOfOutput) {
taosArrayDestroy(pInfo->pPseudoColInfo); taosArrayDestroy(pInfo->pPseudoColInfo);
cleanupAggSup(&pInfo->aggSup); cleanupAggSup(&pInfo->aggSup);
cleanupExecSupp(&pInfo->scalarSup); cleanupExprSupp(&pInfo->scalarSup);
} }
void destroyExchangeOperatorInfo(void* param, int32_t numOfOutput) { void destroyExchangeOperatorInfo(void* param, int32_t numOfOutput) {

View File

@ -37,7 +37,7 @@ static void destroyGroupOperatorInfo(void* param, int32_t numOfOutput) {
taosMemoryFreeClear(pInfo->keyBuf); taosMemoryFreeClear(pInfo->keyBuf);
taosArrayDestroy(pInfo->pGroupCols); taosArrayDestroy(pInfo->pGroupCols);
taosArrayDestroy(pInfo->pGroupColVals); taosArrayDestroy(pInfo->pGroupColVals);
cleanupExecSupp(&pInfo->scalarSup); cleanupExprSupp(&pInfo->scalarSup);
} }
static int32_t initGroupOptrInfo(SArray** pGroupColVals, int32_t* keyLen, char** keyBuf, const SArray* pGroupColList) { static int32_t initGroupOptrInfo(SArray** pGroupColVals, int32_t* keyLen, char** keyBuf, const SArray* pGroupColList) {
@ -701,7 +701,7 @@ static void destroyPartitionOperatorInfo(void* param, int32_t numOfOutput) {
taosHashCleanup(pInfo->pGroupSet); taosHashCleanup(pInfo->pGroupSet);
taosMemoryFree(pInfo->columnOffset); taosMemoryFree(pInfo->columnOffset);
cleanupExecSupp(&pInfo->scalarSup); cleanupExprSupp(&pInfo->scalarSup);
} }
SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartitionPhysiNode* pPartNode, SExecTaskInfo* pTaskInfo) { SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartitionPhysiNode* pPartNode, SExecTaskInfo* pTaskInfo) {

View File

@ -13,6 +13,7 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#include <executorimpl.h>
#include <vnode.h> #include <vnode.h>
#include "filter.h" #include "filter.h"
#include "function.h" #include "function.h"
@ -413,6 +414,11 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) {
pTableScanInfo->readRecorder.elapsedTime += (taosGetTimestampUs() - st) / 1000.0; pTableScanInfo->readRecorder.elapsedTime += (taosGetTimestampUs() - st) / 1000.0;
pOperator->cost.totalCost = pTableScanInfo->readRecorder.elapsedTime; pOperator->cost.totalCost = pTableScanInfo->readRecorder.elapsedTime;
// todo refactor
pTableScanInfo->scanStatus.uid = pBlock->info.uid;
pTableScanInfo->scanStatus.t = pBlock->info.window.ekey;
return pBlock; return pBlock;
} }
return NULL; return NULL;
@ -459,7 +465,7 @@ static SSDataBlock* doTableScanGroup(SOperatorInfo* pOperator) {
int32_t total = pTableScanInfo->scanInfo.numOfAsc + pTableScanInfo->scanInfo.numOfDesc; int32_t total = pTableScanInfo->scanInfo.numOfAsc + pTableScanInfo->scanInfo.numOfDesc;
if (pTableScanInfo->scanTimes < total) { if (pTableScanInfo->scanTimes < total) {
if (pTableScanInfo->cond.order == TSDB_ORDER_ASC) { if (pTableScanInfo->cond.order == TSDB_ORDER_ASC) {
prepareForDescendingScan(pTableScanInfo, pTableScanInfo->pCtx, pTableScanInfo->numOfOutput); prepareForDescendingScan(pTableScanInfo, pTableScanInfo->pCtx, 0);
tsdbResetReadHandle(pTableScanInfo->dataReader, &pTableScanInfo->cond, 0); tsdbResetReadHandle(pTableScanInfo->dataReader, &pTableScanInfo->cond, 0);
pTableScanInfo->curTWinIdx = 0; pTableScanInfo->curTWinIdx = 0;
} }

View File

@ -813,6 +813,16 @@ static void removeResults(SArray* pWins, SArray* pUpdated) {
} }
} }
bool isOverdue(TSKEY ts, STimeWindowAggSupp* pSup) {
ASSERT(pSup->maxTs == INT64_MIN || pSup->maxTs > 0);
return pSup->maxTs != INT64_MIN && ts < pSup->maxTs - pSup->waterMark;
}
bool isCloseWindow(STimeWindow* pWin, STimeWindowAggSupp* pSup) {
return isOverdue(pWin->ekey, pSup);
}
static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pBlock, static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pBlock,
int32_t scanFlag, SArray* pUpdated) { int32_t scanFlag, SArray* pUpdated) {
SIntervalAggOperatorInfo* pInfo = (SIntervalAggOperatorInfo*)pOperatorInfo->info; SIntervalAggOperatorInfo* pInfo = (SIntervalAggOperatorInfo*)pOperatorInfo->info;
@ -830,15 +840,16 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul
STimeWindow win = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, STimeWindow win = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval,
pInfo->interval.precision, &pInfo->win); pInfo->interval.precision, &pInfo->win);
int32_t ret = TSDB_CODE_SUCCESS;
int32_t ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, if (!pInfo->ignoreCloseWindow || !isCloseWindow(&win, &pInfo->twAggSup)) {
ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId,
pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo); pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
if (ret != TSDB_CODE_SUCCESS || pResult == NULL) { if (ret != TSDB_CODE_SUCCESS || pResult == NULL) {
longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
} }
if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM) { if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM &&
if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) { pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) {
saveResultRow(pResult, tableGroupId, pUpdated); saveResultRow(pResult, tableGroupId, pUpdated);
} }
} }
@ -864,9 +875,11 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul
doWindowBorderInterpolation(pInfo, pBlock, pResult, &win, startPos, forwardRows, pSup); doWindowBorderInterpolation(pInfo, pBlock, pResult, &win, startPos, forwardRows, pSup);
} }
if (!pInfo->ignoreCloseWindow || !isCloseWindow(&win, &pInfo->twAggSup)) {
updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &win, true); updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &win, true);
doApplyFunctions(pTaskInfo, pSup->pCtx, &win, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, tsCols, doApplyFunctions(pTaskInfo, pSup->pCtx, &win, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, tsCols,
pBlock->info.rows, numOfOutput, pInfo->order); pBlock->info.rows, numOfOutput, pInfo->order);
}
doCloseWindow(pResultRowInfo, pInfo, pResult); doCloseWindow(pResultRowInfo, pInfo, pResult);
@ -877,6 +890,12 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul
if (startPos < 0) { if (startPos < 0) {
break; break;
} }
if (pInfo->ignoreCloseWindow && isCloseWindow(&nextWin, &pInfo->twAggSup)) {
ekey = ascScan ? nextWin.ekey : nextWin.skey;
forwardRows =
getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, pInfo->order);
continue;
}
// null data, failed to allocate more memory buffer // null data, failed to allocate more memory buffer
int32_t code = setTimeWindowOutputBuf(pResultRowInfo, &nextWin, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, int32_t code = setTimeWindowOutputBuf(pResultRowInfo, &nextWin, (scanFlag == MAIN_SCAN), &pResult, tableGroupId,
@ -885,11 +904,10 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul
longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
} }
if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM) { if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM &&
if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) { pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) {
saveResultRow(pResult, tableGroupId, pUpdated); saveResultRow(pResult, tableGroupId, pUpdated);
} }
}
ekey = ascScan ? nextWin.ekey : nextWin.skey; ekey = ascScan ? nextWin.ekey : nextWin.skey;
forwardRows = forwardRows =
@ -1292,11 +1310,6 @@ static int32_t getAllIntervalWindow(SHashObj* pHashMap, SArray* resWins) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
bool isCloseWindow(STimeWindow* pWin, STimeWindowAggSupp* pSup) {
ASSERT(pSup->maxTs == INT64_MIN || pSup->maxTs > 0);
return pSup->maxTs != INT64_MIN && pWin->ekey < pSup->maxTs - pSup->waterMark;
}
static int32_t closeIntervalWindow(SHashObj* pHashMap, STimeWindowAggSupp* pSup, static int32_t closeIntervalWindow(SHashObj* pHashMap, STimeWindowAggSupp* pSup,
SInterval* pInterval, SHashObj* pPullDataMap, SArray* closeWins) { SInterval* pInterval, SHashObj* pPullDataMap, SArray* closeWins) {
void* pIte = NULL; void* pIte = NULL;
@ -1411,7 +1424,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) {
doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
pOperator->status = OP_RES_TO_RETURN; pOperator->status = OP_RES_TO_RETURN;
printDataBlock(pInfo->binfo.pRes, "single interval");
return pInfo->binfo.pRes->info.rows == 0 ? NULL : pInfo->binfo.pRes; return pInfo->binfo.pRes->info.rows == 0 ? NULL : pInfo->binfo.pRes;
} }
@ -1521,6 +1534,7 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo*
pInfo->interval = *pInterval; pInfo->interval = *pInterval;
pInfo->execModel = pTaskInfo->execModel; pInfo->execModel = pTaskInfo->execModel;
pInfo->twAggSup = *pTwAggSupp; pInfo->twAggSup = *pTwAggSupp;
pInfo->ignoreCloseWindow = false;
if (pPhyNode->window.pExprs != NULL) { if (pPhyNode->window.pExprs != NULL) {
int32_t numOfScalar = 0; int32_t numOfScalar = 0;
@ -2276,7 +2290,15 @@ static void doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBloc
STimeWindow nextWin = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, STimeWindow nextWin = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval,
pInfo->interval.precision, NULL); pInfo->interval.precision, NULL);
while (1) { while (1) {
if (IS_FINAL_OP(pInfo) && isCloseWindow(&nextWin, &pInfo->twAggSup) && pInfo->pChildren) { bool isClosed = isCloseWindow(&nextWin, &pInfo->twAggSup);
if (pInfo->ignoreCloseWindow && isClosed) {
startPos = getNexWindowPos(&pInfo->interval, &pSDataBlock->info, tsCols, startPos, nextWin.ekey, &nextWin);
if (startPos < 0) {
break;
}
continue;
}
if (IS_FINAL_OP(pInfo) && isClosed && pInfo->pChildren) {
bool ignore = true; bool ignore = true;
SWinRes winRes = {.ts = nextWin.skey, .groupId = tableGroupId,}; SWinRes winRes = {.ts = nextWin.skey, .groupId = tableGroupId,};
void* chIds = taosHashGet(pInfo->pPullDataMap, &winRes, sizeof(SWinRes)); void* chIds = taosHashGet(pInfo->pPullDataMap, &winRes, sizeof(SWinRes));
@ -2684,6 +2706,7 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream,
_hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
pInfo->pPullDataMap = taosHashInit(64, hashFn, false, HASH_NO_LOCK); pInfo->pPullDataMap = taosHashInit(64, hashFn, false, HASH_NO_LOCK);
pInfo->pPullDataRes = createPullDataBlock(); pInfo->pPullDataRes = createPullDataBlock();
pInfo->ignoreCloseWindow = false;
pOperator->operatorType = pPhyNode->type; pOperator->operatorType = pPhyNode->type;
pOperator->blocking = true; pOperator->blocking = true;
@ -2830,6 +2853,7 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh
pInfo->pChildren = NULL; pInfo->pChildren = NULL;
pInfo->isFinal = false; pInfo->isFinal = false;
pInfo->pPhyNode = pPhyNode; pInfo->pPhyNode = pPhyNode;
pInfo->ignoreCloseWindow = false;
pOperator->name = "StreamSessionWindowAggOperator"; pOperator->name = "StreamSessionWindowAggOperator";
pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION;
@ -3007,6 +3031,9 @@ static int32_t doOneWindowAggImpl(int32_t tsColId, SOptrBasicInfo* pBinfo, SStre
updateTimeWindowInfo(pTimeWindowData, &pCurWin->win, false); updateTimeWindowInfo(pTimeWindowData, &pCurWin->win, false);
doApplyFunctions(pTaskInfo, pSup->pCtx, &pCurWin->win, pTimeWindowData, startIndex, winRows, tsCols, doApplyFunctions(pTaskInfo, pSup->pCtx, &pCurWin->win, pTimeWindowData, startIndex, winRows, tsCols,
pSDataBlock->info.rows, numOutput, TSDB_ORDER_ASC); pSDataBlock->info.rows, numOutput, TSDB_ORDER_ASC);
SFilePage* bufPage = getBufPage(pAggSup->pResultBuf, pCurWin->pos.pageId);
setBufPageDirty(bufPage, true);
releaseBufPage(pAggSup->pResultBuf, bufPage);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -3063,7 +3090,13 @@ void compactTimeWindow(SStreamSessionAggOperatorInfo* pInfo, int32_t startIndex,
pWinInfo->isOutput = false; pWinInfo->isOutput = false;
} }
taosArrayRemove(pInfo->streamAggSup.pCurWins, i); taosArrayRemove(pInfo->streamAggSup.pCurWins, i);
SFilePage* tmpPage = getBufPage(pInfo->streamAggSup.pResultBuf, pWinInfo->pos.pageId);
releaseBufPage(pInfo->streamAggSup.pResultBuf, tmpPage);
} }
SFilePage* bufPage = getBufPage(pInfo->streamAggSup.pResultBuf, pCurWin->pos.pageId);
ASSERT(num > 0);
setBufPageDirty(bufPage, true);
releaseBufPage(pInfo->streamAggSup.pResultBuf, bufPage);
} }
static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, SHashObj* pStUpdated, static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, SHashObj* pStUpdated,
@ -3083,7 +3116,7 @@ static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSData
SResultRow* pResult = NULL; SResultRow* pResult = NULL;
int32_t winRows = 0; int32_t winRows = 0;
if (pSDataBlock->pDataBlock != NULL) { ASSERT(pSDataBlock->pDataBlock);
SColumnInfoData* pStartTsCol = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex); SColumnInfoData* pStartTsCol = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);
startTsCols = (int64_t*)pStartTsCol->pData; startTsCols = (int64_t*)pStartTsCol->pData;
SColumnInfoData* pEndTsCol = NULL; SColumnInfoData* pEndTsCol = NULL;
@ -3093,12 +3126,13 @@ static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSData
pEndTsCol = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex); pEndTsCol = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);
} }
endTsCols = (int64_t*)pEndTsCol->pData; endTsCols = (int64_t*)pEndTsCol->pData;
} else {
return;
}
SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
for (int32_t i = 0; i < pSDataBlock->info.rows;) { for (int32_t i = 0; i < pSDataBlock->info.rows;) {
if (pInfo->ignoreCloseWindow && isOverdue(endTsCols[i], &pInfo->twAggSup)) {
i++;
continue;
}
int32_t winIndex = 0; int32_t winIndex = 0;
SResultWindowInfo* pCurWin = getSessionTimeWindow(pAggSup, startTsCols[i], endTsCols[i], groupId, gap, &winIndex); SResultWindowInfo* pCurWin = getSessionTimeWindow(pAggSup, startTsCols[i], endTsCols[i], groupId, gap, &winIndex);
winRows = winRows =
@ -3205,17 +3239,24 @@ static void rebuildTimeWindow(SStreamSessionAggOperatorInfo* pInfo, SArray* pWin
index = 0; index = 0;
} }
for (int32_t k = index; k < chWinSize; k++) { for (int32_t k = index; k < chWinSize; k++) {
SResultWindowInfo* pcw = taosArrayGet(pChWins, k); SResultWindowInfo* pChWin = taosArrayGet(pChWins, k);
if (pParentWin->win.skey <= pcw->win.skey && pcw->win.ekey <= pParentWin->win.ekey) { if (pParentWin->win.skey <= pChWin->win.skey && pChWin->win.ekey <= pParentWin->win.ekey) {
SResultRow* pChResult = NULL; SResultRow* pChResult = NULL;
setWindowOutputBuf(pcw, &pChResult, pChild->exprSupp.pCtx, groupId, numOfOutput, setWindowOutputBuf(pChWin, &pChResult, pChild->exprSupp.pCtx, groupId, numOfOutput,
pChild->exprSupp.rowEntryInfoOffset, &pChInfo->streamAggSup, pTaskInfo); pChild->exprSupp.rowEntryInfoOffset, &pChInfo->streamAggSup, pTaskInfo);
compactFunctions(pSup->pCtx, pChild->exprSupp.pCtx, numOfOutput, pTaskInfo); compactFunctions(pSup->pCtx, pChild->exprSupp.pCtx, numOfOutput, pTaskInfo);
SFilePage* bufPage = getBufPage(pInfo->streamAggSup.pResultBuf, pChWin->pos.pageId);
setBufPageDirty(bufPage, true);
releaseBufPage(pInfo->streamAggSup.pResultBuf, bufPage);
continue; continue;
} }
break; break;
} }
} }
SFilePage* bufPage = getBufPage(pInfo->streamAggSup.pResultBuf, pParentWin->pos.pageId);
ASSERT(size > 0);
setBufPageDirty(bufPage, true);
releaseBufPage(pInfo->streamAggSup.pResultBuf, bufPage);
} }
} }
@ -3234,7 +3275,7 @@ int32_t closeSessionWindow(SHashObj* pHashMap, STimeWindowAggSupp* pTwSup, SArra
for (int32_t i = 0; i < size; i++) { for (int32_t i = 0; i < size; i++) {
void* pWin = taosArrayGet(pWins, i); void* pWin = taosArrayGet(pWins, i);
SResultWindowInfo* pSeWin = fn(pWin); SResultWindowInfo* pSeWin = fn(pWin);
if (pSeWin->win.ekey < pTwSup->maxTs - pTwSup->waterMark) { if (isCloseWindow(&pSeWin->win, pTwSup)) {
if (!pSeWin->isClosed) { if (!pSeWin->isClosed) {
pSeWin->isClosed = true; pSeWin->isClosed = true;
if (pTwSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) { if (pTwSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) {
@ -3745,6 +3786,10 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
SColumnInfoData* pKeyColInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->stateCol.slotId); SColumnInfoData* pKeyColInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->stateCol.slotId);
for (int32_t i = 0; i < pSDataBlock->info.rows; i += winRows) { for (int32_t i = 0; i < pSDataBlock->info.rows; i += winRows) {
if (pInfo->ignoreCloseWindow && isOverdue(tsCols[i], &pInfo->twAggSup)) {
i++;
continue;
}
char* pKeyData = colDataGetData(pKeyColInfo, i); char* pKeyData = colDataGetData(pKeyColInfo, i);
int32_t winIndex = 0; int32_t winIndex = 0;
bool allEqual = true; bool allEqual = true;
@ -3895,6 +3940,7 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys
pInfo->pDelRes->info.type = STREAM_DELETE; pInfo->pDelRes->info.type = STREAM_DELETE;
blockDataEnsureCapacity(pInfo->pDelRes, 64); blockDataEnsureCapacity(pInfo->pDelRes, 64);
pInfo->pChildren = NULL; pInfo->pChildren = NULL;
pInfo->ignoreCloseWindow = false;
pOperator->name = "StreamStateAggOperator"; pOperator->name = "StreamStateAggOperator";
pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE;

View File

@ -1770,7 +1770,7 @@ int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, char* tN
} }
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
SSchema* pSchema = pDataBlock->pTableMeta->schema; SSchema* pSchema = getTableTagSchema(pDataBlock->pTableMeta);
bool isJson = false; bool isJson = false;
STag* pTag = NULL; STag* pTag = NULL;

View File

@ -936,6 +936,25 @@ EDealRes sclCalcWalker(SNode* pNode, void* pContext) {
return DEAL_RES_ERROR; return DEAL_RES_ERROR;
} }
int32_t sclExtendResRows(SScalarParam *pDst, SScalarParam *pSrc, SArray *pBlockList) {
SSDataBlock* pb = taosArrayGetP(pBlockList, 0);
SScalarParam *pLeft = taosMemoryCalloc(1, sizeof(SScalarParam));
if (NULL == pLeft) {
sclError("calloc %d failed", (int32_t)sizeof(SScalarParam));
SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
}
pLeft->numOfRows = pb->info.rows;
colInfoDataEnsureCapacity(pDst->columnData, pb->info.rows);
_bin_scalar_fn_t OperatorFn = getBinScalarOperatorFn(OP_TYPE_ASSIGN);
OperatorFn(pLeft, pSrc, pDst, TSDB_ORDER_ASC);
taosMemoryFree(pLeft);
return TSDB_CODE_SUCCESS;
}
int32_t scalarCalculateConstants(SNode *pNode, SNode **pRes) { int32_t scalarCalculateConstants(SNode *pNode, SNode **pRes) {
if (NULL == pNode) { if (NULL == pNode) {
SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
@ -983,9 +1002,14 @@ int32_t scalarCalculate(SNode *pNode, SArray *pBlockList, SScalarParam *pDst) {
SCL_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); SCL_ERR_JRET(TSDB_CODE_QRY_APP_ERROR);
} }
if (1 == res->numOfRows) {
SCL_ERR_JRET(sclExtendResRows(pDst, res, pBlockList));
} else {
colInfoDataEnsureCapacity(pDst->columnData, res->numOfRows); colInfoDataEnsureCapacity(pDst->columnData, res->numOfRows);
colDataAssign(pDst->columnData, res->columnData, res->numOfRows, NULL); colDataAssign(pDst->columnData, res->columnData, res->numOfRows, NULL);
pDst->numOfRows = res->numOfRows; pDst->numOfRows = res->numOfRows;
}
taosHashRemove(ctx.pRes, (void *)&pNode, POINTER_BYTES); taosHashRemove(ctx.pRes, (void *)&pNode, POINTER_BYTES);
} }

View File

@ -99,8 +99,8 @@ typedef struct SSchStat {
typedef struct SSchResInfo { typedef struct SSchResInfo {
SQueryResult* queryRes; SQueryResult* queryRes;
void** fetchRes; void** fetchRes;
schedulerExecCallback execFp; schedulerExecFp execFp;
schedulerFetchCallback fetchFp; schedulerFetchFp fetchFp;
void* userParam; void* userParam;
} SSchResInfo; } SSchResInfo;
@ -212,15 +212,13 @@ typedef struct SSchJob {
SRequestConnInfo conn; SRequestConnInfo conn;
SArray *nodeList; // qnode/vnode list, SArray<SQueryNodeLoad> SArray *nodeList; // qnode/vnode list, SArray<SQueryNodeLoad>
SArray *levels; // starting from 0. SArray<SSchLevel> SArray *levels; // starting from 0. SArray<SSchLevel>
SNodeList *subPlans; // subplan pointer copied from DAG, no need to free it in scheduler SQueryPlan *pDag;
SArray *dataSrcTasks; // SArray<SQueryTask*> SArray *dataSrcTasks; // SArray<SQueryTask*>
int32_t levelIdx; int32_t levelIdx;
SEpSet dataSrcEps; SEpSet dataSrcEps;
SHashObj *taskList; SHashObj *taskList;
SHashObj *execTasks; // executing tasks, key:taskid, value:SQueryTask* SHashObj *execTasks; // executing and executed tasks, key:taskid, value:SQueryTask*
SHashObj *succTasks; // succeed tasks, key:taskid, value:SQueryTask*
SHashObj *failTasks; // failed tasks, key:taskid, value:SQueryTask*
SHashObj *flowCtrl; // key is ep, element is SSchFlowControl SHashObj *flowCtrl; // key is ep, element is SSchFlowControl
SExplainCtx *explainCtx; SExplainCtx *explainCtx;
@ -228,7 +226,8 @@ typedef struct SSchJob {
SQueryNodeAddr resNode; SQueryNodeAddr resNode;
tsem_t rspSem; tsem_t rspSem;
SSchOpStatus opStatus; SSchOpStatus opStatus;
bool *reqKilled; schedulerChkKillFp chkKillFp;
void* chkKillParam;
SSchTask *fetchTask; SSchTask *fetchTask;
int32_t errCode; int32_t errCode;
SRWLatch resLock; SRWLatch resLock;
@ -368,7 +367,7 @@ void schFreeRpcCtxVal(const void *arg);
int32_t schMakeBrokenLinkVal(SSchJob *pJob, SSchTask *pTask, SRpcBrokenlinkVal *brokenVal, bool isHb); int32_t schMakeBrokenLinkVal(SSchJob *pJob, SSchTask *pTask, SRpcBrokenlinkVal *brokenVal, bool isHb);
int32_t schAppendTaskExecNode(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, int32_t execIdx); int32_t schAppendTaskExecNode(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, int32_t execIdx);
int32_t schExecStaticExplainJob(SSchedulerReq *pReq, int64_t *job, bool sync); int32_t schExecStaticExplainJob(SSchedulerReq *pReq, int64_t *job, bool sync);
int32_t schExecJobImpl(SSchedulerReq *pReq, int64_t *job, SQueryResult* pRes, bool sync); int32_t schExecJobImpl(SSchedulerReq *pReq, SSchJob *pJob, bool sync);
int32_t schUpdateJobStatus(SSchJob *pJob, int8_t newStatus); int32_t schUpdateJobStatus(SSchJob *pJob, int8_t newStatus);
int32_t schCancelJob(SSchJob *pJob); int32_t schCancelJob(SSchJob *pJob);
int32_t schProcessOnJobDropped(SSchJob *pJob, int32_t errCode); int32_t schProcessOnJobDropped(SSchJob *pJob, int32_t errCode);
@ -383,6 +382,8 @@ int32_t schProcessOnTaskStatusRsp(SQueryNodeEpId* pEpId, SArray* pStatusList);
void schFreeSMsgSendInfo(SMsgSendInfo *msgSendInfo); void schFreeSMsgSendInfo(SMsgSendInfo *msgSendInfo);
char* schGetOpStr(SCH_OP_TYPE type); char* schGetOpStr(SCH_OP_TYPE type);
int32_t schBeginOperation(SSchJob *pJob, SCH_OP_TYPE type, bool sync); int32_t schBeginOperation(SSchJob *pJob, SCH_OP_TYPE type, bool sync);
int32_t schInitJob(SSchedulerReq *pReq, SSchJob **pSchJob);
int32_t schSetJobQueryRes(SSchJob* pJob, SQueryResult* pRes);
#ifdef __cplusplus #ifdef __cplusplus

View File

@ -42,7 +42,7 @@ int32_t schInitTask(SSchJob *pJob, SSchTask *pTask, SSubplan *pPlan, SSchLevel *
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t schInitJob(SSchedulerReq *pReq, SSchJob **pSchJob, SQueryResult* pRes, bool syncSchedule) { int32_t schInitJob(SSchedulerReq *pReq, SSchJob **pSchJob) {
int32_t code = 0; int32_t code = 0;
int64_t refId = -1; int64_t refId = -1;
SSchJob *pJob = taosMemoryCalloc(1, sizeof(SSchJob)); SSchJob *pJob = taosMemoryCalloc(1, sizeof(SSchJob));
@ -54,12 +54,15 @@ int32_t schInitJob(SSchedulerReq *pReq, SSchJob **pSchJob, SQueryResult* pRes, b
pJob->attr.explainMode = pReq->pDag->explainInfo.mode; pJob->attr.explainMode = pReq->pDag->explainInfo.mode;
pJob->conn = *pReq->pConn; pJob->conn = *pReq->pConn;
pJob->sql = pReq->sql; pJob->sql = pReq->sql;
pJob->reqKilled = pReq->reqKilled; pJob->pDag = pReq->pDag;
pJob->userRes.queryRes = pRes; pJob->chkKillFp = pReq->chkKillFp;
pJob->userRes.execFp = pReq->fp; pJob->chkKillParam = pReq->chkKillParam;
pJob->userRes.userParam = pReq->cbParam; pJob->userRes.execFp = pReq->execFp;
pJob->userRes.userParam = pReq->execParam;
if (pReq->pNodeList != NULL) { if (pReq->pNodeList == NULL || taosArrayGetSize(pReq->pNodeList) <= 0) {
qDebug("QID:0x%" PRIx64 " input exec nodeList is empty", pReq->pDag->queryId);
} else {
pJob->nodeList = taosArrayDup(pReq->pNodeList); pJob->nodeList = taosArrayDup(pReq->pNodeList);
} }
@ -83,20 +86,6 @@ int32_t schInitJob(SSchedulerReq *pReq, SSchJob **pSchJob, SQueryResult* pRes, b
SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY);
} }
pJob->succTasks =
taosHashInit(pReq->pDag->numOfSubplans, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_ENTRY_LOCK);
if (NULL == pJob->succTasks) {
SCH_JOB_ELOG("taosHashInit %d succTasks failed", pReq->pDag->numOfSubplans);
SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY);
}
pJob->failTasks =
taosHashInit(pReq->pDag->numOfSubplans, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_ENTRY_LOCK);
if (NULL == pJob->failTasks) {
SCH_JOB_ELOG("taosHashInit %d failTasks failed", pReq->pDag->numOfSubplans);
SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY);
}
tsem_init(&pJob->rspSem, 0, 0); tsem_init(&pJob->rspSem, 0, 0);
refId = taosAddRef(schMgmt.jobRef, pJob); refId = taosAddRef(schMgmt.jobRef, pJob);
@ -194,7 +183,7 @@ FORCE_INLINE bool schJobNeedToStop(SSchJob *pJob, int8_t *pStatus) {
*pStatus = status; *pStatus = status;
} }
if (*pJob->reqKilled) { if ((*pJob->chkKillFp)(pJob->chkKillParam)) {
schUpdateJobStatus(pJob, JOB_TASK_STATUS_DROPPING); schUpdateJobStatus(pJob, JOB_TASK_STATUS_DROPPING);
schUpdateJobErrCode(pJob, TSDB_CODE_TSC_QUERY_KILLED); schUpdateJobErrCode(pJob, TSDB_CODE_TSC_QUERY_KILLED);
@ -547,8 +536,6 @@ int32_t schValidateAndBuildJob(SQueryPlan *pDag, SSchJob *pJob) {
pJob->levelNum = levelNum; pJob->levelNum = levelNum;
pJob->levelIdx = levelNum - 1; pJob->levelIdx = levelNum - 1;
pJob->subPlans = pDag->pSubplans;
SSchLevel level = {0}; SSchLevel level = {0};
SNodeListNode *plans = NULL; SNodeListNode *plans = NULL;
int32_t taskNum = 0; int32_t taskNum = 0;
@ -724,6 +711,7 @@ int32_t schPushTaskToExecList(SSchJob *pJob, SSchTask *pTask) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
/*
int32_t schMoveTaskToSuccList(SSchJob *pJob, SSchTask *pTask, bool *moved) { int32_t schMoveTaskToSuccList(SSchJob *pJob, SSchTask *pTask, bool *moved) {
if (0 != taosHashRemove(pJob->execTasks, &pTask->taskId, sizeof(pTask->taskId))) { if (0 != taosHashRemove(pJob->execTasks, &pTask->taskId, sizeof(pTask->taskId))) {
SCH_TASK_WLOG("remove task from execTask list failed, may not exist, status:%s", SCH_GET_TASK_STATUS_STR(pTask)); SCH_TASK_WLOG("remove task from execTask list failed, may not exist, status:%s", SCH_GET_TASK_STATUS_STR(pTask));
@ -801,6 +789,7 @@ int32_t schMoveTaskToExecList(SSchJob *pJob, SSchTask *pTask, bool *moved) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
*/
int32_t schTaskCheckSetRetry(SSchJob *pJob, SSchTask *pTask, int32_t errCode, bool *needRetry) { int32_t schTaskCheckSetRetry(SSchJob *pJob, SSchTask *pTask, int32_t errCode, bool *needRetry) {
int8_t status = 0; int8_t status = 0;
@ -1047,9 +1036,7 @@ int32_t schProcessOnTaskFailure(SSchJob *pJob, SSchTask *pTask, int32_t errCode)
if (!needRetry) { if (!needRetry) {
SCH_TASK_ELOG("task failed and no more retry, code:%s", tstrerror(errCode)); SCH_TASK_ELOG("task failed and no more retry, code:%s", tstrerror(errCode));
if (SCH_GET_TASK_STATUS(pTask) == JOB_TASK_STATUS_EXECUTING) { if (SCH_GET_TASK_STATUS(pTask) != JOB_TASK_STATUS_EXECUTING) {
SCH_ERR_JRET(schMoveTaskToFailList(pJob, pTask, &moved));
} else {
SCH_TASK_ELOG("task not in executing list, status:%s", SCH_GET_TASK_STATUS_STR(pTask)); SCH_TASK_ELOG("task not in executing list, status:%s", SCH_GET_TASK_STATUS_STR(pTask));
SCH_ERR_JRET(TSDB_CODE_SCH_STATUS_ERROR); SCH_ERR_JRET(TSDB_CODE_SCH_STATUS_ERROR);
} }
@ -1115,8 +1102,6 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) {
SCH_LOG_TASK_END_TS(pTask); SCH_LOG_TASK_END_TS(pTask);
SCH_ERR_JRET(schMoveTaskToSuccList(pJob, pTask, &moved));
SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_PARTIAL_SUCCEED); SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_PARTIAL_SUCCEED);
SCH_ERR_JRET(schRecordTaskSucceedNode(pJob, pTask)); SCH_ERR_JRET(schRecordTaskSucceedNode(pJob, pTask));
@ -1150,8 +1135,6 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) {
pJob->fetchTask = pTask; pJob->fetchTask = pTask;
SCH_ERR_JRET(schMoveTaskToExecList(pJob, pTask, &moved));
SCH_RET(schProcessOnJobPartialSuccess(pJob)); SCH_RET(schProcessOnJobPartialSuccess(pJob));
} }
@ -1466,8 +1449,8 @@ void schDropTaskInHashList(SSchJob *pJob, SHashObj *list) {
void schDropJobAllTasks(SSchJob *pJob) { void schDropJobAllTasks(SSchJob *pJob) {
schDropTaskInHashList(pJob, pJob->execTasks); schDropTaskInHashList(pJob, pJob->execTasks);
schDropTaskInHashList(pJob, pJob->succTasks); // schDropTaskInHashList(pJob, pJob->succTasks);
schDropTaskInHashList(pJob, pJob->failTasks); // schDropTaskInHashList(pJob, pJob->failTasks);
} }
int32_t schCancelJob(SSchJob *pJob) { int32_t schCancelJob(SSchJob *pJob) {
@ -1491,8 +1474,6 @@ void schFreeJobImpl(void *job) {
schDropJobAllTasks(pJob); schDropJobAllTasks(pJob);
pJob->subPlans = NULL; // it is a reference to pDag->pSubplans
int32_t numOfLevels = taosArrayGetSize(pJob->levels); int32_t numOfLevels = taosArrayGetSize(pJob->levels);
for (int32_t i = 0; i < numOfLevels; ++i) { for (int32_t i = 0; i < numOfLevels; ++i) {
SSchLevel *pLevel = taosArrayGet(pJob->levels, i); SSchLevel *pLevel = taosArrayGet(pJob->levels, i);
@ -1509,8 +1490,8 @@ void schFreeJobImpl(void *job) {
schFreeFlowCtrl(pJob); schFreeFlowCtrl(pJob);
taosHashCleanup(pJob->execTasks); taosHashCleanup(pJob->execTasks);
taosHashCleanup(pJob->failTasks); // taosHashCleanup(pJob->failTasks);
taosHashCleanup(pJob->succTasks); // taosHashCleanup(pJob->succTasks);
taosHashCleanup(pJob->taskList); taosHashCleanup(pJob->taskList);
taosArrayDestroy(pJob->levels); taosArrayDestroy(pJob->levels);
@ -1521,6 +1502,8 @@ void schFreeJobImpl(void *job) {
destroyQueryExecRes(&pJob->execRes); destroyQueryExecRes(&pJob->execRes);
qDestroyQueryPlan(pJob->pDag);
taosMemoryFreeClear(pJob->userRes.queryRes); taosMemoryFreeClear(pJob->userRes.queryRes);
taosMemoryFreeClear(pJob->resData); taosMemoryFreeClear(pJob->resData);
taosMemoryFree(pJob); taosMemoryFree(pJob);
@ -1533,88 +1516,11 @@ void schFreeJobImpl(void *job) {
} }
} }
int32_t schExecJobImpl(SSchedulerReq *pReq, int64_t *job, SQueryResult* pRes, bool sync) { int32_t schLaunchStaticExplainJob(SSchedulerReq *pReq, SSchJob *pJob, bool sync) {
if (pReq->pNodeList == NULL || taosArrayGetSize(pReq->pNodeList) <= 0) {
qDebug("QID:0x%" PRIx64 " input exec nodeList is empty", pReq->pDag->queryId);
}
int32_t code = 0;
SSchJob *pJob = NULL;
SCH_ERR_JRET(schInitJob(pReq, &pJob, pRes, sync));
qDebug("QID:0x%" PRIx64 " sch job refId 0x%"PRIx64 " started", pReq->pDag->queryId, pJob->refId);
*job = pJob->refId;
SCH_ERR_JRET(schBeginOperation(pJob, SCH_OP_EXEC, sync));
code = schLaunchJob(pJob);
if (sync) {
SCH_JOB_DLOG("will wait for rsp now, job status:%s", SCH_GET_JOB_STATUS_STR(pJob));
tsem_wait(&pJob->rspSem);
schEndOperation(pJob);
} else if (code) {
schPostJobRes(pJob, SCH_OP_EXEC);
}
SCH_JOB_DLOG("job exec done, job status:%s, jobId:0x%" PRIx64, SCH_GET_JOB_STATUS_STR(pJob), pJob->refId);
schReleaseJob(pJob->refId);
SCH_RET(code);
_return:
if (!sync) {
pReq->fp(NULL, pReq->cbParam, code);
}
schReleaseJob(pJob->refId);
SCH_RET(code);
}
int32_t schExecJob(SSchedulerReq *pReq, int64_t *pJob, SQueryResult *pRes) {
int32_t code = 0;
*pJob = 0;
if (EXPLAIN_MODE_STATIC == pReq->pDag->explainInfo.mode) {
SCH_ERR_JRET(schExecStaticExplainJob(pReq, pJob, true));
} else {
SCH_ERR_JRET(schExecJobImpl(pReq, pJob, NULL, true));
}
_return:
if (*pJob) {
SSchJob *job = schAcquireJob(*pJob);
schSetJobQueryRes(job, pRes);
schReleaseJob(*pJob);
}
return code;
}
int32_t schAsyncExecJob(SSchedulerReq *pReq, int64_t *pJob) {
int32_t code = 0;
*pJob = 0;
if (EXPLAIN_MODE_STATIC == pReq->pDag->explainInfo.mode) {
SCH_RET(schExecStaticExplainJob(pReq, pJob, false));
}
SCH_ERR_RET(schExecJobImpl(pReq, pJob, NULL, false));
return code;
}
int32_t schExecStaticExplainJob(SSchedulerReq *pReq, int64_t *job, bool sync) {
qDebug("QID:0x%" PRIx64 " job started", pReq->pDag->queryId); qDebug("QID:0x%" PRIx64 " job started", pReq->pDag->queryId);
int32_t code = 0; int32_t code = 0;
/*
SSchJob *pJob = taosMemoryCalloc(1, sizeof(SSchJob)); SSchJob *pJob = taosMemoryCalloc(1, sizeof(SSchJob));
if (NULL == pJob) { if (NULL == pJob) {
qError("QID:0x%" PRIx64 " calloc %d failed", pReq->pDag->queryId, (int32_t)sizeof(SSchJob)); qError("QID:0x%" PRIx64 " calloc %d failed", pReq->pDag->queryId, (int32_t)sizeof(SSchJob));
@ -1625,10 +1531,10 @@ int32_t schExecStaticExplainJob(SSchedulerReq *pReq, int64_t *job, bool sync) {
pJob->sql = pReq->sql; pJob->sql = pReq->sql;
pJob->reqKilled = pReq->reqKilled; pJob->reqKilled = pReq->reqKilled;
pJob->pDag = pReq->pDag;
pJob->attr.queryJob = true; pJob->attr.queryJob = true;
pJob->attr.explainMode = pReq->pDag->explainInfo.mode; pJob->attr.explainMode = pReq->pDag->explainInfo.mode;
pJob->queryId = pReq->pDag->queryId; pJob->queryId = pReq->pDag->queryId;
pJob->subPlans = pReq->pDag->pSubplans;
pJob->userRes.execFp = pReq->fp; pJob->userRes.execFp = pReq->fp;
pJob->userRes.userParam = pReq->cbParam; pJob->userRes.userParam = pReq->cbParam;
@ -1637,11 +1543,14 @@ int32_t schExecStaticExplainJob(SSchedulerReq *pReq, int64_t *job, bool sync) {
code = schBeginOperation(pJob, SCH_OP_EXEC, sync); code = schBeginOperation(pJob, SCH_OP_EXEC, sync);
if (code) { if (code) {
pReq->fp(NULL, pReq->cbParam, code); pReq->fp(NULL, pReq->cbParam, code);
schFreeJobImpl(pJob);
SCH_ERR_RET(code); SCH_ERR_RET(code);
} }
*/
SCH_ERR_JRET(qExecStaticExplain(pReq->pDag, (SRetrieveTableRsp **)&pJob->resData)); SCH_ERR_JRET(qExecStaticExplain(pReq->pDag, (SRetrieveTableRsp **)&pJob->resData));
/*
int64_t refId = taosAddRef(schMgmt.jobRef, pJob); int64_t refId = taosAddRef(schMgmt.jobRef, pJob);
if (refId < 0) { if (refId < 0) {
SCH_JOB_ELOG("taosAddRef job failed, error:%s", tstrerror(terrno)); SCH_JOB_ELOG("taosAddRef job failed, error:%s", tstrerror(terrno));
@ -1656,10 +1565,10 @@ int32_t schExecStaticExplainJob(SSchedulerReq *pReq, int64_t *job, bool sync) {
pJob->refId = refId; pJob->refId = refId;
SCH_JOB_DLOG("job refId:0x%" PRIx64, pJob->refId); SCH_JOB_DLOG("job refId:0x%" PRIx64, pJob->refId);
*/
pJob->status = JOB_TASK_STATUS_PARTIAL_SUCCEED; pJob->status = JOB_TASK_STATUS_PARTIAL_SUCCEED;
*job = pJob->refId;
SCH_JOB_DLOG("job exec done, job status:%s", SCH_GET_JOB_STATUS_STR(pJob)); SCH_JOB_DLOG("job exec done, job status:%s", SCH_GET_JOB_STATUS_STR(pJob));
if (!sync) { if (!sync) {
@ -1668,7 +1577,7 @@ int32_t schExecStaticExplainJob(SSchedulerReq *pReq, int64_t *job, bool sync) {
schEndOperation(pJob); schEndOperation(pJob);
} }
schReleaseJob(pJob->refId); // schReleaseJob(pJob->refId);
SCH_RET(code); SCH_RET(code);
@ -1676,7 +1585,7 @@ _return:
schEndOperation(pJob); schEndOperation(pJob);
if (!sync) { if (!sync) {
pReq->fp(NULL, pReq->cbParam, code); pReq->execFp(NULL, pReq->execParam, code);
} }
schFreeJobImpl(pJob); schFreeJobImpl(pJob);
@ -1715,3 +1624,39 @@ int32_t schAsyncFetchRows(SSchJob *pJob) {
} }
int32_t schExecJobImpl(SSchedulerReq *pReq, SSchJob *pJob, bool sync) {
int32_t code = 0;
qDebug("QID:0x%" PRIx64 " sch job refId 0x%"PRIx64 " started", pReq->pDag->queryId, pJob->refId);
SCH_ERR_JRET(schBeginOperation(pJob, SCH_OP_EXEC, sync));
if (EXPLAIN_MODE_STATIC == pReq->pDag->explainInfo.mode) {
code = schLaunchStaticExplainJob(pReq, pJob, sync);
} else {
code = schLaunchJob(pJob);
if (sync) {
SCH_JOB_DLOG("will wait for rsp now, job status:%s", SCH_GET_JOB_STATUS_STR(pJob));
tsem_wait(&pJob->rspSem);
schEndOperation(pJob);
} else if (code) {
schPostJobRes(pJob, SCH_OP_EXEC);
}
}
SCH_JOB_DLOG("job exec done, job status:%s, jobId:0x%" PRIx64, SCH_GET_JOB_STATUS_STR(pJob), pJob->refId);
SCH_RET(code);
_return:
if (!sync) {
pReq->execFp(NULL, pReq->execParam, code);
}
SCH_RET(code);
}

View File

@ -67,33 +67,53 @@ int32_t schedulerInit(SSchedulerCfg *cfg) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t schedulerExecJob(SSchedulerReq *pReq, int64_t *pJob, SQueryResult *pRes) { int32_t schedulerExecJob(SSchedulerReq *pReq, int64_t *pJobId, SQueryResult *pRes) {
qDebug("scheduler sync exec job start"); qDebug("scheduler sync exec job start");
if (NULL == pReq || NULL == pJob || NULL == pRes) {
SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
}
SCH_RET(schExecJob(pReq, pJob, pRes));
}
int32_t schedulerAsyncExecJob(SSchedulerReq *pReq, int64_t *pJob) {
qDebug("scheduler async exec job start");
int32_t code = 0; int32_t code = 0;
if (NULL == pReq || NULL == pJob) { SSchJob *pJob = NULL;
SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); SCH_ERR_JRET(schInitJob(pReq, &pJob));
}
schAsyncExecJob(pReq, pJob); *pJobId = pJob->refId;
SCH_ERR_JRET(schExecJobImpl(pReq, pJob, true));
_return: _return:
if (code != TSDB_CODE_SUCCESS) { if (code && NULL == pJob) {
pReq->fp(NULL, pReq->cbParam, code); qDestroyQueryPlan(pReq->pDag);
} }
SCH_RET(code); if (pJob) {
schSetJobQueryRes(pJob, pRes);
schReleaseJob(pJob->refId);
}
return code;
}
int32_t schedulerAsyncExecJob(SSchedulerReq *pReq, int64_t *pJobId) {
qDebug("scheduler async exec job start");
int32_t code = 0;
SSchJob *pJob = NULL;
SCH_ERR_JRET(schInitJob(pReq, &pJob));
*pJobId = pJob->refId;
SCH_ERR_JRET(schExecJobImpl(pReq, pJob, false));
_return:
if (code && NULL == pJob) {
qDestroyQueryPlan(pReq->pDag);
}
if (pJob) {
schReleaseJob(pJob->refId);
}
return code;
} }
int32_t schedulerFetchRows(int64_t job, void **pData) { int32_t schedulerFetchRows(int64_t job, void **pData) {
@ -120,7 +140,7 @@ int32_t schedulerFetchRows(int64_t job, void **pData) {
SCH_RET(code); SCH_RET(code);
} }
void schedulerAsyncFetchRows(int64_t job, schedulerFetchCallback fp, void* param) { void schedulerAsyncFetchRows(int64_t job, schedulerFetchFp fp, void* param) {
qDebug("scheduler async fetch rows start"); qDebug("scheduler async fetch rows start");
int32_t code = 0; int32_t code = 0;

View File

@ -511,8 +511,8 @@ void* schtRunJobThread(void *aa) {
req.pNodeList = qnodeList; req.pNodeList = qnodeList;
req.pDag = &dag; req.pDag = &dag;
req.sql = "select * from tb"; req.sql = "select * from tb";
req.fp = schtQueryCb; req.execFp = schtQueryCb;
req.cbParam = &queryDone; req.execParam = &queryDone;
code = schedulerAsyncExecJob(&req, &queryJobRefId); code = schedulerAsyncExecJob(&req, &queryJobRefId);
assert(code == 0); assert(code == 0);
@ -663,8 +663,8 @@ TEST(queryTest, normalCase) {
req.pNodeList = qnodeList; req.pNodeList = qnodeList;
req.pDag = &dag; req.pDag = &dag;
req.sql = "select * from tb"; req.sql = "select * from tb";
req.fp = schtQueryCb; req.execFp = schtQueryCb;
req.cbParam = &queryDone; req.execParam = &queryDone;
code = schedulerAsyncExecJob(&req, &job); code = schedulerAsyncExecJob(&req, &job);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
@ -767,8 +767,8 @@ TEST(queryTest, readyFirstCase) {
req.pNodeList = qnodeList; req.pNodeList = qnodeList;
req.pDag = &dag; req.pDag = &dag;
req.sql = "select * from tb"; req.sql = "select * from tb";
req.fp = schtQueryCb; req.execFp = schtQueryCb;
req.cbParam = &queryDone; req.execParam = &queryDone;
code = schedulerAsyncExecJob(&req, &job); code = schedulerAsyncExecJob(&req, &job);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
@ -874,8 +874,8 @@ TEST(queryTest, flowCtrlCase) {
req.pNodeList = qnodeList; req.pNodeList = qnodeList;
req.pDag = &dag; req.pDag = &dag;
req.sql = "select * from tb"; req.sql = "select * from tb";
req.fp = schtQueryCb; req.execFp = schtQueryCb;
req.cbParam = &queryDone; req.execParam = &queryDone;
code = schedulerAsyncExecJob(&req, &job); code = schedulerAsyncExecJob(&req, &job);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
@ -987,8 +987,8 @@ TEST(insertTest, normalCase) {
req.pNodeList = qnodeList; req.pNodeList = qnodeList;
req.pDag = &dag; req.pDag = &dag;
req.sql = "insert into tb values(now,1)"; req.sql = "insert into tb values(now,1)";
req.fp = schtQueryCb; req.execFp = schtQueryCb;
req.cbParam = NULL; req.execParam = NULL;
code = schedulerExecJob(&req, &insertJobRefId, &res); code = schedulerExecJob(&req, &insertJobRefId, &res);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);

View File

@ -221,7 +221,6 @@ void syncNodeVoteForSelf(SSyncNode* pSyncNode);
// snapshot -------------- // snapshot --------------
bool syncNodeHasSnapshot(SSyncNode* pSyncNode); bool syncNodeHasSnapshot(SSyncNode* pSyncNode);
bool syncNodeIsIndexInSnapshot(SSyncNode* pSyncNode, SyncIndex index);
SyncIndex syncNodeGetLastIndex(SSyncNode* pSyncNode); SyncIndex syncNodeGetLastIndex(SSyncNode* pSyncNode);
SyncTerm syncNodeGetLastTerm(SSyncNode* pSyncNode); SyncTerm syncNodeGetLastTerm(SSyncNode* pSyncNode);

View File

@ -860,7 +860,9 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs
} }
code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pAppendEntry); code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pAppendEntry);
ASSERT(code == 0); if (code != 0) {
return -1;
}
// pre commit // pre commit
code = syncNodePreCommit(ths, pAppendEntry); code = syncNodePreCommit(ths, pAppendEntry);
@ -971,7 +973,9 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs
ASSERT(pAppendEntry != NULL); ASSERT(pAppendEntry != NULL);
code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pAppendEntry); code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pAppendEntry);
ASSERT(code == 0); if (code != 0) {
return -1;
}
// pre commit // pre commit
code = syncNodePreCommit(ths, pAppendEntry); code = syncNodePreCommit(ths, pAppendEntry);

View File

@ -75,7 +75,11 @@ int32_t syncNodeElect(SSyncNode* pSyncNode) {
if (pSyncNode->state == TAOS_SYNC_STATE_FOLLOWER) { if (pSyncNode->state == TAOS_SYNC_STATE_FOLLOWER) {
syncNodeFollower2Candidate(pSyncNode); syncNodeFollower2Candidate(pSyncNode);
} }
ASSERT(pSyncNode->state == TAOS_SYNC_STATE_CANDIDATE);
if (pSyncNode->state != TAOS_SYNC_STATE_CANDIDATE) {
syncNodeErrorLog(pSyncNode, "not candidate, can not elect");
return -1;
}
// start election // start election
raftStoreNextTerm(pSyncNode->pRaftStore); raftStoreNextTerm(pSyncNode->pRaftStore);

View File

@ -1923,6 +1923,8 @@ void syncNodeVoteForSelf(SSyncNode* pSyncNode) {
} }
// snapshot -------------- // snapshot --------------
// return if has a snapshot
bool syncNodeHasSnapshot(SSyncNode* pSyncNode) { bool syncNodeHasSnapshot(SSyncNode* pSyncNode) {
bool ret = false; bool ret = false;
SSnapshot snapshot = {.data = NULL, .lastApplyIndex = -1, .lastApplyTerm = 0, .lastConfigIndex = -1}; SSnapshot snapshot = {.data = NULL, .lastApplyIndex = -1, .lastApplyTerm = 0, .lastConfigIndex = -1};
@ -1935,21 +1937,10 @@ bool syncNodeHasSnapshot(SSyncNode* pSyncNode) {
return ret; return ret;
} }
#if 0 // return max(logLastIndex, snapshotLastIndex)
bool syncNodeIsIndexInSnapshot(SSyncNode* pSyncNode, SyncIndex index) { // if no snapshot and log, return -1
ASSERT(syncNodeHasSnapshot(pSyncNode));
ASSERT(pSyncNode->pFsm->FpGetSnapshotInfo != NULL);
ASSERT(index >= SYNC_INDEX_BEGIN);
SSnapshot snapshot;
pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapshot);
bool b = (index <= snapshot.lastApplyIndex);
return b;
}
#endif
SyncIndex syncNodeGetLastIndex(SSyncNode* pSyncNode) { SyncIndex syncNodeGetLastIndex(SSyncNode* pSyncNode) {
SSnapshot snapshot = {.data = NULL, .lastApplyIndex = -1, .lastApplyTerm = 0}; SSnapshot snapshot = {.data = NULL, .lastApplyIndex = -1, .lastApplyTerm = 0, .lastConfigIndex = -1};
if (pSyncNode->pFsm->FpGetSnapshotInfo != NULL) { if (pSyncNode->pFsm->FpGetSnapshotInfo != NULL) {
pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapshot); pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapshot);
} }
@ -1959,6 +1950,8 @@ SyncIndex syncNodeGetLastIndex(SSyncNode* pSyncNode) {
return lastIndex; return lastIndex;
} }
// return the last term of snapshot and log
// if error, return SYNC_TERM_INVALID (by syncLogLastTerm)
SyncTerm syncNodeGetLastTerm(SSyncNode* pSyncNode) { SyncTerm syncNodeGetLastTerm(SSyncNode* pSyncNode) {
SyncTerm lastTerm = 0; SyncTerm lastTerm = 0;
if (syncNodeHasSnapshot(pSyncNode)) { if (syncNodeHasSnapshot(pSyncNode)) {
@ -1990,11 +1983,14 @@ int32_t syncNodeGetLastIndexTerm(SSyncNode* pSyncNode, SyncIndex* pLastIndex, Sy
return 0; return 0;
} }
// return append-entries first try index
SyncIndex syncNodeSyncStartIndex(SSyncNode* pSyncNode) { SyncIndex syncNodeSyncStartIndex(SSyncNode* pSyncNode) {
SyncIndex syncStartIndex = syncNodeGetLastIndex(pSyncNode) + 1; SyncIndex syncStartIndex = syncNodeGetLastIndex(pSyncNode) + 1;
return syncStartIndex; return syncStartIndex;
} }
// if index > 0, return index - 1
// else, return -1
SyncIndex syncNodeGetPreIndex(SSyncNode* pSyncNode, SyncIndex index) { SyncIndex syncNodeGetPreIndex(SSyncNode* pSyncNode, SyncIndex index) {
SyncIndex preIndex = index - 1; SyncIndex preIndex = index - 1;
if (preIndex < SYNC_INDEX_INVALID) { if (preIndex < SYNC_INDEX_INVALID) {
@ -2004,21 +2000,10 @@ SyncIndex syncNodeGetPreIndex(SSyncNode* pSyncNode, SyncIndex index) {
return preIndex; return preIndex;
} }
/* // if index < 0, return SYNC_TERM_INVALID
SyncIndex syncNodeGetPreIndex(SSyncNode* pSyncNode, SyncIndex index) { // if index == 0, return 0
ASSERT(index >= SYNC_INDEX_BEGIN); // if index > 0, return preTerm
// if error, return SYNC_TERM_INVALID
SyncIndex syncStartIndex = syncNodeSyncStartIndex(pSyncNode);
if (index > syncStartIndex) {
syncNodeLog3("syncNodeGetPreIndex", pSyncNode);
ASSERT(0);
}
SyncIndex preIndex = index - 1;
return preIndex;
}
*/
SyncTerm syncNodeGetPreTerm(SSyncNode* pSyncNode, SyncIndex index) { SyncTerm syncNodeGetPreTerm(SSyncNode* pSyncNode, SyncIndex index) {
if (index < SYNC_INDEX_BEGIN) { if (index < SYNC_INDEX_BEGIN) {
return SYNC_TERM_INVALID; return SYNC_TERM_INVALID;
@ -2056,112 +2041,6 @@ SyncTerm syncNodeGetPreTerm(SSyncNode* pSyncNode, SyncIndex index) {
return SYNC_TERM_INVALID; return SYNC_TERM_INVALID;
} }
#if 0
SyncTerm syncNodeGetPreTerm(SSyncNode* pSyncNode, SyncIndex index) {
ASSERT(index >= SYNC_INDEX_BEGIN);
SyncIndex syncStartIndex = syncNodeSyncStartIndex(pSyncNode);
if (index > syncStartIndex) {
syncNodeLog3("syncNodeGetPreTerm", pSyncNode);
ASSERT(0);
}
if (index == SYNC_INDEX_BEGIN) {
return 0;
}
SyncTerm preTerm = 0;
SyncIndex preIndex = index - 1;
SSyncRaftEntry* pPreEntry = NULL;
int32_t code = pSyncNode->pLogStore->syncLogGetEntry(pSyncNode->pLogStore, preIndex, &pPreEntry);
if (code == 0) {
ASSERT(pPreEntry != NULL);
preTerm = pPreEntry->term;
taosMemoryFree(pPreEntry);
return preTerm;
} else {
if (terrno == TSDB_CODE_WAL_LOG_NOT_EXIST) {
SSnapshot snapshot = {.data = NULL, .lastApplyIndex = -1, .lastApplyTerm = 0, .lastConfigIndex = -1};
if (pSyncNode->pFsm->FpGetSnapshotInfo != NULL) {
pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapshot);
if (snapshot.lastApplyIndex == preIndex) {
return snapshot.lastApplyTerm;
}
}
}
}
ASSERT(0);
return -1;
}
#endif
#if 0
SyncTerm syncNodeGetPreTerm(SSyncNode* pSyncNode, SyncIndex index) {
ASSERT(index >= SYNC_INDEX_BEGIN);
SyncIndex syncStartIndex = syncNodeSyncStartIndex(pSyncNode);
if (index > syncStartIndex) {
syncNodeLog3("syncNodeGetPreTerm", pSyncNode);
ASSERT(0);
}
if (index == SYNC_INDEX_BEGIN) {
return 0;
}
SyncTerm preTerm = 0;
if (syncNodeHasSnapshot(pSyncNode)) {
// has snapshot
SSnapshot snapshot = {.data = NULL, .lastApplyIndex = -1, .lastApplyTerm = 0, .lastConfigIndex = -1};
if (pSyncNode->pFsm->FpGetSnapshotInfo != NULL) {
pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapshot);
}
if (index > snapshot.lastApplyIndex + 1) {
// should be log preTerm
SSyncRaftEntry* pPreEntry = NULL;
int32_t code = pSyncNode->pLogStore->syncLogGetEntry(pSyncNode->pLogStore, index - 1, &pPreEntry);
ASSERT(code == 0);
ASSERT(pPreEntry != NULL);
preTerm = pPreEntry->term;
taosMemoryFree(pPreEntry);
} else if (index == snapshot.lastApplyIndex + 1) {
preTerm = snapshot.lastApplyTerm;
} else {
// maybe snapshot change
sError("sync get pre term, bad scene. index:%ld", index);
logStoreLog2("sync get pre term, bad scene", pSyncNode->pLogStore);
SSyncRaftEntry* pPreEntry = NULL;
int32_t code = pSyncNode->pLogStore->syncLogGetEntry(pSyncNode->pLogStore, index - 1, &pPreEntry);
ASSERT(code == 0);
ASSERT(pPreEntry != NULL);
preTerm = pPreEntry->term;
taosMemoryFree(pPreEntry);
}
} else {
// no snapshot
ASSERT(index > SYNC_INDEX_BEGIN);
SSyncRaftEntry* pPreEntry = NULL;
int32_t code = pSyncNode->pLogStore->syncLogGetEntry(pSyncNode->pLogStore, index - 1, &pPreEntry);
ASSERT(code == 0);
ASSERT(pPreEntry != NULL);
preTerm = pPreEntry->term;
taosMemoryFree(pPreEntry);
}
return preTerm;
}
#endif
// get pre index and term of "index" // get pre index and term of "index"
int32_t syncNodeGetPreIndexTerm(SSyncNode* pSyncNode, SyncIndex index, SyncIndex* pPreIndex, SyncTerm* pPreTerm) { int32_t syncNodeGetPreIndexTerm(SSyncNode* pSyncNode, SyncIndex index, SyncIndex* pPreIndex, SyncTerm* pPreTerm) {
*pPreIndex = syncNodeGetPreIndex(pSyncNode, index); *pPreIndex = syncNodeGetPreIndex(pSyncNode, index);
@ -2351,8 +2230,8 @@ static int32_t syncNodeAppendNoop(SSyncNode* ths) {
ASSERT(pEntry != NULL); ASSERT(pEntry != NULL);
if (ths->state == TAOS_SYNC_STATE_LEADER) { if (ths->state == TAOS_SYNC_STATE_LEADER) {
// ths->pLogStore->appendEntry(ths->pLogStore, pEntry); int32_t code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pEntry);
ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pEntry); ASSERT(code == 0);
syncNodeReplicate(ths); syncNodeReplicate(ths);
} }
@ -2406,6 +2285,7 @@ int32_t syncNodeOnPingReplyCb(SSyncNode* ths, SyncPingReply* pMsg) {
// //
int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg, SyncIndex* pRetIndex) { int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg, SyncIndex* pRetIndex) {
int32_t ret = 0; int32_t ret = 0;
int32_t code = 0;
syncClientRequestLog2("==syncNodeOnClientRequestCb==", pMsg); syncClientRequestLog2("==syncNodeOnClientRequestCb==", pMsg);
SyncIndex index = ths->pLogStore->syncLogWriteIndex(ths->pLogStore); SyncIndex index = ths->pLogStore->syncLogWriteIndex(ths->pLogStore);
@ -2414,18 +2294,24 @@ int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg, SyncI
ASSERT(pEntry != NULL); ASSERT(pEntry != NULL);
if (ths->state == TAOS_SYNC_STATE_LEADER) { if (ths->state == TAOS_SYNC_STATE_LEADER) {
// ths->pLogStore->appendEntry(ths->pLogStore, pEntry); // append entry
ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pEntry); code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pEntry);
if (code != 0) {
// del resp mgr, call FpCommitCb
ASSERT(0);
return -1;
}
// start replicate right now! // if mulit replica, start replicate right now
if (ths->replicaNum > 1) {
syncNodeReplicate(ths); syncNodeReplicate(ths);
}
// pre commit // pre commit
SRpcMsg rpcMsg; SRpcMsg rpcMsg;
syncEntry2OriginalRpc(pEntry, &rpcMsg); syncEntry2OriginalRpc(pEntry, &rpcMsg);
if (ths->pFsm != NULL) { if (ths->pFsm != NULL) {
// if (ths->pFsm->FpPreCommitCb != NULL && pEntry->originalRpcType != TDMT_SYNC_NOOP) {
if (ths->pFsm->FpPreCommitCb != NULL && syncUtilUserPreCommit(pEntry->originalRpcType)) { if (ths->pFsm->FpPreCommitCb != NULL && syncUtilUserPreCommit(pEntry->originalRpcType)) {
SFsmCbMeta cbMeta = {0}; SFsmCbMeta cbMeta = {0};
cbMeta.index = pEntry->index; cbMeta.index = pEntry->index;
@ -2439,8 +2325,10 @@ int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg, SyncI
} }
rpcFreeCont(rpcMsg.pCont); rpcFreeCont(rpcMsg.pCont);
// only myself, maybe commit // if only myself, maybe commit right now
if (ths->replicaNum == 1) {
syncMaybeAdvanceCommitIndex(ths); syncMaybeAdvanceCommitIndex(ths);
}
} else { } else {
// pre commit // pre commit
@ -2448,7 +2336,6 @@ int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg, SyncI
syncEntry2OriginalRpc(pEntry, &rpcMsg); syncEntry2OriginalRpc(pEntry, &rpcMsg);
if (ths->pFsm != NULL) { if (ths->pFsm != NULL) {
// if (ths->pFsm->FpPreCommitCb != NULL && pEntry->originalRpcType != TDMT_SYNC_NOOP) {
if (ths->pFsm->FpPreCommitCb != NULL && syncUtilUserPreCommit(pEntry->originalRpcType)) { if (ths->pFsm->FpPreCommitCb != NULL && syncUtilUserPreCommit(pEntry->originalRpcType)) {
SFsmCbMeta cbMeta = {0}; SFsmCbMeta cbMeta = {0};
cbMeta.index = pEntry->index; cbMeta.index = pEntry->index;

View File

@ -101,7 +101,7 @@ cJSON *syncCfg2Json(SSyncCfg *pSyncCfg) {
char *syncCfg2Str(SSyncCfg *pSyncCfg) { char *syncCfg2Str(SSyncCfg *pSyncCfg) {
cJSON *pJson = syncCfg2Json(pSyncCfg); cJSON *pJson = syncCfg2Json(pSyncCfg);
char * serialized = cJSON_Print(pJson); char *serialized = cJSON_Print(pJson);
cJSON_Delete(pJson); cJSON_Delete(pJson);
return serialized; return serialized;
} }
@ -109,7 +109,7 @@ char *syncCfg2Str(SSyncCfg *pSyncCfg) {
char *syncCfg2SimpleStr(SSyncCfg *pSyncCfg) { char *syncCfg2SimpleStr(SSyncCfg *pSyncCfg) {
if (pSyncCfg != NULL) { if (pSyncCfg != NULL) {
int32_t len = 512; int32_t len = 512;
char * s = taosMemoryMalloc(len); char *s = taosMemoryMalloc(len);
memset(s, 0, len); memset(s, 0, len);
snprintf(s, len, "{replica-num:%d, my-index:%d, ", pSyncCfg->replicaNum, pSyncCfg->myIndex); snprintf(s, len, "{replica-num:%d, my-index:%d, ", pSyncCfg->replicaNum, pSyncCfg->myIndex);
@ -205,7 +205,7 @@ cJSON *raftCfg2Json(SRaftCfg *pRaftCfg) {
char *raftCfg2Str(SRaftCfg *pRaftCfg) { char *raftCfg2Str(SRaftCfg *pRaftCfg) {
cJSON *pJson = raftCfg2Json(pRaftCfg); cJSON *pJson = raftCfg2Json(pRaftCfg);
char * serialized = cJSON_Print(pJson); char *serialized = cJSON_Print(pJson);
cJSON_Delete(pJson); cJSON_Delete(pJson);
return serialized; return serialized;
} }
@ -214,7 +214,16 @@ int32_t raftCfgCreateFile(SSyncCfg *pCfg, SRaftCfgMeta meta, const char *path) {
ASSERT(pCfg != NULL); ASSERT(pCfg != NULL);
TdFilePtr pFile = taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE); TdFilePtr pFile = taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE);
ASSERT(pFile != NULL); if (pFile == NULL) {
int32_t err = terrno;
const char *errStr = tstrerror(err);
int32_t sysErr = errno;
const char *sysErrStr = strerror(errno);
sError("create raft cfg file error, err:%d %X, msg:%s, syserr:%d, sysmsg:%s", err, err, errStr, sysErr, sysErrStr);
ASSERT(0);
return -1;
}
SRaftCfg raftCfg; SRaftCfg raftCfg;
raftCfg.cfg = *pCfg; raftCfg.cfg = *pCfg;
@ -271,7 +280,7 @@ int32_t raftCfgFromJson(const cJSON *pRoot, SRaftCfg *pRaftCfg) {
(pRaftCfg->configIndexArr)[i] = atoll(pIndex->valuestring); (pRaftCfg->configIndexArr)[i] = atoll(pIndex->valuestring);
} }
cJSON * pJsonSyncCfg = cJSON_GetObjectItem(pJson, "SSyncCfg"); cJSON *pJsonSyncCfg = cJSON_GetObjectItem(pJson, "SSyncCfg");
int32_t code = syncCfgFromJson(pJsonSyncCfg, &(pRaftCfg->cfg)); int32_t code = syncCfgFromJson(pJsonSyncCfg, &(pRaftCfg->cfg));
ASSERT(code == 0); ASSERT(code == 0);

View File

@ -168,6 +168,9 @@ static SyncIndex raftLogWriteIndex(struct SSyncLogStore* pLogStore) {
return lastVer + 1; return lastVer + 1;
} }
// if success, return last term
// if not log, return 0
// if error, return SYNC_TERM_INVALID
static SyncTerm raftLogLastTerm(struct SSyncLogStore* pLogStore) { static SyncTerm raftLogLastTerm(struct SSyncLogStore* pLogStore) {
SSyncLogStoreData* pData = pLogStore->data; SSyncLogStoreData* pData = pLogStore->data;
SWal* pWal = pData->pWal; SWal* pWal = pData->pWal;
@ -176,15 +179,17 @@ static SyncTerm raftLogLastTerm(struct SSyncLogStore* pLogStore) {
} else { } else {
SSyncRaftEntry* pLastEntry; SSyncRaftEntry* pLastEntry;
int32_t code = raftLogGetLastEntry(pLogStore, &pLastEntry); int32_t code = raftLogGetLastEntry(pLogStore, &pLastEntry);
ASSERT(code == 0); if (code == 0 && pLastEntry != NULL) {
ASSERT(pLastEntry != NULL);
SyncTerm lastTerm = pLastEntry->term; SyncTerm lastTerm = pLastEntry->term;
taosMemoryFree(pLastEntry); taosMemoryFree(pLastEntry);
return lastTerm; return lastTerm;
} else {
return SYNC_TERM_INVALID;
}
} }
return 0; // can not be here!
return SYNC_TERM_INVALID;
} }
static int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry) { static int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry) {
@ -218,16 +223,21 @@ static int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncRaftEntr
ASSERT(0); ASSERT(0);
} }
walFsync(pWal, true); // walFsync(pWal, true);
do {
char eventLog[128]; char eventLog[128];
snprintf(eventLog, sizeof(eventLog), "write index:%ld, type:%s,%d, type2:%s,%d", pEntry->index, snprintf(eventLog, sizeof(eventLog), "write index:%ld, type:%s,%d, type2:%s,%d", pEntry->index,
TMSG_INFO(pEntry->msgType), pEntry->msgType, TMSG_INFO(pEntry->originalRpcType), pEntry->originalRpcType); TMSG_INFO(pEntry->msgType), pEntry->msgType, TMSG_INFO(pEntry->originalRpcType), pEntry->originalRpcType);
syncNodeEventLog(pData->pSyncNode, eventLog); syncNodeEventLog(pData->pSyncNode, eventLog);
} while (0);
return code; return code;
} }
// entry found, return 0
// entry not found, return -1, terrno = TSDB_CODE_WAL_LOG_NOT_EXIST
// other error, return -1
static int32_t raftLogGetEntry(struct SSyncLogStore* pLogStore, SyncIndex index, SSyncRaftEntry** ppEntry) { static int32_t raftLogGetEntry(struct SSyncLogStore* pLogStore, SyncIndex index, SSyncRaftEntry** ppEntry) {
SSyncLogStoreData* pData = pLogStore->data; SSyncLogStoreData* pData = pLogStore->data;
SWal* pWal = pData->pWal; SWal* pWal = pData->pWal;
@ -238,6 +248,7 @@ static int32_t raftLogGetEntry(struct SSyncLogStore* pLogStore, SyncIndex index,
// SWalReadHandle* pWalHandle = walOpenReadHandle(pWal); // SWalReadHandle* pWalHandle = walOpenReadHandle(pWal);
SWalReadHandle* pWalHandle = pData->pWalHandle; SWalReadHandle* pWalHandle = pData->pWalHandle;
if (pWalHandle == NULL) { if (pWalHandle == NULL) {
terrno = TSDB_CODE_SYN_INTERNAL_ERROR;
return -1; return -1;
} }
@ -309,6 +320,9 @@ static int32_t raftLogTruncate(struct SSyncLogStore* pLogStore, SyncIndex fromIn
return code; return code;
} }
// entry found, return 0
// entry not found, return -1, terrno = TSDB_CODE_WAL_LOG_NOT_EXIST
// other error, return -1
static int32_t raftLogGetLastEntry(SSyncLogStore* pLogStore, SSyncRaftEntry** ppLastEntry) { static int32_t raftLogGetLastEntry(SSyncLogStore* pLogStore, SSyncRaftEntry** ppLastEntry) {
SSyncLogStoreData* pData = pLogStore->data; SSyncLogStoreData* pData = pLogStore->data;
SWal* pWal = pData->pWal; SWal* pWal = pData->pWal;
@ -320,6 +334,7 @@ static int32_t raftLogGetLastEntry(SSyncLogStore* pLogStore, SSyncRaftEntry** pp
return -1; return -1;
} else { } else {
SyncIndex lastIndex = raftLogLastIndex(pLogStore); SyncIndex lastIndex = raftLogLastIndex(pLogStore);
ASSERT(lastIndex >= SYNC_INDEX_BEGIN);
int32_t code = raftLogGetEntry(pLogStore, lastIndex, ppLastEntry); int32_t code = raftLogGetEntry(pLogStore, lastIndex, ppLastEntry);
return code; return code;
} }
@ -356,7 +371,7 @@ int32_t logStoreAppendEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry) {
ASSERT(0); ASSERT(0);
} }
walFsync(pWal, true); // walFsync(pWal, true);
char eventLog[128]; char eventLog[128];
snprintf(eventLog, sizeof(eventLog), "old write index:%ld, type:%s,%d, type2:%s,%d", pEntry->index, snprintf(eventLog, sizeof(eventLog), "old write index:%ld, type:%s,%d, type2:%s,%d", pEntry->index,

View File

@ -156,6 +156,10 @@ static bool syncNodeOnRequestVoteLogOK(SSyncNode* pSyncNode, SyncRequestVote* pM
SyncTerm myLastTerm = syncNodeGetLastTerm(pSyncNode); SyncTerm myLastTerm = syncNodeGetLastTerm(pSyncNode);
SyncIndex myLastIndex = syncNodeGetLastIndex(pSyncNode); SyncIndex myLastIndex = syncNodeGetLastIndex(pSyncNode);
if (myLastTerm == SYNC_TERM_INVALID) {
return false;
}
if (pMsg->lastLogTerm > myLastTerm) { if (pMsg->lastLogTerm > myLastTerm) {
return true; return true;
} }

View File

@ -148,6 +148,7 @@ typedef struct {
char release : 2; char release : 2;
char secured : 2; char secured : 2;
char spi : 2; char spi : 2;
char hasEpSet : 2; // contain epset or not, 0(default): no epset, 1: contain epset
char user[TSDB_UNI_LEN]; char user[TSDB_UNI_LEN];
STraceId traceId; STraceId traceId;
@ -252,7 +253,7 @@ int transAsyncSend(SAsyncPool* pool, queue* mq);
do { \ do { \
if (id > 0) { \ if (id > 0) { \
tTrace("handle step1"); \ tTrace("handle step1"); \
SExHandle* exh2 = transAcquireExHandle(refMgt, id); \ SExHandle* exh2 = transAcquireExHandle(id); \
if (exh2 == NULL || id != exh2->refId) { \ if (exh2 == NULL || id != exh2->refId) { \
tTrace("handle %p except, may already freed, ignore msg, ref1: %" PRIu64 ", ref2 : %" PRIu64 "", exh1, \ tTrace("handle %p except, may already freed, ignore msg, ref1: %" PRIu64 ", ref2 : %" PRIu64 "", exh1, \
exh2 ? exh2->refId : 0, id); \ exh2 ? exh2->refId : 0, id); \
@ -260,7 +261,7 @@ int transAsyncSend(SAsyncPool* pool, queue* mq);
} \ } \
} else if (id == 0) { \ } else if (id == 0) { \
tTrace("handle step2"); \ tTrace("handle step2"); \
SExHandle* exh2 = transAcquireExHandle(refMgt, id); \ SExHandle* exh2 = transAcquireExHandle(id); \
if (exh2 == NULL || id == exh2->refId) { \ if (exh2 == NULL || id == exh2->refId) { \
tTrace("handle %p except, may already freed, ignore msg, ref1: %" PRIu64 ", ref2 : %" PRIu64 "", exh1, id, \ tTrace("handle %p except, may already freed, ignore msg, ref1: %" PRIu64 ", ref2 : %" PRIu64 "", exh1, id, \
exh2 ? exh2->refId : 0); \ exh2 ? exh2->refId : 0); \
@ -273,6 +274,7 @@ int transAsyncSend(SAsyncPool* pool, queue* mq);
goto _return2; \ goto _return2; \
} \ } \
} while (0) } while (0)
int transInitBuffer(SConnBuffer* buf); int transInitBuffer(SConnBuffer* buf);
int transClearBuffer(SConnBuffer* buf); int transClearBuffer(SConnBuffer* buf);
int transDestroyBuffer(SConnBuffer* buf); int transDestroyBuffer(SConnBuffer* buf);
@ -294,7 +296,6 @@ void transSendRequest(void* shandle, const SEpSet* pEpSet, STransMsg* pMsg, STra
void transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pMsg, STransMsg* pRsp); void transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pMsg, STransMsg* pRsp);
void transSendResponse(const STransMsg* msg); void transSendResponse(const STransMsg* msg);
void transRegisterMsg(const STransMsg* msg); void transRegisterMsg(const STransMsg* msg);
int transGetConnInfo(void* thandle, STransHandleInfo* pInfo);
void transSetDefaultAddr(void* shandle, const char* ip, const char* fqdn); void transSetDefaultAddr(void* shandle, const char* ip, const char* fqdn);
void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle); void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle);
@ -378,25 +379,24 @@ typedef struct SDelayQueue {
} SDelayQueue; } SDelayQueue;
int transDQCreate(uv_loop_t* loop, SDelayQueue** queue); int transDQCreate(uv_loop_t* loop, SDelayQueue** queue);
void transDQDestroy(SDelayQueue* queue); void transDQDestroy(SDelayQueue* queue);
int transDQSched(SDelayQueue* queue, void (*func)(void* arg), void* arg, uint64_t timeoutMs); int transDQSched(SDelayQueue* queue, void (*func)(void* arg), void* arg, uint64_t timeoutMs);
// void transPrintEpSet(SEpSet* pEpSet);
bool transEpSetIsEqual(SEpSet* a, SEpSet* b); bool transEpSetIsEqual(SEpSet* a, SEpSet* b);
/* /*
* init global func * init global func
*/ */
void transThreadOnce(); void transThreadOnce();
void transInitEnv(); void transInit();
void transCleanup();
int32_t transOpenExHandleMgt(int size); int32_t transOpenExHandleMgt(int size);
void transCloseExHandleMgt(int32_t mgt); void transCloseExHandleMgt();
int64_t transAddExHandle(int32_t mgt, void* p); int64_t transAddExHandle(void* p);
int32_t transRemoveExHandle(int32_t mgt, int64_t refId); int32_t transRemoveExHandle(int64_t refId);
SExHandle* transAcquireExHandle(int32_t mgt, int64_t refId); SExHandle* transAcquireExHandle(int64_t refId);
int32_t transReleaseExHandle(int32_t mgt, int64_t refId); int32_t transReleaseExHandle(int64_t refId);
void transDestoryExHandle(void* handle); void transDestoryExHandle(void* handle);
#ifdef __cplusplus #ifdef __cplusplus

View File

@ -36,7 +36,7 @@ static int32_t transValidLocalFqdn(const char* localFqdn, uint32_t* ip) {
return 0; return 0;
} }
void* rpcOpen(const SRpcInit* pInit) { void* rpcOpen(const SRpcInit* pInit) {
transInitEnv(); rpcInit();
SRpcInfo* pRpc = taosMemoryCalloc(1, sizeof(SRpcInfo)); SRpcInfo* pRpc = taosMemoryCalloc(1, sizeof(SRpcInfo));
if (pRpc == NULL) { if (pRpc == NULL) {
@ -82,8 +82,8 @@ void rpcClose(void* arg) {
tInfo("start to close rpc"); tInfo("start to close rpc");
SRpcInfo* pRpc = (SRpcInfo*)arg; SRpcInfo* pRpc = (SRpcInfo*)arg;
(*taosCloseHandle[pRpc->connType])(pRpc->tcphandle); (*taosCloseHandle[pRpc->connType])(pRpc->tcphandle);
transCloseExHandleMgt(pRpc->refMgt);
taosMemoryFree(pRpc); taosMemoryFree(pRpc);
tInfo("finish to close rpc");
return; return;
} }
@ -141,7 +141,9 @@ void rpcSendRecv(void* shandle, SEpSet* pEpSet, SRpcMsg* pMsg, SRpcMsg* pRsp) {
} }
void rpcSendResponse(const SRpcMsg* pMsg) { transSendResponse(pMsg); } void rpcSendResponse(const SRpcMsg* pMsg) { transSendResponse(pMsg); }
int32_t rpcGetConnInfo(void* thandle, SRpcConnInfo* pInfo) { return transGetConnInfo((void*)thandle, pInfo); }
int32_t rpcGetConnInfo(void* thandle, SRpcConnInfo* pInfo) { return 0; }
void rpcRefHandle(void* handle, int8_t type) { void rpcRefHandle(void* handle, int8_t type) {
assert(type == TAOS_CONN_SERVER || type == TAOS_CONN_CLIENT); assert(type == TAOS_CONN_SERVER || type == TAOS_CONN_CLIENT);
@ -165,11 +167,11 @@ void rpcSetDefaultAddr(void* thandle, const char* ip, const char* fqdn) {
} }
int32_t rpcInit() { int32_t rpcInit() {
// impl later transInit();
return 0; return 0;
} }
void rpcCleanup(void) { void rpcCleanup(void) {
// impl later transCleanup();
return; return;
} }

View File

@ -15,9 +15,6 @@
#ifdef USE_UV #ifdef USE_UV
#include "transComm.h" #include "transComm.h"
static int32_t transSCliInst = 0;
static int32_t refMgt = 0;
typedef struct SCliConn { typedef struct SCliConn {
T_REF_DECLARE() T_REF_DECLARE()
uv_connect_t connReq; uv_connect_t connReq;
@ -312,6 +309,7 @@ void cliHandleResp(SCliConn* conn) {
transMsg.msgType = pHead->msgType; transMsg.msgType = pHead->msgType;
transMsg.info.ahandle = NULL; transMsg.info.ahandle = NULL;
transMsg.info.traceId = pHead->traceId; transMsg.info.traceId = pHead->traceId;
transMsg.info.hasEpSet = pHead->hasEpSet;
SCliMsg* pMsg = NULL; SCliMsg* pMsg = NULL;
STransConnCtx* pCtx = NULL; STransConnCtx* pCtx = NULL;
@ -319,8 +317,9 @@ void cliHandleResp(SCliConn* conn) {
if (CONN_NO_PERSIST_BY_APP(conn)) { if (CONN_NO_PERSIST_BY_APP(conn)) {
pMsg = transQueuePop(&conn->cliMsgs); pMsg = transQueuePop(&conn->cliMsgs);
pCtx = pMsg->ctx;
transMsg.info.ahandle = pCtx->ahandle; pCtx = pMsg ? pMsg->ctx : NULL;
transMsg.info.ahandle = pCtx ? pCtx->ahandle : NULL;
tDebug("%s conn %p get ahandle %p, persist: 0", CONN_GET_INST_LABEL(conn), conn, transMsg.info.ahandle); tDebug("%s conn %p get ahandle %p, persist: 0", CONN_GET_INST_LABEL(conn), conn, transMsg.info.ahandle);
} else { } else {
uint64_t ahandle = (uint64_t)pHead->ahandle; uint64_t ahandle = (uint64_t)pHead->ahandle;
@ -467,7 +466,6 @@ void* destroyConnPool(void* pool) {
while (connList != NULL) { while (connList != NULL) {
while (!QUEUE_IS_EMPTY(&connList->conn)) { while (!QUEUE_IS_EMPTY(&connList->conn)) {
queue* h = QUEUE_HEAD(&connList->conn); queue* h = QUEUE_HEAD(&connList->conn);
// QUEUE_REMOVE(h);
SCliConn* c = QUEUE_DATA(h, SCliConn, conn); SCliConn* c = QUEUE_DATA(h, SCliConn, conn);
cliDestroyConn(c, true); cliDestroyConn(c, true);
} }
@ -503,12 +501,13 @@ static SCliConn* getConnFromPool(void* pool, char* ip, uint32_t port) {
} }
static void allocConnRef(SCliConn* conn, bool update) { static void allocConnRef(SCliConn* conn, bool update) {
if (update) { if (update) {
transRemoveExHandle(refMgt, conn->refId); transRemoveExHandle(conn->refId);
conn->refId = -1;
} }
SExHandle* exh = taosMemoryCalloc(1, sizeof(SExHandle)); SExHandle* exh = taosMemoryCalloc(1, sizeof(SExHandle));
exh->handle = conn; exh->handle = conn;
exh->pThrd = conn->hostThrd; exh->pThrd = conn->hostThrd;
exh->refId = transAddExHandle(refMgt, exh); exh->refId = transAddExHandle(exh);
conn->refId = exh->refId; conn->refId = exh->refId;
} }
static void addConnToPool(void* pool, SCliConn* conn) { static void addConnToPool(void* pool, SCliConn* conn) {
@ -602,14 +601,26 @@ static void cliDestroyConn(SCliConn* conn, bool clear) {
tTrace("%s conn %p remove from conn pool", CONN_GET_INST_LABEL(conn), conn); tTrace("%s conn %p remove from conn pool", CONN_GET_INST_LABEL(conn), conn);
QUEUE_REMOVE(&conn->conn); QUEUE_REMOVE(&conn->conn);
QUEUE_INIT(&conn->conn); QUEUE_INIT(&conn->conn);
transRemoveExHandle(refMgt, conn->refId); transRemoveExHandle(conn->refId);
conn->refId = -1;
if (clear) { if (clear) {
if (!uv_is_closing((uv_handle_t*)conn->stream)) {
uv_close((uv_handle_t*)conn->stream, cliDestroy); uv_close((uv_handle_t*)conn->stream, cliDestroy);
} else {
cliDestroy((uv_handle_t*)conn->stream);
}
} }
} }
static void cliDestroy(uv_handle_t* handle) { static void cliDestroy(uv_handle_t* handle) {
if (uv_handle_get_type(handle) != UV_TCP || handle->data == NULL) {
return;
}
SCliConn* conn = handle->data; SCliConn* conn = handle->data;
transRemoveExHandle(conn->refId);
taosMemoryFree(conn->ip); taosMemoryFree(conn->ip);
conn->stream->data = NULL;
taosMemoryFree(conn->stream); taosMemoryFree(conn->stream);
transCtxCleanup(&conn->ctx); transCtxCleanup(&conn->ctx);
transQueueDestroy(&conn->cliMsgs); transQueueDestroy(&conn->cliMsgs);
@ -735,7 +746,7 @@ static void cliHandleQuit(SCliMsg* pMsg, SCliThrd* pThrd) {
} }
static void cliHandleRelease(SCliMsg* pMsg, SCliThrd* pThrd) { static void cliHandleRelease(SCliMsg* pMsg, SCliThrd* pThrd) {
int64_t refId = (int64_t)(pMsg->msg.info.handle); int64_t refId = (int64_t)(pMsg->msg.info.handle);
SExHandle* exh = transAcquireExHandle(refMgt, refId); SExHandle* exh = transAcquireExHandle(refId);
if (exh == NULL) { if (exh == NULL) {
tDebug("%" PRId64 " already release", refId); tDebug("%" PRId64 " already release", refId);
} }
@ -761,7 +772,7 @@ SCliConn* cliGetConn(SCliMsg* pMsg, SCliThrd* pThrd, bool* ignore) {
SCliConn* conn = NULL; SCliConn* conn = NULL;
int64_t refId = (int64_t)(pMsg->msg.info.handle); int64_t refId = (int64_t)(pMsg->msg.info.handle);
if (refId != 0) { if (refId != 0) {
SExHandle* exh = transAcquireExHandle(refMgt, refId); SExHandle* exh = transAcquireExHandle(refId);
if (exh == NULL) { if (exh == NULL) {
*ignore = true; *ignore = true;
destroyCmsg(pMsg); destroyCmsg(pMsg);
@ -769,7 +780,7 @@ SCliConn* cliGetConn(SCliMsg* pMsg, SCliThrd* pThrd, bool* ignore) {
// assert(0); // assert(0);
} else { } else {
conn = exh->handle; conn = exh->handle;
transReleaseExHandle(refMgt, refId); transReleaseExHandle(refId);
} }
return conn; return conn;
}; };
@ -899,10 +910,6 @@ void* transInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads,
} }
cli->pThreadObj[i] = pThrd; cli->pThreadObj[i] = pThrd;
} }
int ref = atomic_add_fetch_32(&transSCliInst, 1);
if (ref == 1) {
refMgt = transOpenExHandleMgt(50000);
}
return cli; return cli;
} }
@ -971,7 +978,7 @@ void cliSendQuit(SCliThrd* thrd) {
} }
void cliWalkCb(uv_handle_t* handle, void* arg) { void cliWalkCb(uv_handle_t* handle, void* arg) {
if (!uv_is_closing(handle)) { if (!uv_is_closing(handle)) {
uv_close(handle, NULL); uv_close(handle, cliDestroy);
} }
} }
@ -1014,6 +1021,23 @@ void cliCompareAndSwap(int8_t* val, int8_t exp, int8_t newVal) {
*val = newVal; *val = newVal;
} }
} }
bool cliTryToExtractEpSet(STransMsg* pResp, SEpSet* dst) {
if (pResp == NULL || pResp->info.hasEpSet == 0) {
return false;
}
tDeserializeSEpSet(pResp->pCont, pResp->contLen, dst);
int32_t tlen = tSerializeSEpSet(NULL, 0, dst);
int32_t bufLen = pResp->contLen - tlen;
char* buf = rpcMallocCont(bufLen);
memcpy(buf, (char*)pResp->pCont + tlen, bufLen);
pResp->pCont = buf;
pResp->contLen = bufLen;
return true;
}
int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) {
SCliThrd* pThrd = pConn->hostThrd; SCliThrd* pThrd = pConn->hostThrd;
STrans* pTransInst = pThrd->pTransInst; STrans* pTransInst = pThrd->pTransInst;
@ -1058,6 +1082,12 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) {
} }
STraceId* trace = &pResp->info.traceId; STraceId* trace = &pResp->info.traceId;
if (cliTryToExtractEpSet(pResp, &pCtx->epSet)) {
char tbuf[256] = {0};
EPSET_DEBUG_STR(&pCtx->epSet, tbuf);
tGTrace("%s conn %p extract epset from msg", CONN_GET_INST_LABEL(pConn), pConn);
}
if (pCtx->pSem != NULL) { if (pCtx->pSem != NULL) {
tGTrace("%s conn %p(sync) handle resp", CONN_GET_INST_LABEL(pConn), pConn); tGTrace("%s conn %p(sync) handle resp", CONN_GET_INST_LABEL(pConn), pConn);
if (pCtx->pRsp == NULL) { if (pCtx->pRsp == NULL) {
@ -1086,10 +1116,6 @@ void transCloseClient(void* arg) {
} }
taosMemoryFree(cli->pThreadObj); taosMemoryFree(cli->pThreadObj);
taosMemoryFree(cli); taosMemoryFree(cli);
int ref = atomic_sub_fetch_32(&transSCliInst, 1);
if (ref == 0) {
transCloseExHandleMgt(refMgt);
}
} }
void transRefCliHandle(void* handle) { void transRefCliHandle(void* handle) {
if (handle == NULL) { if (handle == NULL) {
@ -1111,12 +1137,12 @@ void transUnrefCliHandle(void* handle) {
} }
SCliThrd* transGetWorkThrdFromHandle(int64_t handle) { SCliThrd* transGetWorkThrdFromHandle(int64_t handle) {
SCliThrd* pThrd = NULL; SCliThrd* pThrd = NULL;
SExHandle* exh = transAcquireExHandle(refMgt, handle); SExHandle* exh = transAcquireExHandle(handle);
if (exh == NULL) { if (exh == NULL) {
return NULL; return NULL;
} }
pThrd = exh->pThrd; pThrd = exh->pThrd;
transReleaseExHandle(refMgt, handle); transReleaseExHandle(handle);
return pThrd; return pThrd;
} }
SCliThrd* transGetWorkThrd(STrans* trans, int64_t handle) { SCliThrd* transGetWorkThrd(STrans* trans, int64_t handle) {

View File

@ -16,7 +16,9 @@
#include "transComm.h" #include "transComm.h"
// static TdThreadOnce transModuleInit = PTHREAD_ONCE_INIT; static TdThreadOnce transModuleInit = PTHREAD_ONCE_INIT;
static int32_t refMgt;
int transAuthenticateMsg(void* pMsg, int msgLen, void* pAuth, void* pKey) { int transAuthenticateMsg(void* pMsg, int msgLen, void* pAuth, void* pKey) {
T_MD5_CTX context; T_MD5_CTX context;
@ -478,35 +480,47 @@ bool transEpSetIsEqual(SEpSet* a, SEpSet* b) {
return true; return true;
} }
void transInitEnv() { static void transInitEnv() {
// refMgt = transOpenExHandleMgt(50000);
uv_os_setenv("UV_TCP_SINGLE_ACCEPT", "1"); uv_os_setenv("UV_TCP_SINGLE_ACCEPT", "1");
} }
static void transDestroyEnv() {
// close ref
transCloseExHandleMgt(refMgt);
}
void transInit() {
// init env
taosThreadOnce(&transModuleInit, transInitEnv);
}
void transCleanup() {
// clean env
transDestroyEnv();
}
int32_t transOpenExHandleMgt(int size) { int32_t transOpenExHandleMgt(int size) {
// added into once later // added into once later
return taosOpenRef(size, transDestoryExHandle); return taosOpenRef(size, transDestoryExHandle);
} }
void transCloseExHandleMgt(int32_t mgt) { void transCloseExHandleMgt() {
// close ref // close ref
taosCloseRef(mgt); taosCloseRef(refMgt);
} }
int64_t transAddExHandle(int32_t mgt, void* p) { int64_t transAddExHandle(void* p) {
// acquire extern handle // acquire extern handle
return taosAddRef(mgt, p); return taosAddRef(refMgt, p);
} }
int32_t transRemoveExHandle(int32_t mgt, int64_t refId) { int32_t transRemoveExHandle(int64_t refId) {
// acquire extern handle // acquire extern handle
return taosRemoveRef(mgt, refId); return taosRemoveRef(refMgt, refId);
} }
SExHandle* transAcquireExHandle(int32_t mgt, int64_t refId) { SExHandle* transAcquireExHandle(int64_t refId) {
// acquire extern handle // acquire extern handle
return (SExHandle*)taosAcquireRef(mgt, refId); return (SExHandle*)taosAcquireRef(refMgt, refId);
} }
int32_t transReleaseExHandle(int32_t mgt, int64_t refId) { int32_t transReleaseExHandle(int64_t refId) {
// release extern handle // release extern handle
return taosReleaseRef(mgt, refId); return taosReleaseRef(refMgt, refId);
} }
void transDestoryExHandle(void* handle) { void transDestoryExHandle(void* handle) {
if (handle == NULL) { if (handle == NULL) {

View File

@ -20,8 +20,6 @@
static TdThreadOnce transModuleInit = PTHREAD_ONCE_INIT; static TdThreadOnce transModuleInit = PTHREAD_ONCE_INIT;
static char* notify = "a"; static char* notify = "a";
static int32_t tranSSvrInst = 0;
static int32_t refMgt = 0;
typedef struct { typedef struct {
int notifyCount; // int notifyCount; //
@ -142,16 +140,6 @@ static void uvHandleRegister(SSvrMsg* msg, SWorkThrd* thrd);
static void (*transAsyncHandle[])(SSvrMsg* msg, SWorkThrd* thrd) = {uvHandleResp, uvHandleQuit, uvHandleRelease, static void (*transAsyncHandle[])(SSvrMsg* msg, SWorkThrd* thrd) = {uvHandleResp, uvHandleQuit, uvHandleRelease,
uvHandleRegister, NULL}; uvHandleRegister, NULL};
static int32_t exHandlesMgt;
// void uvInitEnv();
// void uvOpenExHandleMgt(int size);
// void uvCloseExHandleMgt();
// int64_t uvAddExHandle(void* p);
// int32_t uvRemoveExHandle(int64_t refId);
// int32_t uvReleaseExHandle(int64_t refId);
// void uvDestoryExHandle(void* handle);
// SExHandle* uvAcquireExHandle(int64_t refId);
static void uvDestroyConn(uv_handle_t* handle); static void uvDestroyConn(uv_handle_t* handle);
@ -274,7 +262,7 @@ static void uvHandleReq(SSvrConn* pConn) {
// 2. once send out data, cli conn released to conn pool immediately // 2. once send out data, cli conn released to conn pool immediately
// 3. not mixed with persist // 3. not mixed with persist
transMsg.info.ahandle = (void*)pHead->ahandle; transMsg.info.ahandle = (void*)pHead->ahandle;
transMsg.info.handle = (void*)transAcquireExHandle(refMgt, pConn->refId); transMsg.info.handle = (void*)transAcquireExHandle(pConn->refId);
transMsg.info.refId = pConn->refId; transMsg.info.refId = pConn->refId;
transMsg.info.traceId = pHead->traceId; transMsg.info.traceId = pHead->traceId;
@ -292,7 +280,7 @@ static void uvHandleReq(SSvrConn* pConn) {
pConnInfo->clientPort = ntohs(pConn->addr.sin_port); pConnInfo->clientPort = ntohs(pConn->addr.sin_port);
tstrncpy(pConnInfo->user, pConn->user, sizeof(pConnInfo->user)); tstrncpy(pConnInfo->user, pConn->user, sizeof(pConnInfo->user));
transReleaseExHandle(refMgt, pConn->refId); transReleaseExHandle(pConn->refId);
STrans* pTransInst = pConn->pTransInst; STrans* pTransInst = pConn->pTransInst;
(*pTransInst->cfp)(pTransInst->parent, &transMsg, NULL); (*pTransInst->cfp)(pTransInst->parent, &transMsg, NULL);
@ -345,21 +333,12 @@ void uvOnTimeoutCb(uv_timer_t* handle) {
void uvOnSendCb(uv_write_t* req, int status) { void uvOnSendCb(uv_write_t* req, int status) {
SSvrConn* conn = req->data; SSvrConn* conn = req->data;
// transClearBuffer(&conn->readBuf);
if (status == 0) { if (status == 0) {
tTrace("conn %p data already was written on stream", conn); tTrace("conn %p data already was written on stream", conn);
if (!transQueueEmpty(&conn->srvMsgs)) { if (!transQueueEmpty(&conn->srvMsgs)) {
SSvrMsg* msg = transQueuePop(&conn->srvMsgs); SSvrMsg* msg = transQueuePop(&conn->srvMsgs);
// if (msg->type == Release && conn->status != ConnNormal) {
// conn->status = ConnNormal;
// transUnrefSrvHandle(conn);
// reallocConnRef(conn);
// destroySmsg(msg);
// transQueueClear(&conn->srvMsgs);
// return;
//}
destroySmsg(msg); destroySmsg(msg);
// send second data, just use for push // send cached data
if (!transQueueEmpty(&conn->srvMsgs)) { if (!transQueueEmpty(&conn->srvMsgs)) {
msg = (SSvrMsg*)transQueueGet(&conn->srvMsgs, 0); msg = (SSvrMsg*)transQueueGet(&conn->srvMsgs, 0);
if (msg->type == Register && conn->status == ConnAcquire) { if (msg->type == Register && conn->status == ConnAcquire) {
@ -383,6 +362,7 @@ void uvOnSendCb(uv_write_t* req, int status) {
} }
} }
} }
transUnrefSrvHandle(conn);
} else { } else {
tError("conn %p failed to write data, %s", conn, uv_err_name(status)); tError("conn %p failed to write data, %s", conn, uv_err_name(status));
conn->broken = true; conn->broken = true;
@ -396,7 +376,6 @@ static void uvOnPipeWriteCb(uv_write_t* req, int status) {
tError("fail to dispatch conn to work thread"); tError("fail to dispatch conn to work thread");
} }
uv_close((uv_handle_t*)req->data, uvFreeCb); uv_close((uv_handle_t*)req->data, uvFreeCb);
// taosMemoryFree(req->data);
taosMemoryFree(req); taosMemoryFree(req);
} }
@ -410,6 +389,7 @@ static void uvPrepareSendData(SSvrMsg* smsg, uv_buf_t* wb) {
STransMsgHead* pHead = transHeadFromCont(pMsg->pCont); STransMsgHead* pHead = transHeadFromCont(pMsg->pCont);
pHead->ahandle = (uint64_t)pMsg->info.ahandle; pHead->ahandle = (uint64_t)pMsg->info.ahandle;
pHead->traceId = pMsg->info.traceId; pHead->traceId = pMsg->info.traceId;
pHead->hasEpSet = pMsg->info.hasEpSet;
if (pConn->status == ConnNormal) { if (pConn->status == ConnNormal) {
pHead->msgType = pConn->inType + 1; pHead->msgType = pConn->inType + 1;
@ -422,6 +402,7 @@ static void uvPrepareSendData(SSvrMsg* smsg, uv_buf_t* wb) {
transUnrefSrvHandle(pConn); transUnrefSrvHandle(pConn);
} else { } else {
pHead->msgType = pMsg->msgType; pHead->msgType = pMsg->msgType;
// set up resp msg type
if (pHead->msgType == 0 && transMsgLenFromCont(pMsg->contLen) == sizeof(STransMsgHead)) if (pHead->msgType == 0 && transMsgLenFromCont(pMsg->contLen) == sizeof(STransMsgHead))
pHead->msgType = pConn->inType + 1; pHead->msgType = pConn->inType + 1;
} }
@ -444,11 +425,15 @@ static void uvPrepareSendData(SSvrMsg* smsg, uv_buf_t* wb) {
} }
static void uvStartSendRespInternal(SSvrMsg* smsg) { static void uvStartSendRespInternal(SSvrMsg* smsg) {
SSvrConn* pConn = smsg->pConn;
if (pConn->broken) {
return;
}
uv_buf_t wb; uv_buf_t wb;
uvPrepareSendData(smsg, &wb); uvPrepareSendData(smsg, &wb);
SSvrConn* pConn = smsg->pConn; transRefSrvHandle(pConn);
// uv_timer_stop(&pConn->pTimer);
uv_write(&pConn->pWriter, (uv_stream_t*)pConn->pTcp, &wb, 1, uvOnSendCb); uv_write(&pConn->pWriter, (uv_stream_t*)pConn->pTcp, &wb, 1, uvOnSendCb);
} }
static void uvStartSendResp(SSvrMsg* smsg) { static void uvStartSendResp(SSvrMsg* smsg) {
@ -523,15 +508,15 @@ void uvWorkerAsyncCb(uv_async_t* handle) {
SExHandle* exh1 = transMsg.info.handle; SExHandle* exh1 = transMsg.info.handle;
int64_t refId = transMsg.info.refId; int64_t refId = transMsg.info.refId;
SExHandle* exh2 = transAcquireExHandle(refMgt, refId); SExHandle* exh2 = transAcquireExHandle(refId);
if (exh2 == NULL || exh1 != exh2) { if (exh2 == NULL || exh1 != exh2) {
tTrace("handle except msg %p, ignore it", exh1); tTrace("handle except msg %p, ignore it", exh1);
transReleaseExHandle(refMgt, refId); transReleaseExHandle(refId);
destroySmsg(msg); destroySmsg(msg);
continue; continue;
} }
msg->pConn = exh1->handle; msg->pConn = exh1->handle;
transReleaseExHandle(refMgt, refId); transReleaseExHandle(refId);
(*transAsyncHandle[msg->type])(msg, pThrd); (*transAsyncHandle[msg->type])(msg, pThrd);
} }
} }
@ -773,8 +758,8 @@ static SSvrConn* createConn(void* hThrd) {
SExHandle* exh = taosMemoryMalloc(sizeof(SExHandle)); SExHandle* exh = taosMemoryMalloc(sizeof(SExHandle));
exh->handle = pConn; exh->handle = pConn;
exh->pThrd = pThrd; exh->pThrd = pThrd;
exh->refId = transAddExHandle(refMgt, exh); exh->refId = transAddExHandle(exh);
transAcquireExHandle(refMgt, exh->refId); transAcquireExHandle(exh->refId);
pConn->refId = exh->refId; pConn->refId = exh->refId;
transRefSrvHandle(pConn); transRefSrvHandle(pConn);
@ -789,11 +774,12 @@ static void destroyConn(SSvrConn* conn, bool clear) {
transDestroyBuffer(&conn->readBuf); transDestroyBuffer(&conn->readBuf);
if (clear) { if (clear) {
if (!uv_is_closing((uv_handle_t*)conn->pTcp)) {
tTrace("conn %p to be destroyed", conn); tTrace("conn %p to be destroyed", conn);
// uv_shutdown_t* req = taosMemoryMalloc(sizeof(uv_shutdown_t));
uv_close((uv_handle_t*)conn->pTcp, uvDestroyConn); uv_close((uv_handle_t*)conn->pTcp, uvDestroyConn);
// uv_close(conn->pTcp) } else {
// uv_shutdown(req, (uv_stream_t*)conn->pTcp, uvShutDownCb); uvDestroyConn((uv_handle_t*)conn->pTcp);
}
} }
} }
static void destroyConnRegArg(SSvrConn* conn) { static void destroyConnRegArg(SSvrConn* conn) {
@ -803,14 +789,14 @@ static void destroyConnRegArg(SSvrConn* conn) {
} }
} }
static int reallocConnRef(SSvrConn* conn) { static int reallocConnRef(SSvrConn* conn) {
transReleaseExHandle(refMgt, conn->refId); transReleaseExHandle(conn->refId);
transRemoveExHandle(refMgt, conn->refId); transRemoveExHandle(conn->refId);
// avoid app continue to send msg on invalid handle // avoid app continue to send msg on invalid handle
SExHandle* exh = taosMemoryMalloc(sizeof(SExHandle)); SExHandle* exh = taosMemoryMalloc(sizeof(SExHandle));
exh->handle = conn; exh->handle = conn;
exh->pThrd = conn->hostThrd; exh->pThrd = conn->hostThrd;
exh->refId = transAddExHandle(refMgt, exh); exh->refId = transAddExHandle(exh);
transAcquireExHandle(refMgt, exh->refId); transAcquireExHandle(exh->refId);
conn->refId = exh->refId; conn->refId = exh->refId;
return 0; return 0;
@ -822,11 +808,10 @@ static void uvDestroyConn(uv_handle_t* handle) {
} }
SWorkThrd* thrd = conn->hostThrd; SWorkThrd* thrd = conn->hostThrd;
transReleaseExHandle(refMgt, conn->refId); transReleaseExHandle(conn->refId);
transRemoveExHandle(refMgt, conn->refId); transRemoveExHandle(conn->refId);
tDebug("%s conn %p destroy", transLabel(thrd->pTransInst), conn); tDebug("%s conn %p destroy", transLabel(thrd->pTransInst), conn);
// uv_timer_stop(&conn->pTimer);
transQueueDestroy(&conn->srvMsgs); transQueueDestroy(&conn->srvMsgs);
QUEUE_REMOVE(&conn->queue); QUEUE_REMOVE(&conn->queue);
@ -871,12 +856,6 @@ void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads,
srv->port = port; srv->port = port;
uv_loop_init(srv->loop); uv_loop_init(srv->loop);
// taosThreadOnce(&transModuleInit, uvInitEnv);
int ref = atomic_add_fetch_32(&tranSSvrInst, 1);
if (ref == 1) {
refMgt = transOpenExHandleMgt(50000);
}
assert(0 == uv_pipe_init(srv->loop, &srv->pipeListen, 0)); assert(0 == uv_pipe_init(srv->loop, &srv->pipeListen, 0));
#ifdef WINDOWS #ifdef WINDOWS
char pipeName[64]; char pipeName[64];
@ -1028,11 +1007,6 @@ void transCloseServer(void* arg) {
taosMemoryFree(srv->pipe); taosMemoryFree(srv->pipe);
taosMemoryFree(srv); taosMemoryFree(srv);
int ref = atomic_sub_fetch_32(&tranSSvrInst, 1);
if (ref == 0) {
transCloseExHandleMgt(refMgt);
}
} }
void transRefSrvHandle(void* handle) { void transRefSrvHandle(void* handle) {
@ -1071,11 +1045,11 @@ void transReleaseSrvHandle(void* handle) {
tTrace("%s conn %p start to release", transLabel(pThrd->pTransInst), exh->handle); tTrace("%s conn %p start to release", transLabel(pThrd->pTransInst), exh->handle);
transAsyncSend(pThrd->asyncPool, &m->q); transAsyncSend(pThrd->asyncPool, &m->q);
transReleaseExHandle(refMgt, refId); transReleaseExHandle(refId);
return; return;
_return1: _return1:
tTrace("handle %p failed to send to release handle", exh); tTrace("handle %p failed to send to release handle", exh);
transReleaseExHandle(refMgt, refId); transReleaseExHandle(refId);
return; return;
_return2: _return2:
tTrace("handle %p failed to send to release handle", exh); tTrace("handle %p failed to send to release handle", exh);
@ -1100,12 +1074,12 @@ void transSendResponse(const STransMsg* msg) {
STraceId* trace = (STraceId*)&msg->info.traceId; STraceId* trace = (STraceId*)&msg->info.traceId;
tGTrace("conn %p start to send resp (1/2)", exh->handle); tGTrace("conn %p start to send resp (1/2)", exh->handle);
transAsyncSend(pThrd->asyncPool, &m->q); transAsyncSend(pThrd->asyncPool, &m->q);
transReleaseExHandle(refMgt, refId); transReleaseExHandle(refId);
return; return;
_return1: _return1:
tTrace("handle %p failed to send resp", exh); tTrace("handle %p failed to send resp", exh);
rpcFreeCont(msg->pCont); rpcFreeCont(msg->pCont);
transReleaseExHandle(refMgt, refId); transReleaseExHandle(refId);
return; return;
_return2: _return2:
tTrace("handle %p failed to send resp", exh); tTrace("handle %p failed to send resp", exh);
@ -1129,19 +1103,17 @@ void transRegisterMsg(const STransMsg* msg) {
tTrace("%s conn %p start to register brokenlink callback", transLabel(pThrd->pTransInst), exh->handle); tTrace("%s conn %p start to register brokenlink callback", transLabel(pThrd->pTransInst), exh->handle);
transAsyncSend(pThrd->asyncPool, &m->q); transAsyncSend(pThrd->asyncPool, &m->q);
transReleaseExHandle(refMgt, refId); transReleaseExHandle(refId);
return; return;
_return1: _return1:
tTrace("handle %p failed to register brokenlink", exh); tTrace("handle %p failed to register brokenlink", exh);
rpcFreeCont(msg->pCont); rpcFreeCont(msg->pCont);
transReleaseExHandle(refMgt, refId); transReleaseExHandle(refId);
return; return;
_return2: _return2:
tTrace("handle %p failed to register brokenlink", exh); tTrace("handle %p failed to register brokenlink", exh);
rpcFreeCont(msg->pCont); rpcFreeCont(msg->pCont);
} }
int transGetConnInfo(void* thandle, STransHandleInfo* pConnInfo) { return -1; }
#endif #endif

13
tests/pytest/a.sh Executable file
View File

@ -0,0 +1,13 @@
#!/bin/bash
for i in {1..100}
do
echo $i
python3 ./test.py -f query/nestedQuery/nestedQuery_datacheck.py >>log 2>&1
if [ $? -eq 0 ]
then
echo success
else
echo failed
break
fi
done

View File

@ -915,7 +915,7 @@ int32_t prepareInsertData(BindData *data) {
data->colNum = 0; data->colNum = 0;
data->colTypes = taosMemoryCalloc(30, sizeof(int32_t)); data->colTypes = taosMemoryCalloc(30, sizeof(int32_t));
data->sql = taosMemoryCalloc(1, 1024); data->sql = taosMemoryCalloc(1, 1024);
data->pBind = taosMemoryCalloc((allRowNum/gCurCase->bindRowNum)*gCurCase->bindColNum, sizeof(TAOS_MULTI_BIND)); data->pBind = taosMemoryCalloc((int32_t)(allRowNum/gCurCase->bindRowNum)*gCurCase->bindColNum, sizeof(TAOS_MULTI_BIND));
data->pTags = taosMemoryCalloc(gCurCase->tblNum*gCurCase->bindTagNum, sizeof(TAOS_MULTI_BIND)); data->pTags = taosMemoryCalloc(gCurCase->tblNum*gCurCase->bindTagNum, sizeof(TAOS_MULTI_BIND));
data->tsData = taosMemoryMalloc(allRowNum * sizeof(int64_t)); data->tsData = taosMemoryMalloc(allRowNum * sizeof(int64_t));
data->boolData = taosMemoryMalloc(allRowNum * sizeof(bool)); data->boolData = taosMemoryMalloc(allRowNum * sizeof(bool));
@ -932,7 +932,7 @@ int32_t prepareInsertData(BindData *data) {
data->binaryData = taosMemoryMalloc(allRowNum * gVarCharSize); data->binaryData = taosMemoryMalloc(allRowNum * gVarCharSize);
data->binaryLen = taosMemoryMalloc(allRowNum * sizeof(int32_t)); data->binaryLen = taosMemoryMalloc(allRowNum * sizeof(int32_t));
if (gCurCase->bindNullNum) { if (gCurCase->bindNullNum) {
data->isNull = taosMemoryCalloc(allRowNum, sizeof(char)); data->isNull = taosMemoryCalloc((int32_t)allRowNum, sizeof(char));
} }
for (int32_t i = 0; i < allRowNum; ++i) { for (int32_t i = 0; i < allRowNum; ++i) {
@ -950,7 +950,7 @@ int32_t prepareInsertData(BindData *data) {
data->doubleData[i] = (double)(i+1); data->doubleData[i] = (double)(i+1);
memset(data->binaryData + gVarCharSize * i, 'a'+i%26, gVarCharLen); memset(data->binaryData + gVarCharSize * i, 'a'+i%26, gVarCharLen);
if (gCurCase->bindNullNum) { if (gCurCase->bindNullNum) {
data->isNull[i] = i % 2; data->isNull[i] = (char)(i % 2);
} }
data->binaryLen[i] = gVarCharLen; data->binaryLen[i] = gVarCharLen;
} }
@ -979,7 +979,7 @@ int32_t prepareQueryCondData(BindData *data, int32_t tblIdx) {
data->colNum = 0; data->colNum = 0;
data->colTypes = taosMemoryCalloc(30, sizeof(int32_t)); data->colTypes = taosMemoryCalloc(30, sizeof(int32_t));
data->sql = taosMemoryCalloc(1, 1024); data->sql = taosMemoryCalloc(1, 1024);
data->pBind = taosMemoryCalloc(bindNum*gCurCase->bindColNum, sizeof(TAOS_MULTI_BIND)); data->pBind = taosMemoryCalloc((int32_t)bindNum*gCurCase->bindColNum, sizeof(TAOS_MULTI_BIND));
data->tsData = taosMemoryMalloc(bindNum * sizeof(int64_t)); data->tsData = taosMemoryMalloc(bindNum * sizeof(int64_t));
data->boolData = taosMemoryMalloc(bindNum * sizeof(bool)); data->boolData = taosMemoryMalloc(bindNum * sizeof(bool));
data->tinyData = taosMemoryMalloc(bindNum * sizeof(int8_t)); data->tinyData = taosMemoryMalloc(bindNum * sizeof(int8_t));
@ -995,7 +995,7 @@ int32_t prepareQueryCondData(BindData *data, int32_t tblIdx) {
data->binaryData = taosMemoryMalloc(bindNum * gVarCharSize); data->binaryData = taosMemoryMalloc(bindNum * gVarCharSize);
data->binaryLen = taosMemoryMalloc(bindNum * sizeof(int32_t)); data->binaryLen = taosMemoryMalloc(bindNum * sizeof(int32_t));
if (gCurCase->bindNullNum) { if (gCurCase->bindNullNum) {
data->isNull = taosMemoryCalloc(bindNum, sizeof(char)); data->isNull = taosMemoryCalloc((int32_t)bindNum, sizeof(char));
} }
for (int32_t i = 0; i < bindNum; ++i) { for (int32_t i = 0; i < bindNum; ++i) {
@ -1013,7 +1013,7 @@ int32_t prepareQueryCondData(BindData *data, int32_t tblIdx) {
data->doubleData[i] = (double)(tblIdx*gCurCase->rowNum + rand() % gCurCase->rowNum); data->doubleData[i] = (double)(tblIdx*gCurCase->rowNum + rand() % gCurCase->rowNum);
memset(data->binaryData + gVarCharSize * i, 'a'+i%26, gVarCharLen); memset(data->binaryData + gVarCharSize * i, 'a'+i%26, gVarCharLen);
if (gCurCase->bindNullNum) { if (gCurCase->bindNullNum) {
data->isNull[i] = i % 2; data->isNull[i] = (char)(i % 2);
} }
data->binaryLen[i] = gVarCharLen; data->binaryLen[i] = gVarCharLen;
} }
@ -1036,7 +1036,7 @@ int32_t prepareQueryMiscData(BindData *data, int32_t tblIdx) {
data->colNum = 0; data->colNum = 0;
data->colTypes = taosMemoryCalloc(30, sizeof(int32_t)); data->colTypes = taosMemoryCalloc(30, sizeof(int32_t));
data->sql = taosMemoryCalloc(1, 1024); data->sql = taosMemoryCalloc(1, 1024);
data->pBind = taosMemoryCalloc(bindNum*gCurCase->bindColNum, sizeof(TAOS_MULTI_BIND)); data->pBind = taosMemoryCalloc((int32_t)bindNum*gCurCase->bindColNum, sizeof(TAOS_MULTI_BIND));
data->tsData = taosMemoryMalloc(bindNum * sizeof(int64_t)); data->tsData = taosMemoryMalloc(bindNum * sizeof(int64_t));
data->boolData = taosMemoryMalloc(bindNum * sizeof(bool)); data->boolData = taosMemoryMalloc(bindNum * sizeof(bool));
data->tinyData = taosMemoryMalloc(bindNum * sizeof(int8_t)); data->tinyData = taosMemoryMalloc(bindNum * sizeof(int8_t));
@ -1052,7 +1052,7 @@ int32_t prepareQueryMiscData(BindData *data, int32_t tblIdx) {
data->binaryData = taosMemoryMalloc(bindNum * gVarCharSize); data->binaryData = taosMemoryMalloc(bindNum * gVarCharSize);
data->binaryLen = taosMemoryMalloc(bindNum * sizeof(int32_t)); data->binaryLen = taosMemoryMalloc(bindNum * sizeof(int32_t));
if (gCurCase->bindNullNum) { if (gCurCase->bindNullNum) {
data->isNull = taosMemoryCalloc(bindNum, sizeof(char)); data->isNull = taosMemoryCalloc((int32_t)bindNum, sizeof(char));
} }
for (int32_t i = 0; i < bindNum; ++i) { for (int32_t i = 0; i < bindNum; ++i) {
@ -1070,7 +1070,7 @@ int32_t prepareQueryMiscData(BindData *data, int32_t tblIdx) {
data->doubleData[i] = (double)(tblIdx*gCurCase->rowNum + rand() % gCurCase->rowNum); data->doubleData[i] = (double)(tblIdx*gCurCase->rowNum + rand() % gCurCase->rowNum);
memset(data->binaryData + gVarCharSize * i, 'a'+i%26, gVarCharLen); memset(data->binaryData + gVarCharSize * i, 'a'+i%26, gVarCharLen);
if (gCurCase->bindNullNum) { if (gCurCase->bindNullNum) {
data->isNull[i] = i % 2; data->isNull[i] = (char)(i % 2);
} }
data->binaryLen[i] = gVarCharLen; data->binaryLen[i] = gVarCharLen;
} }
@ -1279,7 +1279,7 @@ void bpCheckQueryResult(TAOS_STMT *stmt, TAOS *taos, char *stmtSql, TAOS_MULTI_B
} }
memcpy(&sql[len], p, (int64_t)s - (int64_t)p); memcpy(&sql[len], p, (int64_t)s - (int64_t)p);
len += (int64_t)s - (int64_t)p; len += (int32_t)((int64_t)s - (int64_t)p);
if (bind[i].is_null && bind[i].is_null[0]) { if (bind[i].is_null && bind[i].is_null[0]) {
bpAppendValueString(sql, TSDB_DATA_TYPE_NULL, NULL, 0, &len); bpAppendValueString(sql, TSDB_DATA_TYPE_NULL, NULL, 0, &len);
@ -2669,7 +2669,7 @@ int main(int argc, char *argv[])
{ {
TAOS *taos = NULL; TAOS *taos = NULL;
srand(time(NULL)); srand((unsigned int)time(NULL));
// connect to server // connect to server
if (argc < 2) { if (argc < 2) {

View File

@ -12,6 +12,7 @@ all: $(TARGET)
exe: exe:
gcc $(CFLAGS) ./batchprepare.c -o $(ROOT)batchprepare $(LFLAGS) gcc $(CFLAGS) ./batchprepare.c -o $(ROOT)batchprepare $(LFLAGS)
gcc $(CFLAGS) ./stopquery.c -o $(ROOT)stopquery $(LFLAGS)
clean: clean:
rm $(ROOT)batchprepare rm $(ROOT)batchprepare

View File

@ -0,0 +1,434 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
// TAOS asynchronous API example
// this example opens multiple tables, insert/retrieve multiple tables
// it is used by TAOS internally for one performance testing
// to compiple: gcc -o asyncdemo asyncdemo.c -ltaos
#include <stdio.h>
#include <stdlib.h>
#include <sys/time.h>
#include <unistd.h>
#include <string.h>
#include <pthread.h>
#include "taos.h"
int points = 5;
int numOfTables = 3;
int tablesInsertProcessed = 0;
int tablesSelectProcessed = 0;
int64_t st, et;
char hostName[128];
char dbName[128];
char tbName[128];
int32_t runTimes = 10000;
typedef struct {
int id;
TAOS *taos;
char name[16];
time_t timeStamp;
int value;
int rowsInserted;
int rowsTried;
int rowsRetrieved;
} STable;
typedef struct SSP_CB_PARAM {
TAOS *taos;
bool fetch;
int32_t *end;
} SSP_CB_PARAM;
#define CASE_ENTER() do { printf("enter case %s\n", __FUNCTION__); } while (0)
#define CASE_LEAVE() do { printf("leave case %s, runTimes %d\n", __FUNCTION__, runTimes); } while (0)
static void sqExecSQL(TAOS *taos, char *command) {
int i;
int32_t code = -1;
TAOS_RES *pSql = taos_query(taos, command);
code = taos_errno(pSql);
if (code != 0) {
fprintf(stderr, "Failed to run %s, reason: %s\n", command, taos_errstr(pSql));
taos_free_result(pSql);
taos_close(taos);
taos_cleanup();
exit(EXIT_FAILURE);
}
taos_free_result(pSql);
}
static void sqExecSQLE(TAOS *taos, char *command) {
int i;
int32_t code = -1;
TAOS_RES *pSql = taos_query(taos, command);
taos_free_result(pSql);
}
void sqExit(char* prefix, const char* errMsg) {
fprintf(stderr, "%s error: %s\n", prefix, errMsg);
exit(1);
}
void sqStopFetchCb(void *param, TAOS_RES *pRes, int numOfRows) {
SSP_CB_PARAM *qParam = (SSP_CB_PARAM *)param;
taos_stop_query(pRes);
taos_free_result(pRes);
*qParam->end = 1;
}
void sqStopQueryCb(void *param, TAOS_RES *pRes, int code) {
SSP_CB_PARAM *qParam = (SSP_CB_PARAM *)param;
if (code == 0 && pRes) {
if (qParam->fetch) {
taos_fetch_rows_a(pRes, sqStopFetchCb, param);
} else {
taos_stop_query(pRes);
taos_free_result(pRes);
*qParam->end = 1;
}
} else {
sqExit("select", taos_errstr(pRes));
}
}
void sqFreeFetchCb(void *param, TAOS_RES *pRes, int numOfRows) {
SSP_CB_PARAM *qParam = (SSP_CB_PARAM *)param;
taos_free_result(pRes);
*qParam->end = 1;
}
void sqFreeQueryCb(void *param, TAOS_RES *pRes, int code) {
SSP_CB_PARAM *qParam = (SSP_CB_PARAM *)param;
if (code == 0 && pRes) {
if (qParam->fetch) {
taos_fetch_rows_a(pRes, sqFreeFetchCb, param);
} else {
taos_free_result(pRes);
*qParam->end = 1;
}
} else {
sqExit("select", taos_errstr(pRes));
}
}
void sqCloseFetchCb(void *param, TAOS_RES *pRes, int numOfRows) {
SSP_CB_PARAM *qParam = (SSP_CB_PARAM *)param;
taos_close(qParam->taos);
*qParam->end = 1;
}
void sqCloseQueryCb(void *param, TAOS_RES *pRes, int code) {
SSP_CB_PARAM *qParam = (SSP_CB_PARAM *)param;
if (code == 0 && pRes) {
if (qParam->fetch) {
taos_fetch_rows_a(pRes, sqFreeFetchCb, param);
} else {
taos_close(qParam->taos);
*qParam->end = 1;
}
} else {
sqExit("select", taos_errstr(pRes));
}
}
int sqSyncStopQuery(bool fetch) {
CASE_ENTER();
for (int32_t i = 0; i < runTimes; ++i) {
char sql[1024] = {0};
int32_t code = 0;
TAOS *taos = taos_connect(hostName, "root", "taosdata", NULL, 0);
if (taos == NULL) sqExit("taos_connect", taos_errstr(NULL));
sprintf(sql, "reset query cache");
sqExecSQL(taos, sql);
sprintf(sql, "use %s", dbName);
sqExecSQL(taos, sql);
sprintf(sql, "select * from %s", tbName);
TAOS_RES* pRes = taos_query(taos, sql);
code = taos_errno(pRes);
if (code) {
sqExit("taos_query", taos_errstr(pRes));
}
if (fetch) {
taos_fetch_row(pRes);
}
taos_stop_query(pRes);
taos_free_result(pRes);
taos_close(taos);
}
CASE_LEAVE();
}
int sqAsyncStopQuery(bool fetch) {
CASE_ENTER();
for (int32_t i = 0; i < runTimes; ++i) {
char sql[1024] = {0};
int32_t code = 0;
TAOS *taos = taos_connect(hostName, "root", "taosdata", NULL, 0);
if (taos == NULL) sqExit("taos_connect", taos_errstr(NULL));
sprintf(sql, "reset query cache");
sqExecSQL(taos, sql);
sprintf(sql, "use %s", dbName);
sqExecSQL(taos, sql);
sprintf(sql, "select * from %s", tbName);
int32_t qEnd = 0;
SSP_CB_PARAM param = {0};
param.fetch = fetch;
param.end = &qEnd;
taos_query_a(taos, sql, sqStopQueryCb, &param);
while (0 == qEnd) {
usleep(5000);
}
taos_close(taos);
}
CASE_LEAVE();
}
int sqSyncFreeQuery(bool fetch) {
CASE_ENTER();
for (int32_t i = 0; i < runTimes; ++i) {
char sql[1024] = {0};
int32_t code = 0;
TAOS *taos = taos_connect(hostName, "root", "taosdata", NULL, 0);
if (taos == NULL) sqExit("taos_connect", taos_errstr(NULL));
sprintf(sql, "reset query cache");
sqExecSQL(taos, sql);
sprintf(sql, "use %s", dbName);
sqExecSQL(taos, sql);
sprintf(sql, "select * from %s", tbName);
TAOS_RES* pRes = taos_query(taos, sql);
code = taos_errno(pRes);
if (code) {
sqExit("taos_query", taos_errstr(pRes));
}
if (fetch) {
taos_fetch_row(pRes);
}
taos_free_result(pRes);
taos_close(taos);
}
CASE_LEAVE();
}
int sqAsyncFreeQuery(bool fetch) {
CASE_ENTER();
for (int32_t i = 0; i < runTimes; ++i) {
char sql[1024] = {0};
int32_t code = 0;
TAOS *taos = taos_connect(hostName, "root", "taosdata", NULL, 0);
if (taos == NULL) sqExit("taos_connect", taos_errstr(NULL));
sprintf(sql, "reset query cache");
sqExecSQL(taos, sql);
sprintf(sql, "use %s", dbName);
sqExecSQL(taos, sql);
sprintf(sql, "select * from %s", tbName);
int32_t qEnd = 0;
SSP_CB_PARAM param = {0};
param.fetch = fetch;
param.end = &qEnd;
taos_query_a(taos, sql, sqFreeQueryCb, &param);
while (0 == qEnd) {
usleep(5000);
}
taos_close(taos);
}
CASE_LEAVE();
}
int sqSyncCloseQuery(bool fetch) {
CASE_ENTER();
for (int32_t i = 0; i < runTimes; ++i) {
char sql[1024] = {0};
int32_t code = 0;
TAOS *taos = taos_connect(hostName, "root", "taosdata", NULL, 0);
if (taos == NULL) sqExit("taos_connect", taos_errstr(NULL));
sprintf(sql, "reset query cache");
sqExecSQL(taos, sql);
sprintf(sql, "use %s", dbName);
sqExecSQL(taos, sql);
sprintf(sql, "select * from %s", tbName);
TAOS_RES* pRes = taos_query(taos, sql);
code = taos_errno(pRes);
if (code) {
sqExit("taos_query", taos_errstr(pRes));
}
if (fetch) {
taos_fetch_row(pRes);
}
taos_close(taos);
}
CASE_LEAVE();
}
int sqAsyncCloseQuery(bool fetch) {
CASE_ENTER();
for (int32_t i = 0; i < runTimes; ++i) {
char sql[1024] = {0};
int32_t code = 0;
TAOS *taos = taos_connect(hostName, "root", "taosdata", NULL, 0);
if (taos == NULL) sqExit("taos_connect", taos_errstr(NULL));
sprintf(sql, "reset query cache");
sqExecSQL(taos, sql);
sprintf(sql, "use %s", dbName);
sqExecSQL(taos, sql);
sprintf(sql, "select * from %s", tbName);
int32_t qEnd = 0;
SSP_CB_PARAM param = {0};
param.fetch = fetch;
param.end = &qEnd;
taos_query_a(taos, sql, sqFreeQueryCb, &param);
while (0 == qEnd) {
usleep(5000);
}
}
CASE_LEAVE();
}
void *syncQueryThreadFp(void *arg) {
SSP_CB_PARAM* qParam = (SSP_CB_PARAM*)arg;
char sql[1024] = {0};
int32_t code = 0;
TAOS *taos = taos_connect(hostName, "root", "taosdata", NULL, 0);
if (taos == NULL) sqExit("taos_connect", taos_errstr(NULL));
qParam->taos = taos;
sprintf(sql, "reset query cache");
sqExecSQLE(taos, sql);
sprintf(sql, "use %s", dbName);
sqExecSQLE(taos, sql);
sprintf(sql, "select * from %s", tbName);
TAOS_RES* pRes = taos_query(taos, sql);
if (qParam->fetch) {
taos_fetch_row(pRes);
}
taos_free_result(pRes);
}
void *closeThreadFp(void *arg) {
SSP_CB_PARAM* qParam = (SSP_CB_PARAM*)arg;
while (true) {
if (qParam->taos) {
usleep(rand() % 10000);
taos_close(qParam->taos);
break;
}
usleep(1);
}
}
int sqConSyncCloseQuery(bool fetch) {
CASE_ENTER();
pthread_t qid, cid;
for (int32_t i = 0; i < runTimes; ++i) {
SSP_CB_PARAM param = {0};
param.fetch = fetch;
pthread_create(&qid, NULL, syncQueryThreadFp, (void*)&param);
pthread_create(&cid, NULL, closeThreadFp, (void*)&param);
pthread_join(qid, NULL);
pthread_join(cid, NULL);
}
CASE_LEAVE();
}
void sqRunAllCase(void) {
/*
sqSyncStopQuery(false);
sqSyncStopQuery(true);
sqAsyncStopQuery(false);
sqAsyncStopQuery(true);
sqSyncFreeQuery(false);
sqSyncFreeQuery(true);
sqAsyncFreeQuery(false);
sqAsyncFreeQuery(true);
sqSyncCloseQuery(false);
sqSyncCloseQuery(true);
sqAsyncCloseQuery(false);
sqAsyncCloseQuery(true);
*/
sqConSyncCloseQuery(false);
sqConSyncCloseQuery(true);
}
int main(int argc, char *argv[]) {
if (argc != 4) {
printf("usage: %s server-ip dbname tablename\n", argv[0]);
exit(0);
}
srand((unsigned int)time(NULL));
strcpy(hostName, argv[1]);
strcpy(dbName, argv[2]);
strcpy(tbName, argv[3]);
sqRunAllCase();
return 0;
}

View File

@ -124,7 +124,7 @@ if $data(5)[3] != 3 then
return -1 return -1
endi endi
#sql reset query cache sql reset query cache
print =============== step4: select data print =============== step4: select data
sql show d1.tables sql show d1.tables

View File

@ -199,7 +199,7 @@ class TDTestCase:
tdSql.checkData(0, 0, "true") tdSql.checkData(0, 0, "true")
# test select json tag->'key', value is null # test select json tag->'key', value is null
tdSql.query("select jtag->'tag1' from jsons1_4") tdSql.query("select jtag->'tag1' from jsons1_4")
tdSql.checkData(0, 0, "null") tdSql.checkData(0, 0, None)
# test select json tag->'key', value is double # test select json tag->'key', value is double
tdSql.query("select jtag->'tag1' from jsons1_5") tdSql.query("select jtag->'tag1' from jsons1_5")
tdSql.checkData(0, 0, "1.232000000") tdSql.checkData(0, 0, "1.232000000")

View File

@ -23,7 +23,7 @@ python3 ./test.py -f 1-insert/alter_stable.py
python3 ./test.py -f 1-insert/alter_table.py python3 ./test.py -f 1-insert/alter_table.py
python3 ./test.py -f 1-insert/insertWithMoreVgroup.py python3 ./test.py -f 1-insert/insertWithMoreVgroup.py
python3 ./test.py -f 1-insert/table_comment.py python3 ./test.py -f 1-insert/table_comment.py
python3 ./test.py -f 1-insert/table_param_ttl.py #python3 ./test.py -f 1-insert/table_param_ttl.py
python3 ./test.py -f 2-query/between.py python3 ./test.py -f 2-query/between.py
python3 ./test.py -f 2-query/distinct.py python3 ./test.py -f 2-query/distinct.py
python3 ./test.py -f 2-query/varchar.py python3 ./test.py -f 2-query/varchar.py