xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

[td-11818] merge 3.0

This commit is contained in:
Haojun Liao 2022-01-24 13:44:20 +08:00
parent b3fadd5c93 f739262007
commit c87fbd87df
84 changed files with 2164 additions and 1040 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
2.0/src/client/src/tscGlobalmerge.c
View File

@ -701,7 +701,7 @@ SGlobalMerger* tscInitResObjForLocalQuery(int32_t numOfRes, int32_t rowLen, uint
// todo remove it // todo remove it
int32_t doArithmeticCalculate(SQueryInfo* pQueryInfo, tFilePage* pOutput, int32_t rowSize, int32_t finalRowSize) { int32_t doArithmeticCalculate(SQueryInfo* pQueryInfo, tFilePage* pOutput, int32_t rowSize, int32_t finalRowSize) {
int32_t maxRowSize = MAX(rowSize, finalRowSize); int32_t maxRowSize = TMAX(rowSize, finalRowSize);
char* pbuf = calloc(1, (size_t)(pOutput->num * maxRowSize)); char* pbuf = calloc(1, (size_t)(pOutput->num * maxRowSize));
size_t size = tscNumOfFields(pQueryInfo); size_t size = tscNumOfFields(pQueryInfo);
@ -965,7 +965,7 @@ SSDataBlock* doGlobalAggregate(void* param, bool* newgroup) {
w->ekey = *(int64_t*)(((char*)pInfoData->pData) + TSDB_KEYSIZE * (pRes->info.rows - 1)); w->ekey = *(int64_t*)(((char*)pInfoData->pData) + TSDB_KEYSIZE * (pRes->info.rows - 1));
if (pOperator->pRuntimeEnv->pQueryAttr->order.order == TSDB_ORDER_DESC) { if (pOperator->pRuntimeEnv->pQueryAttr->order.order == TSDB_ORDER_DESC) {
SWAP(w->skey, w->ekey, TSKEY); TSWAP(w->skey, w->ekey, TSKEY);
assert(w->skey <= w->ekey); assert(w->skey <= w->ekey);
} }
} }

4
2.0/src/client/src/tscParseLineProtocol.c
View File

@ -47,7 +47,7 @@ int compareSmlColKv(const void* p1, const void* p2) {
TAOS_SML_KV* kv2 = (TAOS_SML_KV*)p2; TAOS_SML_KV* kv2 = (TAOS_SML_KV*)p2;
int kvLen1 = (int)strlen(kv1->key); int kvLen1 = (int)strlen(kv1->key);
int kvLen2 = (int)strlen(kv2->key); int kvLen2 = (int)strlen(kv2->key);
int res = strncasecmp(kv1->key, kv2->key, MIN(kvLen1, kvLen2)); int res = strncasecmp(kv1->key, kv2->key, TMIN(kvLen1, kvLen2));
if (res != 0) { if (res != 0) {
return res; return res;
} else { } else {
@ -123,7 +123,7 @@ static int32_t buildSmlKvSchema(TAOS_SML_KV* smlKv, SHashObj* hash, SArray* arra
if (code != 0) { if (code != 0) {
return code; return code;
} }
pField->bytes = MAX(pField->bytes, bytes); pField->bytes = TMAX(pField->bytes, bytes);
} else { } else {
SSchema field = {0}; SSchema field = {0};

12
2.0/src/client/src/tscSQLParser.c
View File

@ -1785,7 +1785,7 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32
getNewResColId(pCmd), sizeof(double), false); getNewResColId(pCmd), sizeof(double), false);
char* name = (pItem->aliasName != NULL)? pItem->aliasName:pItem->pNode->exprToken.z; char* name = (pItem->aliasName != NULL)? pItem->aliasName:pItem->pNode->exprToken.z;
size_t len = MIN(sizeof(pExpr->base.aliasName), pItem->pNode->exprToken.n + 1); size_t len = TMIN(sizeof(pExpr->base.aliasName), pItem->pNode->exprToken.n + 1);
tstrncpy(pExpr->base.aliasName, name, len); tstrncpy(pExpr->base.aliasName, name, len);
tExprNode* pNode = NULL; tExprNode* pNode = NULL;
@ -2246,7 +2246,7 @@ SSchema tGetUserSpecifiedColumnSchema(SVariant* pVal, SStrToken* exprStr, const
if (name != NULL) { if (name != NULL) {
tstrncpy(s.name, name, sizeof(s.name)); tstrncpy(s.name, name, sizeof(s.name));
} else { } else {
size_t tlen = MIN(sizeof(s.name), exprStr->n + 1); size_t tlen = TMIN(sizeof(s.name), exprStr->n + 1);
tstrncpy(s.name, exprStr->z, tlen); tstrncpy(s.name, exprStr->z, tlen);
strdequote(s.name); strdequote(s.name);
} }
@ -2419,7 +2419,7 @@ void setResultColName(char* name, tSqlExprItem* pItem, int32_t functionId, SStrT
tstrncpy(name, pItem->aliasName, TSDB_COL_NAME_LEN); tstrncpy(name, pItem->aliasName, TSDB_COL_NAME_LEN);
} else if (multiCols) { } else if (multiCols) {
char uname[TSDB_COL_NAME_LEN] = {0}; char uname[TSDB_COL_NAME_LEN] = {0};
int32_t len = MIN(pToken->n + 1, TSDB_COL_NAME_LEN); int32_t len = TMIN(pToken->n + 1, TSDB_COL_NAME_LEN);
tstrncpy(uname, pToken->z, len); tstrncpy(uname, pToken->z, len);
if (tsKeepOriginalColumnName) { // keep the original column name if (tsKeepOriginalColumnName) { // keep the original column name
@ -2432,7 +2432,7 @@ void setResultColName(char* name, tSqlExprItem* pItem, int32_t functionId, SStrT
tstrncpy(name, tmp, TSDB_COL_NAME_LEN); tstrncpy(name, tmp, TSDB_COL_NAME_LEN);
} }
} else { // use the user-input result column name } else { // use the user-input result column name
int32_t len = MIN(pItem->pNode->exprToken.n + 1, TSDB_COL_NAME_LEN); int32_t len = TMIN(pItem->pNode->exprToken.n + 1, TSDB_COL_NAME_LEN);
tstrncpy(name, pItem->pNode->exprToken.z, len); tstrncpy(name, pItem->pNode->exprToken.z, len);
} }
} }
@ -4380,7 +4380,7 @@ static void exchangeExpr(tSqlExpr* pExpr) {
} }
pExpr->tokenId = optr; pExpr->tokenId = optr;
SWAP(pExpr->pLeft, pExpr->pRight, void*); TSWAP(pExpr->pLeft, pExpr->pRight, void*);
} }
} }
@ -8089,7 +8089,7 @@ int32_t checkQueryRangeForFill(SSqlCmd* pCmd, SQueryInfo* pQueryInfo) {
return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3);
} }
int64_t timeRange = ABS(pQueryInfo->window.skey - pQueryInfo->window.ekey); int64_t timeRange = TABS(pQueryInfo->window.skey - pQueryInfo->window.ekey);
int64_t intervalRange = 0; int64_t intervalRange = 0;
if (pQueryInfo->interval.intervalUnit == 'n' || pQueryInfo->interval.intervalUnit == 'y') { if (pQueryInfo->interval.intervalUnit == 'n' || pQueryInfo->interval.intervalUnit == 'y') {

10
2.0/src/client/src/tscSql.c
View File

@ -120,7 +120,7 @@ static SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pa
pObj->signature = pObj; pObj->signature = pObj;
pObj->pRpcObj = (SRpcObj *)pRpcObj; pObj->pRpcObj = (SRpcObj *)pRpcObj;
tstrncpy(pObj->user, user, sizeof(pObj->user)); tstrncpy(pObj->user, user, sizeof(pObj->user));
secretEncryptLen = MIN(secretEncryptLen, sizeof(pObj->pass)); secretEncryptLen = TMIN(secretEncryptLen, sizeof(pObj->pass));
memcpy(pObj->pass, secretEncrypt, secretEncryptLen); memcpy(pObj->pass, secretEncrypt, secretEncryptLen);
if (db) { if (db) {
@ -245,10 +245,10 @@ TAOS *taos_connect_c(const char *ip, uint8_t ipLen, const char *user, uint8_t us
char userBuf[TSDB_USER_LEN] = {0}; char userBuf[TSDB_USER_LEN] = {0};
char passBuf[TSDB_KEY_LEN] = {0}; char passBuf[TSDB_KEY_LEN] = {0};
char dbBuf[TSDB_DB_NAME_LEN] = {0}; char dbBuf[TSDB_DB_NAME_LEN] = {0};
strncpy(ipBuf, ip, MIN(TSDB_EP_LEN - 1, ipLen)); strncpy(ipBuf, ip, TMIN(TSDB_EP_LEN - 1, ipLen));
strncpy(userBuf, user, MIN(TSDB_USER_LEN - 1, userLen)); strncpy(userBuf, user, TMIN(TSDB_USER_LEN - 1, userLen));
strncpy(passBuf, pass, MIN(TSDB_KEY_LEN - 1, passLen)); strncpy(passBuf, pass, TMIN(TSDB_KEY_LEN - 1, passLen));
strncpy(dbBuf, db, MIN(TSDB_DB_NAME_LEN - 1, dbLen)); strncpy(dbBuf, db, TMIN(TSDB_DB_NAME_LEN - 1, dbLen));
return taos_connect(ipBuf, userBuf, passBuf, dbBuf, port); return taos_connect(ipBuf, userBuf, passBuf, dbBuf, port);
} }

2
2.0/src/client/src/tscSubquery.c
View File

@ -3396,7 +3396,7 @@ static void doBuildResFromSubqueries(SSqlObj* pSql) {
} }
int32_t remain = (int32_t)(pSub->res.numOfRows - pSub->res.row); int32_t remain = (int32_t)(pSub->res.numOfRows - pSub->res.row);
numOfRes = (int32_t)(MIN(numOfRes, remain)); numOfRes = (int32_t)(TMIN(numOfRes, remain));
} }
if (numOfRes == 0) { // no result any more, free all subquery objects if (numOfRes == 0) { // no result any more, free all subquery objects

12
2.0/src/query/src/qAggMain.c
View File

@ -373,7 +373,7 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI
*interBytes = (int16_t)sizeof(SPercentileInfo); *interBytes = (int16_t)sizeof(SPercentileInfo);
} else if (functionId == TSDB_FUNC_LEASTSQR) { } else if (functionId == TSDB_FUNC_LEASTSQR) {
*type = TSDB_DATA_TYPE_BINARY; *type = TSDB_DATA_TYPE_BINARY;
*bytes = MAX(TSDB_AVG_FUNCTION_INTER_BUFFER_SIZE, sizeof(SLeastsquaresInfo)); // string *bytes = TMAX(TSDB_AVG_FUNCTION_INTER_BUFFER_SIZE, sizeof(SLeastsquaresInfo)); // string
*interBytes = *bytes; *interBytes = *bytes;
} else if (functionId == TSDB_FUNC_FIRST_DST || functionId == TSDB_FUNC_LAST_DST) { } else if (functionId == TSDB_FUNC_FIRST_DST || functionId == TSDB_FUNC_LAST_DST) {
*type = TSDB_DATA_TYPE_BINARY; *type = TSDB_DATA_TYPE_BINARY;
@ -1275,7 +1275,7 @@ static void max_func_merge(SQLFunctionCtx *pCtx) {
continue; \ continue; \
} \ } \
(num) += 1; \ (num) += 1; \
(r) += POW2(((type *)d)[i] - (delta)); \ (r) += TPOW2(((type *)d)[i] - (delta)); \
} }
static void stddev_function(SQLFunctionCtx *pCtx) { static void stddev_function(SQLFunctionCtx *pCtx) {
@ -1314,7 +1314,7 @@ static void stddev_function(SQLFunctionCtx *pCtx) {
continue; continue;
} }
num += 1; num += 1;
*retVal += POW2(((int32_t *)pData)[i] - avg); *retVal += TPOW2(((int32_t *)pData)[i] - avg);
} }
break; break;
} }
@ -1427,7 +1427,7 @@ static void stddev_dst_function(SQLFunctionCtx *pCtx) {
continue; continue;
} }
num += 1; num += 1;
*retVal += POW2(((int32_t *)pData)[i] - avg); *retVal += TPOW2(((int32_t *)pData)[i] - avg);
} }
break; break;
} }
@ -4109,8 +4109,8 @@ static void mergeTableBlockDist(SResultRowCellInfo* pResInfo, const STableBlockD
pDist->totalRows += pSrc->totalRows; pDist->totalRows += pSrc->totalRows;
if (pResInfo->hasResult == DATA_SET_FLAG) { if (pResInfo->hasResult == DATA_SET_FLAG) {
pDist->maxRows = MAX(pDist->maxRows, pSrc->maxRows); pDist->maxRows = TMAX(pDist->maxRows, pSrc->maxRows);
pDist->minRows = MIN(pDist->minRows, pSrc->minRows); pDist->minRows = TMIN(pDist->minRows, pSrc->minRows);
} else { } else {
pDist->maxRows = pSrc->maxRows; pDist->maxRows = pSrc->maxRows;
pDist->minRows = pSrc->minRows; pDist->minRows = pSrc->minRows;

34
2.0/src/query/src/qExecutor.c
View File

@ -328,7 +328,7 @@ SSDataBlock* createOutputBuf(SExprInfo* pExpr, int32_t numOfOutput, int32_t numO
idata.info.bytes = pExpr[i].base.resBytes; idata.info.bytes = pExpr[i].base.resBytes;
idata.info.colId = pExpr[i].base.resColId; idata.info.colId = pExpr[i].base.resColId;
int32_t size = MAX(idata.info.bytes * numOfRows, minSize); int32_t size = TMAX(idata.info.bytes * numOfRows, minSize);
idata.pData = calloc(1, size); // at least to hold a pointer on x64 platform idata.pData = calloc(1, size); // at least to hold a pointer on x64 platform
taosArrayPush(res->pDataBlock, &idata); taosArrayPush(res->pDataBlock, &idata);
} }
@ -2643,7 +2643,7 @@ static void updateDataCheckOrder(SQInfo *pQInfo, SQueryTableMsg* pQueryMsg, bool
pQueryAttr->order.order = TSDB_ORDER_ASC; pQueryAttr->order.order = TSDB_ORDER_ASC;
if (pQueryAttr->window.skey > pQueryAttr->window.ekey) { if (pQueryAttr->window.skey > pQueryAttr->window.ekey) {
SWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); TSWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY);
} }
pQueryAttr->needReverseScan = false; pQueryAttr->needReverseScan = false;
@ -2653,7 +2653,7 @@ static void updateDataCheckOrder(SQInfo *pQInfo, SQueryTableMsg* pQueryMsg, bool
if (pQueryAttr->groupbyColumn && pQueryAttr->order.order == TSDB_ORDER_DESC) { if (pQueryAttr->groupbyColumn && pQueryAttr->order.order == TSDB_ORDER_DESC) {
pQueryAttr->order.order = TSDB_ORDER_ASC; pQueryAttr->order.order = TSDB_ORDER_ASC;
if (pQueryAttr->window.skey > pQueryAttr->window.ekey) { if (pQueryAttr->window.skey > pQueryAttr->window.ekey) {
SWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); TSWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY);
} }
pQueryAttr->needReverseScan = false; pQueryAttr->needReverseScan = false;
@ -2664,7 +2664,7 @@ static void updateDataCheckOrder(SQInfo *pQInfo, SQueryTableMsg* pQueryMsg, bool
if (pQueryAttr->pointInterpQuery && pQueryAttr->interval.interval == 0) { if (pQueryAttr->pointInterpQuery && pQueryAttr->interval.interval == 0) {
if (!QUERY_IS_ASC_QUERY(pQueryAttr)) { if (!QUERY_IS_ASC_QUERY(pQueryAttr)) {
qDebug(msg, pQInfo->qId, "interp", pQueryAttr->order.order, TSDB_ORDER_ASC, pQueryAttr->window.skey, pQueryAttr->window.ekey, pQueryAttr->window.ekey, pQueryAttr->window.skey); qDebug(msg, pQInfo->qId, "interp", pQueryAttr->order.order, TSDB_ORDER_ASC, pQueryAttr->window.skey, pQueryAttr->window.ekey, pQueryAttr->window.ekey, pQueryAttr->window.skey);
SWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); TSWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY);
} }
pQueryAttr->order.order = TSDB_ORDER_ASC; pQueryAttr->order.order = TSDB_ORDER_ASC;
@ -2677,7 +2677,7 @@ static void updateDataCheckOrder(SQInfo *pQInfo, SQueryTableMsg* pQueryMsg, bool
qDebug(msg, pQInfo->qId, "only-first", pQueryAttr->order.order, TSDB_ORDER_ASC, pQueryAttr->window.skey, qDebug(msg, pQInfo->qId, "only-first", pQueryAttr->order.order, TSDB_ORDER_ASC, pQueryAttr->window.skey,
pQueryAttr->window.ekey, pQueryAttr->window.ekey, pQueryAttr->window.skey); pQueryAttr->window.ekey, pQueryAttr->window.ekey, pQueryAttr->window.skey);
SWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); TSWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY);
doUpdateLastKey(pQueryAttr); doUpdateLastKey(pQueryAttr);
} }
@ -2688,7 +2688,7 @@ static void updateDataCheckOrder(SQInfo *pQInfo, SQueryTableMsg* pQueryMsg, bool
qDebug(msg, pQInfo->qId, "only-last", pQueryAttr->order.order, TSDB_ORDER_DESC, pQueryAttr->window.skey, qDebug(msg, pQInfo->qId, "only-last", pQueryAttr->order.order, TSDB_ORDER_DESC, pQueryAttr->window.skey,
pQueryAttr->window.ekey, pQueryAttr->window.ekey, pQueryAttr->window.skey); pQueryAttr->window.ekey, pQueryAttr->window.ekey, pQueryAttr->window.skey);
SWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); TSWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY);
doUpdateLastKey(pQueryAttr); doUpdateLastKey(pQueryAttr);
} }
@ -2703,7 +2703,7 @@ static void updateDataCheckOrder(SQInfo *pQInfo, SQueryTableMsg* pQueryMsg, bool
qDebug(msg, pQInfo->qId, "only-first stable", pQueryAttr->order.order, TSDB_ORDER_ASC, qDebug(msg, pQInfo->qId, "only-first stable", pQueryAttr->order.order, TSDB_ORDER_ASC,
pQueryAttr->window.skey, pQueryAttr->window.ekey, pQueryAttr->window.ekey, pQueryAttr->window.skey); pQueryAttr->window.skey, pQueryAttr->window.ekey, pQueryAttr->window.ekey, pQueryAttr->window.skey);
SWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); TSWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY);
doUpdateLastKey(pQueryAttr); doUpdateLastKey(pQueryAttr);
} }
@ -2714,7 +2714,7 @@ static void updateDataCheckOrder(SQInfo *pQInfo, SQueryTableMsg* pQueryMsg, bool
qDebug(msg, pQInfo->qId, "only-last stable", pQueryAttr->order.order, TSDB_ORDER_DESC, qDebug(msg, pQInfo->qId, "only-last stable", pQueryAttr->order.order, TSDB_ORDER_DESC,
pQueryAttr->window.skey, pQueryAttr->window.ekey, pQueryAttr->window.ekey, pQueryAttr->window.skey); pQueryAttr->window.skey, pQueryAttr->window.ekey, pQueryAttr->window.ekey, pQueryAttr->window.skey);
SWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); TSWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY);
doUpdateLastKey(pQueryAttr); doUpdateLastKey(pQueryAttr);
} }
@ -2754,8 +2754,8 @@ static FORCE_INLINE bool doFilterByBlockStatistics(SQueryRuntimeEnv* pRuntimeEnv
static bool overlapWithTimeWindow(SQueryAttr* pQueryAttr, SDataBlockInfo* pBlockInfo) { static bool overlapWithTimeWindow(SQueryAttr* pQueryAttr, SDataBlockInfo* pBlockInfo) {
STimeWindow w = {0}; STimeWindow w = {0};
TSKEY sk = MIN(pQueryAttr->window.skey, pQueryAttr->window.ekey); TSKEY sk = TMIN(pQueryAttr->window.skey, pQueryAttr->window.ekey);
TSKEY ek = MAX(pQueryAttr->window.skey, pQueryAttr->window.ekey); TSKEY ek = TMAX(pQueryAttr->window.skey, pQueryAttr->window.ekey);
if (QUERY_IS_ASC_QUERY(pQueryAttr)) { if (QUERY_IS_ASC_QUERY(pQueryAttr)) {
getAlignQueryTimeWindow(pQueryAttr, pBlockInfo->window.skey, sk, ek, &w); getAlignQueryTimeWindow(pQueryAttr, pBlockInfo->window.skey, sk, ek, &w);
@ -3523,7 +3523,7 @@ static void updateTableQueryInfoForReverseScan(STableQueryInfo *pTableQueryInfo)
return; return;
} }
SWAP(pTableQueryInfo->win.skey, pTableQueryInfo->win.ekey, TSKEY); TSWAP(pTableQueryInfo->win.skey, pTableQueryInfo->win.ekey, TSKEY);
pTableQueryInfo->lastKey = pTableQueryInfo->win.skey; pTableQueryInfo->lastKey = pTableQueryInfo->win.skey;
SWITCH_ORDER(pTableQueryInfo->cur.order); SWITCH_ORDER(pTableQueryInfo->cur.order);
@ -3730,7 +3730,7 @@ static void setupEnvForReverseScan(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo
} }
// reverse order time range // reverse order time range
SWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); TSWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY);
SET_REVERSE_SCAN_FLAG(pRuntimeEnv); SET_REVERSE_SCAN_FLAG(pRuntimeEnv);
setQueryStatus(pRuntimeEnv, QUERY_NOT_COMPLETED); setQueryStatus(pRuntimeEnv, QUERY_NOT_COMPLETED);
@ -4109,8 +4109,8 @@ void setIntervalQueryRange(SQueryRuntimeEnv *pRuntimeEnv, TSKEY key) {
*/ */
STimeWindow w = TSWINDOW_INITIALIZER; STimeWindow w = TSWINDOW_INITIALIZER;
TSKEY sk = MIN(win.skey, win.ekey); TSKEY sk = TMIN(win.skey, win.ekey);
TSKEY ek = MAX(win.skey, win.ekey); TSKEY ek = TMAX(win.skey, win.ekey);
getAlignQueryTimeWindow(pQueryAttr, win.skey, sk, ek, &w); getAlignQueryTimeWindow(pQueryAttr, win.skey, sk, ek, &w);
// if (pResultRowInfo->prevSKey == TSKEY_INITIAL_VAL) { // if (pResultRowInfo->prevSKey == TSKEY_INITIAL_VAL) {
@ -4235,7 +4235,7 @@ static void updateNumOfRowsInResultRows(SQueryRuntimeEnv* pRuntimeEnv, SQLFuncti
} }
SResultRowCellInfo* pCell = getResultCell(pResult, j, rowCellInfoOffset); SResultRowCellInfo* pCell = getResultCell(pResult, j, rowCellInfoOffset);
pResult->numOfRows = (uint16_t)(MAX(pResult->numOfRows, pCell->numOfRes)); pResult->numOfRows = (uint16_t)(TMAX(pResult->numOfRows, pCell->numOfRes));
} }
} }
} }
@ -6945,8 +6945,8 @@ SOperatorInfo* createFillOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorIn
SFillColInfo* pColInfo = createFillColInfo(pExpr, numOfOutput, pQueryAttr->fillVal); SFillColInfo* pColInfo = createFillColInfo(pExpr, numOfOutput, pQueryAttr->fillVal);
STimeWindow w = TSWINDOW_INITIALIZER; STimeWindow w = TSWINDOW_INITIALIZER;
TSKEY sk = MIN(pQueryAttr->window.skey, pQueryAttr->window.ekey); TSKEY sk = TMIN(pQueryAttr->window.skey, pQueryAttr->window.ekey);
TSKEY ek = MAX(pQueryAttr->window.skey, pQueryAttr->window.ekey); TSKEY ek = TMAX(pQueryAttr->window.skey, pQueryAttr->window.ekey);
getAlignQueryTimeWindow(pQueryAttr, pQueryAttr->window.skey, sk, ek, &w); getAlignQueryTimeWindow(pQueryAttr, pQueryAttr->window.skey, sk, ek, &w);
pInfo->pFillInfo = pInfo->pFillInfo =

2
cmake/traft_CMakeLists.txt.in
View File

@ -7,7 +7,7 @@ ExternalProject_Add(traft
BINARY_DIR "${CMAKE_CONTRIB_DIR}/traft" BINARY_DIR "${CMAKE_CONTRIB_DIR}/traft"
#BUILD_IN_SOURCE TRUE #BUILD_IN_SOURCE TRUE
# https://answers.ros.org/question/333125/how-to-include-external-automakeautoconf-projects-into-ament_cmake/ # https://answers.ros.org/question/333125/how-to-include-external-automakeautoconf-projects-into-ament_cmake/
CONFIGURE_COMMAND COMMAND autoreconf -i COMMAND ./configure --enable-example CONFIGURE_COMMAND COMMAND autoreconf -i COMMAND ./configure
BUILD_COMMAND "$(MAKE)" BUILD_COMMAND "$(MAKE)"
INSTALL_COMMAND "" INSTALL_COMMAND ""
TEST_COMMAND "" TEST_COMMAND ""

29
include/client/taos.h
View File

@ -192,9 +192,38 @@ DLL_EXPORT void taos_close_stream(TAOS_STREAM *tstr);
DLL_EXPORT int taos_load_table_info(TAOS *taos, const char* tableNameList); DLL_EXPORT int taos_load_table_info(TAOS *taos, const char* tableNameList);
DLL_EXPORT TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int protocol, int precision); DLL_EXPORT TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int protocol, int precision);
typedef struct tmq_t tmq_t;
typedef struct tmq_conf_t tmq_conf_t;
typedef struct tmq_list_t tmq_list_t;
typedef struct tmq_message_t tmq_message_t;
typedef struct tmq_message_topic_t tmq_message_topic_t;
typedef struct tmq_message_tb_t tmq_message_tb_t;
typedef struct tmq_tb_iter_t tmq_tb_iter_t;
typedef struct tmq_message_col_t tmq_message_col_t;
typedef struct tmq_col_iter_t tmq_col_iter_t;
DLL_EXPORT tmq_list_t* tmq_list_new();
DLL_EXPORT int32_t tmq_list_append(tmq_list_t*, char*);
DLL_EXPORT tmq_conf_t* tmq_conf_new();
DLL_EXPORT int32_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value);
DLL_EXPORT TAOS_RES *taos_create_topic(TAOS* taos, const char* name, const char* sql, int sqlLen); DLL_EXPORT TAOS_RES *taos_create_topic(TAOS* taos, const char* name, const char* sql, int sqlLen);
DLL_EXPORT tmq_t* taos_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, int32_t errstrLen);
DLL_EXPORT TAOS_RES* tmq_subscribe(tmq_t* tmq, tmq_list_t* topic_list);
DLL_EXPORT tmq_message_t* tmq_consume_poll(tmq_t* tmq, int64_t blocking_time);
DLL_EXPORT int32_t tmq_topic_num(tmq_message_t* msg);
DLL_EXPORT char* tmq_get_topic(tmq_message_topic_t* msg);
DLL_EXPORT int32_t tmq_get_vgId(tmq_message_topic_t* msg);
DLL_EXPORT tmq_message_tb_t* tmq_get_next_tb(tmq_message_topic_t* msg, tmq_tb_iter_t* iter);
DLL_EXPORT tmq_message_col_t* tmq_get_next_col(tmq_message_tb_t* msg, tmq_col_iter_t* iter);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

2
include/common/tdataformat.h
View File

@ -136,7 +136,7 @@ typedef uint64_t TKEY;
#define TKEY_IS_NEGATIVE(tkey) (((tkey)&TKEY_NEGATIVE_FLAG) != 0) #define TKEY_IS_NEGATIVE(tkey) (((tkey)&TKEY_NEGATIVE_FLAG) != 0)
#define TKEY_IS_DELETED(tkey) (((tkey)&TKEY_DELETE_FLAG) != 0) #define TKEY_IS_DELETED(tkey) (((tkey)&TKEY_DELETE_FLAG) != 0)
#define tdSetTKEYDeleted(tkey) ((tkey) | TKEY_DELETE_FLAG) #define tdSetTKEYDeleted(tkey) ((tkey) | TKEY_DELETE_FLAG)
#define tdGetTKEY(key) (((TKEY)ABS(key)) | (TKEY_NEGATIVE_FLAG & (TKEY)(key))) #define tdGetTKEY(key) (((TKEY)TABS(key)) | (TKEY_NEGATIVE_FLAG & (TKEY)(key)))
#define tdGetKey(tkey) (((TSKEY)((tkey)&TKEY_VALUE_FILTER)) * (TKEY_IS_NEGATIVE(tkey) ? -1 : 1)) #define tdGetKey(tkey) (((TSKEY)((tkey)&TKEY_VALUE_FILTER)) * (TKEY_IS_NEGATIVE(tkey) ? -1 : 1))
#define MIN_TS_KEY ((TSKEY)0x8000000000000001) #define MIN_TS_KEY ((TSKEY)0x8000000000000001)

37
include/common/tmsg.h
View File

@ -1519,7 +1519,8 @@ static FORCE_INLINE void* taosDecodeSMqMsg(void* buf, SMqHbMsg* pMsg) {
typedef struct SMqSetCVgReq { typedef struct SMqSetCVgReq {
int32_t vgId; int32_t vgId;
int64_t consumerId; int64_t oldConsumerId;
int64_t newConsumerId;
char topicName[TSDB_TOPIC_FNAME_LEN]; char topicName[TSDB_TOPIC_FNAME_LEN];
char cgroup[TSDB_CONSUMER_GROUP_LEN]; char cgroup[TSDB_CONSUMER_GROUP_LEN];
char* sql; char* sql;
@ -1550,7 +1551,8 @@ static FORCE_INLINE void* tDecodeSSubQueryMsg(void* buf, SSubQueryMsg* pMsg) {
static FORCE_INLINE int32_t tEncodeSMqSetCVgReq(void** buf, const SMqSetCVgReq* pReq) { static FORCE_INLINE int32_t tEncodeSMqSetCVgReq(void** buf, const SMqSetCVgReq* pReq) {
int32_t tlen = 0; int32_t tlen = 0;
tlen += taosEncodeFixedI32(buf, pReq->vgId); tlen += taosEncodeFixedI32(buf, pReq->vgId);
tlen += taosEncodeFixedI64(buf, pReq->consumerId); tlen += taosEncodeFixedI64(buf, pReq->oldConsumerId);
tlen += taosEncodeFixedI64(buf, pReq->newConsumerId);
tlen += taosEncodeString(buf, pReq->topicName); tlen += taosEncodeString(buf, pReq->topicName);
tlen += taosEncodeString(buf, pReq->cgroup); tlen += taosEncodeString(buf, pReq->cgroup);
tlen += taosEncodeString(buf, pReq->sql); tlen += taosEncodeString(buf, pReq->sql);
@ -1562,7 +1564,8 @@ static FORCE_INLINE int32_t tEncodeSMqSetCVgReq(void** buf, const SMqSetCVgReq*
static FORCE_INLINE void* tDecodeSMqSetCVgReq(void* buf, SMqSetCVgReq* pReq) { static FORCE_INLINE void* tDecodeSMqSetCVgReq(void* buf, SMqSetCVgReq* pReq) {
buf = taosDecodeFixedI32(buf, &pReq->vgId); buf = taosDecodeFixedI32(buf, &pReq->vgId);
buf = taosDecodeFixedI64(buf, &pReq->consumerId); buf = taosDecodeFixedI64(buf, &pReq->oldConsumerId);
buf = taosDecodeFixedI64(buf, &pReq->newConsumerId);
buf = taosDecodeStringTo(buf, pReq->topicName); buf = taosDecodeStringTo(buf, pReq->topicName);
buf = taosDecodeStringTo(buf, pReq->cgroup); buf = taosDecodeStringTo(buf, pReq->cgroup);
buf = taosDecodeString(buf, &pReq->sql); buf = taosDecodeString(buf, &pReq->sql);
@ -1579,15 +1582,6 @@ typedef struct SMqSetCVgRsp {
char cGroup[TSDB_CONSUMER_GROUP_LEN]; char cGroup[TSDB_CONSUMER_GROUP_LEN];
} SMqSetCVgRsp; } SMqSetCVgRsp;
typedef struct SMqConsumeReq {
int64_t reqId;
int64_t offset;
int64_t consumerId;
int64_t blockingTime;
char topicName[TSDB_TOPIC_FNAME_LEN];
char cgroup[TSDB_CONSUMER_GROUP_LEN];
} SMqConsumeReq;
typedef struct SMqColData { typedef struct SMqColData {
int16_t colId; int16_t colId;
int16_t type; int16_t type;
@ -1615,12 +1609,29 @@ typedef struct SMqTopicBlk {
typedef struct SMqConsumeRsp { typedef struct SMqConsumeRsp {
int64_t reqId; int64_t reqId;
int64_t clientId; int64_t consumerId;
int32_t bodyLen; int32_t bodyLen;
int32_t numOfTopics; int32_t numOfTopics;
SMqTopicData data[]; SMqTopicData data[];
} SMqConsumeRsp; } SMqConsumeRsp;
// one req for one vg+topic
typedef struct SMqConsumeReq {
//0: commit only, current offset
//1: consume only, poll next offset
//2: commit current and consume next offset
int32_t reqType;
int64_t reqId;
int64_t consumerId;
int64_t blockingTime;
char cgroup[TSDB_CONSUMER_GROUP_LEN];
int64_t offset;
char topic[TSDB_TOPIC_FNAME_LEN];
} SMqConsumeReq;
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

8
include/libs/scheduler/scheduler.h
View File

@ -72,7 +72,7 @@ int32_t schedulerInit(SSchedulerCfg *cfg);
* @param nodeList Qnode/Vnode address list, element is SQueryNodeAddr * @param nodeList Qnode/Vnode address list, element is SQueryNodeAddr
* @return * @return
*/ */
int32_t scheduleExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, struct SSchJob** pJob, SQueryResult *pRes); int32_t schedulerExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, struct SSchJob** pJob, SQueryResult *pRes);
/** /**
* Process the query job, generated according to the query physical plan. * Process the query job, generated according to the query physical plan.
@ -80,7 +80,7 @@ int32_t scheduleExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, stru
* @param pNodeList Qnode/Vnode address list, element is SQueryNodeAddr * @param pNodeList Qnode/Vnode address list, element is SQueryNodeAddr
* @return * @return
*/ */
int32_t scheduleAsyncExecJob(void *transport, SArray *pNodeList, SQueryDag* pDag, struct SSchJob** pJob); int32_t schedulerAsyncExecJob(void *transport, SArray *pNodeList, SQueryDag* pDag, struct SSchJob** pJob);
/** /**
* Fetch query result from the remote query executor * Fetch query result from the remote query executor
@ -88,7 +88,7 @@ int32_t scheduleAsyncExecJob(void *transport, SArray *pNodeList, SQueryDag* pDag
* @param data * @param data
* @return * @return
*/ */
int32_t scheduleFetchRows(struct SSchJob *pJob, void **data); int32_t schedulerFetchRows(struct SSchJob *pJob, void **data);
/** /**
@ -102,7 +102,7 @@ int32_t scheduleFetchRows(struct SSchJob *pJob, void **data);
* Free the query job * Free the query job
* @param pJob * @param pJob
*/ */
void scheduleFreeJob(void *pJob); void schedulerFreeJob(void *pJob);
void schedulerDestroy(void); void schedulerDestroy(void);

2
include/libs/tfs/tfs.h
View File

@ -33,7 +33,7 @@ typedef struct {
typedef struct { typedef struct {
SDiskID did; SDiskID did;
char aname[TSDB_FILENAME_LEN]; // ABS name char aname[TSDB_FILENAME_LEN]; // TABS name
char rname[TSDB_FILENAME_LEN]; // REL name char rname[TSDB_FILENAME_LEN]; // REL name
STfs *pTfs; STfs *pTfs;
} STfsFile; } STfsFile;

16
include/os/osMath.h
View File

@ -20,37 +20,37 @@
extern "C" { extern "C" {
#endif #endif
#define POW2(x) ((x) * (x)) #define TPOW2(x) ((x) * (x))
#define ABS(x) ((x) > 0 ? (x) : -(x)) #define TABS(x) ((x) > 0 ? (x) : -(x))
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
#define SWAP(a, b, c) \ #define TSWAP(a, b, c) \
do { \ do { \
c __tmp = (c)(a); \ c __tmp = (c)(a); \
(a) = (c)(b); \ (a) = (c)(b); \
(b) = __tmp; \ (b) = __tmp; \
} while (0) } while (0)
#define MAX(a, b) (((a) > (b)) ? (a) : (b)) #define TMAX(a, b) (((a) > (b)) ? (a) : (b))
#define MIN(a, b) (((a) < (b)) ? (a) : (b)) #define TMIN(a, b) (((a) < (b)) ? (a) : (b))
#else #else
#define SWAP(a, b, c) \ #define TSWAP(a, b, c) \
do { \ do { \
typeof(a) __tmp = (a); \ typeof(a) __tmp = (a); \
(a) = (b); \ (a) = (b); \
(b) = __tmp; \ (b) = __tmp; \
} while (0) } while (0)
#define MAX(a, b) \ #define TMAX(a, b) \
({ \ ({ \
typeof(a) __a = (a); \ typeof(a) __a = (a); \
typeof(b) __b = (b); \ typeof(b) __b = (b); \
(__a > __b) ? __a : __b; \ (__a > __b) ? __a : __b; \
}) })
#define MIN(a, b) \ #define TMIN(a, b) \
({ \ ({ \
typeof(a) __a = (a); \ typeof(a) __a = (a); \
typeof(b) __b = (b); \ typeof(b) __b = (b); \

2
include/util/taoserror.h
View File

@ -354,7 +354,7 @@ int32_t* taosGetErrno();
#define TSDB_CODE_QRY_SCH_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0710) //"Scheduler not exist") #define TSDB_CODE_QRY_SCH_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0710) //"Scheduler not exist")
#define TSDB_CODE_QRY_TASK_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0711) //"Task not exist") #define TSDB_CODE_QRY_TASK_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0711) //"Task not exist")
#define TSDB_CODE_QRY_TASK_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0712) //"Task already exist") #define TSDB_CODE_QRY_TASK_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0712) //"Task already exist")
#define TSDB_CODE_QRY_RES_CACHE_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0713) //"Task result cache not exist") #define TSDB_CODE_QRY_TASK_CTX_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0713) //"Task context not exist")
#define TSDB_CODE_QRY_TASK_CANCELLED TAOS_DEF_ERROR_CODE(0, 0x0714) //"Task cancelled") #define TSDB_CODE_QRY_TASK_CANCELLED TAOS_DEF_ERROR_CODE(0, 0x0714) //"Task cancelled")
#define TSDB_CODE_QRY_TASK_DROPPED TAOS_DEF_ERROR_CODE(0, 0x0715) //"Task dropped") #define TSDB_CODE_QRY_TASK_DROPPED TAOS_DEF_ERROR_CODE(0, 0x0715) //"Task dropped")
#define TSDB_CODE_QRY_TASK_CANCELLING TAOS_DEF_ERROR_CODE(0, 0x0716) //"Task cancelling") #define TSDB_CODE_QRY_TASK_CANCELLING TAOS_DEF_ERROR_CODE(0, 0x0716) //"Task cancelling")

2
source/client/src/clientEnv.c
View File

@ -112,7 +112,7 @@ void* openTransporter(const char *user, const char *auth, int32_t numOfThread) {
rpcInit.user = (char *)user; rpcInit.user = (char *)user;
rpcInit.idleTime = tsShellActivityTimer * 1000; rpcInit.idleTime = tsShellActivityTimer * 1000;
rpcInit.ckey = "key"; rpcInit.ckey = "key";
// rpcInit.spi = 1; rpcInit.spi = 1;
rpcInit.secret = (char *)auth; rpcInit.secret = (char *)auth;
void* pDnodeConn = rpcOpen(&rpcInit); void* pDnodeConn = rpcOpen(&rpcInit);

246
source/client/src/clientImpl.c
View File

@ -119,7 +119,7 @@ TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass,
SAppInstInfo* p = calloc(1, sizeof(struct SAppInstInfo)); SAppInstInfo* p = calloc(1, sizeof(struct SAppInstInfo));
p->mgmtEp = epSet; p->mgmtEp = epSet;
p->pTransporter = openTransporter(user, secretEncrypt, tsNumOfCores); p->pTransporter = openTransporter(user, secretEncrypt, tsNumOfCores);
/*p->pAppHbMgr = appHbMgrInit(p);*/ p->pAppHbMgr = appHbMgrInit(p);
taosHashPut(appInfo.pInstMap, key, strlen(key), &p, POINTER_BYTES); taosHashPut(appInfo.pInstMap, key, strlen(key), &p, POINTER_BYTES);
pInst = &p; pInst = &p;
@ -219,6 +219,8 @@ int32_t getPlan(SRequestObj* pRequest, SQueryNode* pQueryNode, SQueryDag** pDag,
if (pQueryNode->type == TSDB_SQL_SELECT) { if (pQueryNode->type == TSDB_SQL_SELECT) {
setResSchemaInfo(&pRequest->body.resInfo, pSchema, numOfCols); setResSchemaInfo(&pRequest->body.resInfo, pSchema, numOfCols);
pRequest->type = TDMT_VND_QUERY; pRequest->type = TDMT_VND_QUERY;
} else {
tfree(pSchema);
} }
return code; return code;
@ -241,12 +243,14 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryDag* pDag, SArray* pNodeList)
if (TSDB_SQL_INSERT == pRequest->type || TSDB_SQL_CREATE_TABLE == pRequest->type) { if (TSDB_SQL_INSERT == pRequest->type || TSDB_SQL_CREATE_TABLE == pRequest->type) {
SQueryResult res = {.code = 0, .numOfRows = 0, .msgSize = ERROR_MSG_BUF_DEFAULT_SIZE, .msg = pRequest->msgBuf}; SQueryResult res = {.code = 0, .numOfRows = 0, .msgSize = ERROR_MSG_BUF_DEFAULT_SIZE, .msg = pRequest->msgBuf};
int32_t code = scheduleExecJob(pRequest->pTscObj->pAppInfo->pTransporter, NULL, pDag, &pRequest->body.pQueryJob, &res); taosArrayDestroy(pNodeList);
int32_t code = schedulerExecJob(pRequest->pTscObj->pAppInfo->pTransporter, NULL, pDag, &pRequest->body.pQueryJob, &res);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
// handle error and retry // handle error and retry
} else { } else {
if (pRequest->body.pQueryJob != NULL) { if (pRequest->body.pQueryJob != NULL) {
scheduleFreeJob(pRequest->body.pQueryJob); schedulerFreeJob(pRequest->body.pQueryJob);
} }
} }
@ -255,17 +259,11 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryDag* pDag, SArray* pNodeList)
return pRequest->code; return pRequest->code;
} }
return scheduleAsyncExecJob(pRequest->pTscObj->pAppInfo->pTransporter, pNodeList, pDag, &pRequest->body.pQueryJob); return schedulerAsyncExecJob(pRequest->pTscObj->pAppInfo->pTransporter, pNodeList, pDag, &pRequest->body.pQueryJob);
} }
typedef struct tmq_t tmq_t;
typedef struct SMqClientTopic { typedef struct SMqClientVg {
// subscribe info
int32_t sqlLen;
char* sql;
char* topicName;
int64_t topicId;
// statistics // statistics
int64_t consumeCnt; int64_t consumeCnt;
// offset // offset
@ -274,36 +272,160 @@ typedef struct SMqClientTopic {
//connection info //connection info
int32_t vgId; int32_t vgId;
SEpSet epSet; SEpSet epSet;
} SMqClientVg;
typedef struct SMqClientTopic {
// subscribe info
int32_t sqlLen;
char* sql;
char* topicName;
int64_t topicId;
int32_t nextVgIdx;
SArray* vgs; //SArray<SMqClientVg>
} SMqClientTopic; } SMqClientTopic;
typedef struct tmq_resp_err_t { typedef struct tmq_resp_err_t {
int32_t code; int32_t code;
} tmq_resp_err_t; } tmq_resp_err_t;
typedef struct tmq_topic_vgroup_list_t { typedef struct tmq_topic_vgroup_t {
char* topicName; char* topic;
int32_t vgId; int32_t vgId;
int64_t committedOffset; int64_t commitOffset;
} tmq_topic_vgroup_t;
typedef struct tmq_topic_vgroup_list_t {
int32_t cnt;
int32_t size;
tmq_topic_vgroup_t* elems;
} tmq_topic_vgroup_list_t; } tmq_topic_vgroup_list_t;
typedef void (tmq_commit_cb(tmq_t*, tmq_resp_err_t, tmq_topic_vgroup_list_t*, void* param)); typedef void (tmq_commit_cb(tmq_t*, tmq_resp_err_t, tmq_topic_vgroup_list_t*, void* param));
typedef struct tmq_conf_t{ struct tmq_conf_t {
char* clientId; char clientId[256];
char* groupId; char groupId[256];
char* ip; char* ip;
uint16_t port; uint16_t port;
tmq_commit_cb* commit_cb; tmq_commit_cb* commit_cb;
} tmq_conf_t; };
tmq_conf_t* tmq_conf_new() {
tmq_conf_t* conf = calloc(1, sizeof(tmq_conf_t));
return conf;
}
int32_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value) {
if (strcmp(key, "group.id")) {
strcpy(conf->groupId, value);
}
if (strcmp(key, "client.id")) {
strcpy(conf->clientId, value);
}
return 0;
}
struct tmq_t { struct tmq_t {
char groupId[256]; char groupId[256];
char clientId[256]; char clientId[256];
int64_t consumerId;
int64_t status;
STscObj* pTscObj; STscObj* pTscObj;
tmq_commit_cb* commit_cb; tmq_commit_cb* commit_cb;
int32_t nextTopicIdx;
SArray* clientTopics; //SArray<SMqClientTopic> SArray* clientTopics; //SArray<SMqClientTopic>
}; };
tmq_t* taos_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, int32_t errstrLen) {
tmq_t* pTmq = calloc(sizeof(tmq_t), 1);
if (pTmq == NULL) {
return NULL;
}
pTmq->pTscObj = (STscObj*)conn;
pTmq->status = 0;
strcpy(pTmq->clientId, conf->clientId);
strcpy(pTmq->groupId, conf->groupId);
pTmq->commit_cb = conf->commit_cb;
pTmq->consumerId = generateRequestId() & ((uint64_t)-1 >> 1);
return pTmq;
}
struct tmq_list_t {
int32_t cnt;
int32_t tot;
char* elems[];
};
tmq_list_t* tmq_list_new() {
tmq_list_t *ptr = malloc(sizeof(tmq_list_t) + 8 * sizeof(char*));
if (ptr == NULL) {
return ptr;
}
ptr->cnt = 0;
ptr->tot = 8;
return ptr;
}
int32_t tmq_list_append(tmq_list_t* ptr, char* src) {
if (ptr->cnt >= ptr->tot-1) return -1;
ptr->elems[ptr->cnt] = src;
ptr->cnt++;
return 0;
}
TAOS_RES* tmq_subscribe(tmq_t* tmq, tmq_list_t* topic_list) {
SRequestObj *pRequest = NULL;
tmq->status = 1;
int32_t sz = topic_list->cnt;
tmq->clientTopics = taosArrayInit(sz, sizeof(void*));
for (int i = 0; i < sz; i++) {
char* topicName = strdup(topic_list->elems[i]);
taosArrayPush(tmq->clientTopics, &topicName);
}
SCMSubscribeReq req;
req.topicNum = taosArrayGetSize(tmq->clientTopics);
req.consumerId = tmq->consumerId;
req.consumerGroup = strdup(tmq->groupId);
req.topicNames = tmq->clientTopics;
int tlen = tSerializeSCMSubscribeReq(NULL, &req);
void* buf = malloc(tlen);
if(buf == NULL) {
goto _return;
}
void* abuf = buf;
tSerializeSCMSubscribeReq(&abuf, &req);
/*printf("formatted: %s\n", dagStr);*/
pRequest = createRequest(tmq->pTscObj, NULL, NULL, TSDB_SQL_SELECT);
if (pRequest == NULL) {
tscError("failed to malloc sqlObj");
}
pRequest->body.requestMsg = (SDataBuf){ .pData = buf, .len = tlen };
pRequest->type = TDMT_MND_CREATE_TOPIC;
SMsgSendInfo* body = buildMsgInfoImpl(pRequest);
SEpSet epSet = getEpSet_s(&tmq->pTscObj->pAppInfo->mgmtEp);
int64_t transporterId = 0;
asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, body);
tsem_wait(&pRequest->body.rspSem);
_return:
if (body != NULL) {
destroySendMsgInfo(body);
}
if (pRequest != NULL && terrno != TSDB_CODE_SUCCESS) {
pRequest->code = terrno;
}
return pRequest;
}
void tmq_conf_set_offset_commit_cb(tmq_conf_t* conf, tmq_commit_cb* cb) { void tmq_conf_set_offset_commit_cb(tmq_conf_t* conf, tmq_commit_cb* cb) {
conf->commit_cb = cb; conf->commit_cb = cb;
} }
@ -331,10 +453,10 @@ SArray* tmqGetConnInfo(SClientHbKey connKey, void* param) {
int sz = taosArrayGetSize(clientTopics); int sz = taosArrayGetSize(clientTopics);
for (int i = 0; i < sz; i++) { for (int i = 0; i < sz; i++) {
SMqClientTopic* pCTopic = taosArrayGet(clientTopics, i); SMqClientTopic* pCTopic = taosArrayGet(clientTopics, i);
if (pCTopic->vgId == -1) { /*if (pCTopic->vgId == -1) {*/
pMqHb->status = 1; /*pMqHb->status = 1;*/
break; /*break;*/
} /*}*/
} }
kv.value = pMqHb; kv.value = pMqHb;
kv.valueLen = sizeof(SMqHbMsg); kv.valueLen = sizeof(SMqHbMsg);
@ -455,22 +577,63 @@ _return:
return pRequest; return pRequest;
} }
typedef struct tmq_message_t { /*typedef SMqConsumeRsp tmq_message_t;*/
int32_t numOfRows;
char* topicName;
TAOS_ROW row[];
} tmq_message_t;
tmq_message_t* tmq_consume_poll(tmq_t* mq, int64_t blocking_time) { struct tmq_message_t {
SMqConsumeRsp rsp;
};
tmq_message_t* tmq_consume_poll(tmq_t* tmq, int64_t blocking_time) {
if (tmq->clientTopics == NULL || taosArrayGetSize(tmq->clientTopics) == 0) {
return NULL;
}
SRequestObj *pRequest = NULL;
SMqConsumeReq req = {0};
req.reqType = 1;
req.blockingTime = blocking_time;
req.consumerId = tmq->consumerId;
strcpy(req.cgroup, tmq->groupId);
SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, tmq->nextTopicIdx);
tmq->nextTopicIdx = (tmq->nextTopicIdx + 1) % taosArrayGetSize(tmq->clientTopics);
strcpy(req.topic, pTopic->topicName);
int32_t nextVgIdx = pTopic->nextVgIdx;
pTopic->nextVgIdx = (nextVgIdx + 1) % taosArrayGetSize(pTopic->vgs);
SMqClientVg* pVg = taosArrayGet(pTopic->vgs, nextVgIdx);
req.offset = pVg->currentOffset;
pRequest->body.requestMsg = (SDataBuf){ .pData = &req, .len = sizeof(SMqConsumeReq) };
pRequest->type = TDMT_VND_CONSUME;
SMsgSendInfo* body = buildMsgInfoImpl(pRequest);
int64_t transporterId = 0;
asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, body);
tsem_wait(&pRequest->body.rspSem);
return (tmq_message_t*)pRequest->body.resInfo.pData;
/*tsem_wait(&pRequest->body.rspSem);*/
/*if (body != NULL) {*/
/*destroySendMsgInfo(body);*/
/*}*/
/*if (pRequest != NULL && terrno != TSDB_CODE_SUCCESS) {*/
/*pRequest->code = terrno;*/
/*}*/
/*return pRequest;*/
}
tmq_resp_err_t* tmq_commit(tmq_t* tmq, tmq_topic_vgroup_list_t* tmq_topic_vgroup_list, int32_t async) {
SMqConsumeReq req = {0};
return NULL; return NULL;
} }
tmq_resp_err_t* tmq_commit(tmq_t* mq, void* callback, int32_t async) { void tmq_message_destroy(tmq_message_t* tmq_message) {
return NULL; if (tmq_message == NULL) return;
}
void tmq_message_destroy(tmq_message_t* mq_message) {
} }
@ -486,7 +649,6 @@ TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen) {
SRequestObj *pRequest = NULL; SRequestObj *pRequest = NULL;
SQueryNode *pQueryNode = NULL; SQueryNode *pQueryNode = NULL;
SArray *pNodeList = taosArrayInit(4, sizeof(struct SQueryNodeAddr));
terrno = TSDB_CODE_SUCCESS; terrno = TSDB_CODE_SUCCESS;
CHECK_CODE_GOTO(buildRequest(pTscObj, sql, sqlLen, &pRequest), _return); CHECK_CODE_GOTO(buildRequest(pTscObj, sql, sqlLen, &pRequest), _return);
@ -495,6 +657,8 @@ TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen) {
if (qIsDdlQuery(pQueryNode)) { if (qIsDdlQuery(pQueryNode)) {
CHECK_CODE_GOTO(execDdlQuery(pRequest, pQueryNode), _return); CHECK_CODE_GOTO(execDdlQuery(pRequest, pQueryNode), _return);
} else { } else {
SArray *pNodeList = taosArrayInit(4, sizeof(struct SQueryNodeAddr));
CHECK_CODE_GOTO(getPlan(pRequest, pQueryNode, &pRequest->body.pDag, pNodeList), _return); CHECK_CODE_GOTO(getPlan(pRequest, pQueryNode, &pRequest->body.pDag, pNodeList), _return);
CHECK_CODE_GOTO(scheduleQuery(pRequest, pRequest->body.pDag, pNodeList), _return); CHECK_CODE_GOTO(scheduleQuery(pRequest, pRequest->body.pDag, pNodeList), _return);
pRequest->code = terrno; pRequest->code = terrno;
@ -567,7 +731,7 @@ STscObj* taosConnectImpl(const char *user, const char *auth, const char *db, __t
tsem_wait(&pRequest->body.rspSem); tsem_wait(&pRequest->body.rspSem);
if (pRequest->code != TSDB_CODE_SUCCESS) { if (pRequest->code != TSDB_CODE_SUCCESS) {
const char *errorMsg = (pRequest->code == TSDB_CODE_RPC_FQDN_ERROR) ? taos_errstr(pRequest) : tstrerror(terrno); const char *errorMsg = (pRequest->code == TSDB_CODE_RPC_FQDN_ERROR) ? taos_errstr(pRequest) : tstrerror(pRequest->code);
printf("failed to connect to server, reason: %s\n\n", errorMsg); printf("failed to connect to server, reason: %s\n\n", errorMsg);
destroyRequest(pRequest); destroyRequest(pRequest);
@ -695,10 +859,10 @@ TAOS *taos_connect_l(const char *ip, int ipLen, const char *user, int userLen, c
char userStr[TSDB_USER_LEN] = {0}; char userStr[TSDB_USER_LEN] = {0};
char passStr[TSDB_PASSWORD_LEN] = {0}; char passStr[TSDB_PASSWORD_LEN] = {0};
strncpy(ipStr, ip, MIN(TSDB_EP_LEN - 1, ipLen)); strncpy(ipStr, ip, TMIN(TSDB_EP_LEN - 1, ipLen));
strncpy(userStr, user, MIN(TSDB_USER_LEN - 1, userLen)); strncpy(userStr, user, TMIN(TSDB_USER_LEN - 1, userLen));
strncpy(passStr, pass, MIN(TSDB_PASSWORD_LEN - 1, passLen)); strncpy(passStr, pass, TMIN(TSDB_PASSWORD_LEN - 1, passLen));
strncpy(dbStr, db, MIN(TSDB_DB_NAME_LEN - 1, dbLen)); strncpy(dbStr, db, TMIN(TSDB_DB_NAME_LEN - 1, dbLen));
return taos_connect(ipStr, userStr, passStr, dbStr, port); return taos_connect(ipStr, userStr, passStr, dbStr, port);
} }
@ -716,7 +880,7 @@ void* doFetchRow(SRequestObj* pRequest) {
} }
SReqResultInfo* pResInfo = &pRequest->body.resInfo; SReqResultInfo* pResInfo = &pRequest->body.resInfo;
int32_t code = scheduleFetchRows(pRequest->body.pQueryJob, (void **)&pResInfo->pData); int32_t code = schedulerFetchRows(pRequest->body.pQueryJob, (void **)&pResInfo->pData);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
pRequest->code = code; pRequest->code = code;
return NULL; return NULL;

2
source/client/src/clientMsgHandler.c
View File

@ -72,7 +72,7 @@ int processConnectRsp(void* param, const SDataBuf* pMsg, int32_t code) {
atomic_add_fetch_64(&pTscObj->pAppInfo->numOfConns, 1); atomic_add_fetch_64(&pTscObj->pAppInfo->numOfConns, 1);
SClientHbKey connKey = {.connId = pConnect->connId, .hbType = HEARTBEAT_TYPE_QUERY}; SClientHbKey connKey = {.connId = pConnect->connId, .hbType = HEARTBEAT_TYPE_QUERY};
/*hbRegisterConn(pTscObj->pAppInfo->pAppHbMgr, connKey, NULL);*/ hbRegisterConn(pTscObj->pAppInfo->pAppHbMgr, connKey, NULL);
// pRequest->body.resInfo.pRspMsg = pMsg->pData; // pRequest->body.resInfo.pRspMsg = pMsg->pData;
tscDebug("0x%" PRIx64 " clusterId:%" PRId64 ", totalConn:%" PRId64, pRequest->requestId, pConnect->clusterId, tscDebug("0x%" PRIx64 " clusterId:%" PRId64 ", totalConn:%" PRId64, pRequest->requestId, pConnect->clusterId,

2
source/common/src/tglobal.c
View File

@ -44,7 +44,7 @@ int32_t tsRpcTimer = 300;
int32_t tsRpcMaxTime = 600; // seconds; int32_t tsRpcMaxTime = 600; // seconds;
int32_t tsRpcForceTcp = 1; // disable this, means query, show command use udp protocol as default int32_t tsRpcForceTcp = 1; // disable this, means query, show command use udp protocol as default
int32_t tsMaxShellConns = 50000; int32_t tsMaxShellConns = 50000;
int32_t tsMaxConnections = 5000; int32_t tsMaxConnections = 50000;
int32_t tsShellActivityTimer = 3; // second int32_t tsShellActivityTimer = 3; // second
float tsNumOfThreadsPerCore = 1.0f; float tsNumOfThreadsPerCore = 1.0f;
int32_t tsNumOfCommitThreads = 4; int32_t tsNumOfCommitThreads = 4;

14
source/common/src/ttypes.c
View File

@ -630,7 +630,7 @@ void operateVal(void *dst, void *s1, void *s2, int32_t optr, int32_t type) {
} }
} }
#define SWAP(a, b, c) \ #define TSWAP(a, b, c) \
do { \ do { \
typeof(a) __tmp = (a); \ typeof(a) __tmp = (a); \
(a) = (b); \ (a) = (b); \
@ -642,35 +642,35 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf
switch (type) { switch (type) {
case TSDB_DATA_TYPE_INT: case TSDB_DATA_TYPE_INT:
case TSDB_DATA_TYPE_UINT: { case TSDB_DATA_TYPE_UINT: {
SWAP(*(int32_t *)(pLeft), *(int32_t *)(pRight), int32_t); TSWAP(*(int32_t *)(pLeft), *(int32_t *)(pRight), int32_t);
break; break;
} }
case TSDB_DATA_TYPE_BIGINT: case TSDB_DATA_TYPE_BIGINT:
case TSDB_DATA_TYPE_UBIGINT: case TSDB_DATA_TYPE_UBIGINT:
case TSDB_DATA_TYPE_TIMESTAMP: { case TSDB_DATA_TYPE_TIMESTAMP: {
SWAP(*(int64_t *)(pLeft), *(int64_t *)(pRight), int64_t); TSWAP(*(int64_t *)(pLeft), *(int64_t *)(pRight), int64_t);
break; break;
} }
case TSDB_DATA_TYPE_DOUBLE: { case TSDB_DATA_TYPE_DOUBLE: {
SWAP(*(double *)(pLeft), *(double *)(pRight), double); TSWAP(*(double *)(pLeft), *(double *)(pRight), double);
break; break;
} }
case TSDB_DATA_TYPE_SMALLINT: case TSDB_DATA_TYPE_SMALLINT:
case TSDB_DATA_TYPE_USMALLINT: { case TSDB_DATA_TYPE_USMALLINT: {
SWAP(*(int16_t *)(pLeft), *(int16_t *)(pRight), int16_t); TSWAP(*(int16_t *)(pLeft), *(int16_t *)(pRight), int16_t);
break; break;
} }
case TSDB_DATA_TYPE_FLOAT: { case TSDB_DATA_TYPE_FLOAT: {
SWAP(*(float *)(pLeft), *(float *)(pRight), float); TSWAP(*(float *)(pLeft), *(float *)(pRight), float);
break; break;
} }
case TSDB_DATA_TYPE_BOOL: case TSDB_DATA_TYPE_BOOL:
case TSDB_DATA_TYPE_TINYINT: case TSDB_DATA_TYPE_TINYINT:
case TSDB_DATA_TYPE_UTINYINT: { case TSDB_DATA_TYPE_UTINYINT: {
SWAP(*(int8_t *)(pLeft), *(int8_t *)(pRight), int8_t); TSWAP(*(int8_t *)(pLeft), *(int8_t *)(pRight), int8_t);
break; break;
} }

21
source/dnode/mgmt/impl/src/dndTransport.c
View File

@ -116,6 +116,7 @@ static void dndInitMsgFp(STransMgmt *pMgmt) {
// Requests handled by VNODE // Requests handled by VNODE
pMgmt->msgFp[TMSG_INDEX(TDMT_VND_SUBMIT)] = dndProcessVnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_VND_SUBMIT)] = dndProcessVnodeWriteMsg;
pMgmt->msgFp[TMSG_INDEX(TDMT_VND_QUERY)] = dndProcessVnodeQueryMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_VND_QUERY)] = dndProcessVnodeQueryMsg;
pMgmt->msgFp[TMSG_INDEX(TDMT_VND_QUERY_CONTINUE)] = dndProcessVnodeQueryMsg;
pMgmt->msgFp[TMSG_INDEX(TDMT_VND_FETCH)] = dndProcessVnodeFetchMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_VND_FETCH)] = dndProcessVnodeFetchMsg;
pMgmt->msgFp[TMSG_INDEX(TDMT_VND_ALTER_TABLE)] = dndProcessVnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_VND_ALTER_TABLE)] = dndProcessVnodeWriteMsg;
pMgmt->msgFp[TMSG_INDEX(TDMT_VND_UPDATE_TAG_VAL)] = dndProcessVnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_VND_UPDATE_TAG_VAL)] = dndProcessVnodeWriteMsg;
@ -179,9 +180,13 @@ static int32_t dndInitClient(SDnode *pDnode) {
rpcInit.idleTime = pDnode->cfg.shellActivityTimer * 1000; rpcInit.idleTime = pDnode->cfg.shellActivityTimer * 1000;
rpcInit.user = INTERNAL_USER; rpcInit.user = INTERNAL_USER;
rpcInit.ckey = INTERNAL_CKEY; rpcInit.ckey = INTERNAL_CKEY;
rpcInit.secret = INTERNAL_SECRET; rpcInit.spi = 1;
rpcInit.parent = pDnode; rpcInit.parent = pDnode;
char pass[TSDB_PASSWORD_LEN + 1] = {0};
taosEncryptPass_c((uint8_t *)(INTERNAL_SECRET), strlen(INTERNAL_SECRET), pass);
rpcInit.secret = pass;
pMgmt->clientRpc = rpcOpen(&rpcInit); pMgmt->clientRpc = rpcOpen(&rpcInit);
if (pMgmt->clientRpc == NULL) { if (pMgmt->clientRpc == NULL) {
dError("failed to init rpc client"); dError("failed to init rpc client");
@ -256,20 +261,18 @@ static void dndSendMsgToMnodeRecv(SDnode *pDnode, SRpcMsg *pRpcMsg, SRpcMsg *pRp
static int32_t dndAuthInternalReq(SDnode *pDnode, char *user, char *spi, char *encrypt, char *secret, char *ckey) { static int32_t dndAuthInternalReq(SDnode *pDnode, char *user, char *spi, char *encrypt, char *secret, char *ckey) {
if (strcmp(user, INTERNAL_USER) == 0) { if (strcmp(user, INTERNAL_USER) == 0) {
// A simple temporary implementation char pass[TSDB_PASSWORD_LEN + 1] = {0};
char pass[TSDB_PASSWORD_LEN] = {0}; taosEncryptPass_c((uint8_t *)(INTERNAL_SECRET), strlen(INTERNAL_SECRET), pass);
taosEncryptPass((uint8_t *)(INTERNAL_SECRET), strlen(INTERNAL_SECRET), pass);
memcpy(secret, pass, TSDB_PASSWORD_LEN); memcpy(secret, pass, TSDB_PASSWORD_LEN);
*spi = 0; *spi = 1;
*encrypt = 0; *encrypt = 0;
*ckey = 0; *ckey = 0;
return 0; return 0;
} else if (strcmp(user, TSDB_NETTEST_USER) == 0) { } else if (strcmp(user, TSDB_NETTEST_USER) == 0) {
// A simple temporary implementation char pass[TSDB_PASSWORD_LEN + 1] = {0};
char pass[TSDB_PASSWORD_LEN] = {0}; taosEncryptPass_c((uint8_t *)(TSDB_NETTEST_USER), strlen(TSDB_NETTEST_USER), pass);
taosEncryptPass((uint8_t *)(TSDB_NETTEST_USER), strlen(TSDB_NETTEST_USER), pass);
memcpy(secret, pass, TSDB_PASSWORD_LEN); memcpy(secret, pass, TSDB_PASSWORD_LEN);
*spi = 0; *spi = 1;
*encrypt = 0; *encrypt = 0;
*ckey = 0; *ckey = 0;
return 0; return 0;

10
source/dnode/mgmt/impl/src/dndVnodes.c
View File

@ -892,7 +892,7 @@ int32_t dndPutReqToVQueryQ(SDnode *pDnode, SRpcMsg *pMsg) {
SVnodeObj *pVnode = dndAcquireVnode(pDnode, pHead->vgId); SVnodeObj *pVnode = dndAcquireVnode(pDnode, pHead->vgId);
if (pVnode == NULL) return -1; if (pVnode == NULL) return -1;
int32_t code = dndWriteRpcMsgToVnodeQueue(pVnode->pFetchQ, pMsg, false); int32_t code = dndWriteRpcMsgToVnodeQueue(pVnode->pQueryQ, pMsg, false);
dndReleaseVnode(pDnode, pVnode); dndReleaseVnode(pDnode, pVnode);
return code; return code;
} }
@ -910,11 +910,11 @@ static int32_t dndInitVnodeWorkers(SDnode *pDnode) {
SVnodesMgmt *pMgmt = &pDnode->vmgmt; SVnodesMgmt *pMgmt = &pDnode->vmgmt;
int32_t maxFetchThreads = 4; int32_t maxFetchThreads = 4;
int32_t minFetchThreads = MIN(maxFetchThreads, pDnode->env.numOfCores); int32_t minFetchThreads = TMIN(maxFetchThreads, pDnode->env.numOfCores);
int32_t minQueryThreads = MAX((int32_t)(pDnode->env.numOfCores * pDnode->cfg.ratioOfQueryCores), 1); int32_t minQueryThreads = TMAX((int32_t)(pDnode->env.numOfCores * pDnode->cfg.ratioOfQueryCores), 1);
int32_t maxQueryThreads = minQueryThreads; int32_t maxQueryThreads = minQueryThreads;
int32_t maxWriteThreads = MAX(pDnode->env.numOfCores, 1); int32_t maxWriteThreads = TMAX(pDnode->env.numOfCores, 1);
int32_t maxSyncThreads = MAX(pDnode->env.numOfCores / 2, 1); int32_t maxSyncThreads = TMAX(pDnode->env.numOfCores / 2, 1);
SWorkerPool *pPool = &pMgmt->queryPool; SWorkerPool *pPool = &pMgmt->queryPool;
pPool->name = "vnode-query"; pPool->name = "vnode-query";

6
source/dnode/mgmt/impl/test/sut/src/client.cpp
View File

@ -27,8 +27,8 @@ void TestClient::SetRpcRsp(SRpcMsg* pRsp) { this->pRsp = pRsp; };
tsem_t* TestClient::GetSem() { return &sem; } tsem_t* TestClient::GetSem() { return &sem; }
bool TestClient::Init(const char* user, const char* pass, const char* fqdn, uint16_t port) { bool TestClient::Init(const char* user, const char* pass, const char* fqdn, uint16_t port) {
char secretEncrypt[TSDB_PASSWORD_LEN] = {0}; char secretEncrypt[TSDB_PASSWORD_LEN + 1] = {0};
taosEncryptPass((uint8_t*)pass, strlen(pass), secretEncrypt); taosEncryptPass_c((uint8_t*)pass, strlen(pass), secretEncrypt);
SRpcInit rpcInit; SRpcInit rpcInit;
memset(&rpcInit, 0, sizeof(rpcInit)); memset(&rpcInit, 0, sizeof(rpcInit));
@ -42,7 +42,7 @@ bool TestClient::Init(const char* user, const char* pass, const char* fqdn, uint
rpcInit.ckey = (char*)"key"; rpcInit.ckey = (char*)"key";
rpcInit.parent = this; rpcInit.parent = this;
rpcInit.secret = (char*)secretEncrypt; rpcInit.secret = (char*)secretEncrypt;
// rpcInit.spi = 1; rpcInit.spi = 1;
clientRpc = rpcOpen(&rpcInit); clientRpc = rpcOpen(&rpcInit);
ASSERT(clientRpc); ASSERT(clientRpc);

76
source/dnode/mnode/impl/inc/mndDef.h
View File

@ -19,14 +19,14 @@
#include "os.h" #include "os.h"
#include "cJSON.h" #include "cJSON.h"
#include "scheduler.h"
#include "sync.h" #include "sync.h"
#include "tmsg.h"
#include "thash.h" #include "thash.h"
#include "tlist.h" #include "tlist.h"
#include "tlog.h" #include "tlog.h"
#include "tmsg.h"
#include "trpc.h" #include "trpc.h"
#include "ttimer.h" #include "ttimer.h"
#include "scheduler.h"
#include "mnode.h" #include "mnode.h"
@ -37,12 +37,42 @@ extern "C" {
extern int32_t mDebugFlag; extern int32_t mDebugFlag;
// mnode log function // mnode log function
#define mFatal(...) { if (mDebugFlag & DEBUG_FATAL) { taosPrintLog("MND FATAL ", 255, __VA_ARGS__); }} #define mFatal(...) \
#define mError(...) { if (mDebugFlag & DEBUG_ERROR) { taosPrintLog("MND ERROR ", 255, __VA_ARGS__); }} { \
#define mWarn(...) { if (mDebugFlag & DEBUG_WARN) { taosPrintLog("MND WARN ", 255, __VA_ARGS__); }} if (mDebugFlag & DEBUG_FATAL) { \
#define mInfo(...) { if (mDebugFlag & DEBUG_INFO) { taosPrintLog("MND ", 255, __VA_ARGS__); }} taosPrintLog("MND FATAL ", 255, __VA_ARGS__); \
#define mDebug(...) { if (mDebugFlag & DEBUG_DEBUG) { taosPrintLog("MND ", mDebugFlag, __VA_ARGS__); }} } \
#define mTrace(...) { if (mDebugFlag & DEBUG_TRACE) { taosPrintLog("MND ", mDebugFlag, __VA_ARGS__); }} }
#define mError(...) \
{ \
if (mDebugFlag & DEBUG_ERROR) { \
taosPrintLog("MND ERROR ", 255, __VA_ARGS__); \
} \
}
#define mWarn(...) \
{ \
if (mDebugFlag & DEBUG_WARN) { \
taosPrintLog("MND WARN ", 255, __VA_ARGS__); \
} \
}
#define mInfo(...) \
{ \
if (mDebugFlag & DEBUG_INFO) { \
taosPrintLog("MND ", 255, __VA_ARGS__); \
} \
}
#define mDebug(...) \
{ \
if (mDebugFlag & DEBUG_DEBUG) { \
taosPrintLog("MND ", mDebugFlag, __VA_ARGS__); \
} \
}
#define mTrace(...) \
{ \
if (mDebugFlag & DEBUG_TRACE) { \
taosPrintLog("MND ", mDebugFlag, __VA_ARGS__); \
} \
}
typedef enum { typedef enum {
MND_AUTH_ACCT_START = 0, MND_AUTH_ACCT_START = 0,
@ -328,6 +358,7 @@ typedef struct SMqTopicConsumer {
typedef struct SMqConsumerEp { typedef struct SMqConsumerEp {
int32_t vgId; // -1 for unassigned int32_t vgId; // -1 for unassigned
int32_t status;
SEpSet epSet; SEpSet epSet;
int64_t consumerId; // -1 for unassigned int64_t consumerId; // -1 for unassigned
int64_t lastConsumerHbTs; int64_t lastConsumerHbTs;
@ -339,6 +370,7 @@ typedef struct SMqConsumerEp {
static FORCE_INLINE int32_t tEncodeSMqConsumerEp(void** buf, SMqConsumerEp* pConsumerEp) { static FORCE_INLINE int32_t tEncodeSMqConsumerEp(void** buf, SMqConsumerEp* pConsumerEp) {
int32_t tlen = 0; int32_t tlen = 0;
tlen += taosEncodeFixedI32(buf, pConsumerEp->vgId); tlen += taosEncodeFixedI32(buf, pConsumerEp->vgId);
tlen += taosEncodeFixedI32(buf, pConsumerEp->status);
tlen += taosEncodeSEpSet(buf, &pConsumerEp->epSet); tlen += taosEncodeSEpSet(buf, &pConsumerEp->epSet);
tlen += taosEncodeFixedI64(buf, pConsumerEp->consumerId); tlen += taosEncodeFixedI64(buf, pConsumerEp->consumerId);
tlen += tEncodeSSubQueryMsg(buf, &pConsumerEp->qExec); tlen += tEncodeSSubQueryMsg(buf, &pConsumerEp->qExec);
@ -347,6 +379,7 @@ static FORCE_INLINE int32_t tEncodeSMqConsumerEp(void** buf, SMqConsumerEp* pCon
static FORCE_INLINE void* tDecodeSMqConsumerEp(void** buf, SMqConsumerEp* pConsumerEp) { static FORCE_INLINE void* tDecodeSMqConsumerEp(void** buf, SMqConsumerEp* pConsumerEp) {
buf = taosDecodeFixedI32(buf, &pConsumerEp->vgId); buf = taosDecodeFixedI32(buf, &pConsumerEp->vgId);
buf = taosDecodeFixedI32(buf, &pConsumerEp->status);
buf = taosDecodeSEpSet(buf, &pConsumerEp->epSet); buf = taosDecodeSEpSet(buf, &pConsumerEp->epSet);
buf = taosDecodeFixedI64(buf, &pConsumerEp->consumerId); buf = taosDecodeFixedI64(buf, &pConsumerEp->consumerId);
buf = tDecodeSSubQueryMsg(buf, &pConsumerEp->qExec); buf = tDecodeSSubQueryMsg(buf, &pConsumerEp->qExec);
@ -362,7 +395,8 @@ typedef struct SMqSubscribeObj {
int32_t nextConsumerIdx; int32_t nextConsumerIdx;
SArray* availConsumer; // SArray<int64_t> (consumerId) SArray* availConsumer; // SArray<int64_t> (consumerId)
SArray* assigned; // SArray<SMqConsumerEp> SArray* assigned; // SArray<SMqConsumerEp>
SArray* unassignedConsumer; // SArray<SMqConsumerEp> SArray* idleConsumer; // SArray<SMqConsumerEp>
SArray* lostConsumer; // SArray<SMqConsumerEp>
SArray* unassignedVg; // SArray<SMqConsumerEp> SArray* unassignedVg; // SArray<SMqConsumerEp>
} SMqSubscribeObj; } SMqSubscribeObj;
@ -384,17 +418,17 @@ static FORCE_INLINE SMqSubscribeObj* tNewSubscribeObj() {
free(pSub); free(pSub);
return NULL; return NULL;
} }
pSub->unassignedConsumer = taosArrayInit(0, sizeof(SMqConsumerEp)); pSub->idleConsumer = taosArrayInit(0, sizeof(SMqConsumerEp));
if (pSub->assigned == NULL) { if (pSub->assigned == NULL) {
taosArrayDestroy(pSub->availConsumer); taosArrayDestroy(pSub->availConsumer);
taosArrayDestroy(pSub->unassignedConsumer); taosArrayDestroy(pSub->idleConsumer);
free(pSub); free(pSub);
return NULL; return NULL;
} }
pSub->unassignedVg = taosArrayInit(0, sizeof(SMqConsumerEp)); pSub->unassignedVg = taosArrayInit(0, sizeof(SMqConsumerEp));
if (pSub->assigned == NULL) { if (pSub->assigned == NULL) {
taosArrayDestroy(pSub->availConsumer); taosArrayDestroy(pSub->availConsumer);
taosArrayDestroy(pSub->unassignedConsumer); taosArrayDestroy(pSub->idleConsumer);
taosArrayDestroy(pSub->unassignedVg); taosArrayDestroy(pSub->unassignedVg);
free(pSub); free(pSub);
return NULL; return NULL;
@ -422,10 +456,10 @@ static FORCE_INLINE int32_t tEncodeSubscribeObj(void** buf, const SMqSubscribeOb
tlen += tEncodeSMqConsumerEp(buf, pCEp); tlen += tEncodeSMqConsumerEp(buf, pCEp);
} }
sz = taosArrayGetSize(pSub->unassignedConsumer); sz = taosArrayGetSize(pSub->idleConsumer);
tlen += taosEncodeFixedI32(buf, sz); tlen += taosEncodeFixedI32(buf, sz);
for (int32_t i = 0; i < sz; i++) { for (int32_t i = 0; i < sz; i++) {
SMqConsumerEp* pCEp = taosArrayGet(pSub->unassignedConsumer, i); SMqConsumerEp* pCEp = taosArrayGet(pSub->idleConsumer, i);
tlen += tEncodeSMqConsumerEp(buf, pCEp); tlen += tEncodeSMqConsumerEp(buf, pCEp);
} }
@ -457,22 +491,22 @@ static FORCE_INLINE void* tDecodeSubscribeObj(void* buf, SMqSubscribeObj* pSub)
} }
buf = taosDecodeFixedI32(buf, &sz); buf = taosDecodeFixedI32(buf, &sz);
pSub->unassignedConsumer = taosArrayInit(sz, sizeof(SMqConsumerEp)); pSub->idleConsumer = taosArrayInit(sz, sizeof(SMqConsumerEp));
if (pSub->unassignedConsumer == NULL) { if (pSub->idleConsumer == NULL) {
taosArrayDestroy(pSub->assigned); taosArrayDestroy(pSub->assigned);
return NULL; return NULL;
} }
for (int32_t i = 0; i < sz; i++) { for (int32_t i = 0; i < sz; i++) {
SMqConsumerEp cEp; SMqConsumerEp cEp;
buf = tDecodeSMqConsumerEp(buf, &cEp); buf = tDecodeSMqConsumerEp(buf, &cEp);
taosArrayPush(pSub->unassignedConsumer, &cEp); taosArrayPush(pSub->idleConsumer, &cEp);
} }
buf = taosDecodeFixedI32(buf, &sz); buf = taosDecodeFixedI32(buf, &sz);
pSub->unassignedVg = taosArrayInit(sz, sizeof(SMqConsumerEp)); pSub->unassignedVg = taosArrayInit(sz, sizeof(SMqConsumerEp));
if (pSub->unassignedVg == NULL) { if (pSub->unassignedVg == NULL) {
taosArrayDestroy(pSub->assigned); taosArrayDestroy(pSub->assigned);
taosArrayDestroy(pSub->unassignedConsumer); taosArrayDestroy(pSub->idleConsumer);
return NULL; return NULL;
} }
for (int32_t i = 0; i < sz; i++) { for (int32_t i = 0; i < sz; i++) {
@ -512,13 +546,12 @@ typedef struct SMqTopicObj {
typedef struct SMqConsumerTopic { typedef struct SMqConsumerTopic {
char name[TSDB_TOPIC_FNAME_LEN]; char name[TSDB_TOPIC_FNAME_LEN];
int32_t epoch; int32_t epoch;
//TODO: replace with something with ep
//SList *vgroups; // SList<int32_t>
// vg assigned to the consumer on the topic // vg assigned to the consumer on the topic
SArray* pVgInfo; // SArray<int32_t> SArray* pVgInfo; // SArray<int32_t>
} SMqConsumerTopic; } SMqConsumerTopic;
static FORCE_INLINE SMqConsumerTopic* tNewConsumerTopic(int64_t consumerId, SMqTopicObj* pTopic, SMqSubscribeObj* pSub) { static FORCE_INLINE SMqConsumerTopic* tNewConsumerTopic(int64_t consumerId, SMqTopicObj* pTopic,
SMqSubscribeObj* pSub) {
SMqConsumerTopic* pCTopic = malloc(sizeof(SMqConsumerTopic)); SMqConsumerTopic* pCTopic = malloc(sizeof(SMqConsumerTopic));
if (pCTopic == NULL) { if (pCTopic == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
@ -567,6 +600,7 @@ static FORCE_INLINE void* tDecodeSMqConsumerTopic(void* buf, SMqConsumerTopic* p
typedef struct SMqConsumerObj { typedef struct SMqConsumerObj {
int64_t consumerId; int64_t consumerId;
int64_t connId;
SRWLatch lock; SRWLatch lock;
char cgroup[TSDB_CONSUMER_GROUP_LEN]; char cgroup[TSDB_CONSUMER_GROUP_LEN];
SArray* topics; // SArray<SMqConsumerTopic> SArray* topics; // SArray<SMqConsumerTopic>

20
source/dnode/mnode/impl/src/mndAuth.c
View File

@ -15,6 +15,7 @@
#define _DEFAULT_SOURCE #define _DEFAULT_SOURCE
#include "mndAuth.h" #include "mndAuth.h"
#include "mndUser.h"
static int32_t mndProcessAuthReq(SMnodeMsg *pReq); static int32_t mndProcessAuthReq(SMnodeMsg *pReq);
@ -25,7 +26,24 @@ int32_t mndInitAuth(SMnode *pMnode) {
void mndCleanupAuth(SMnode *pMnode) {} void mndCleanupAuth(SMnode *pMnode) {}
int32_t mndRetriveAuth(SMnode *pMnode, char *user, char *spi, char *encrypt, char *secret, char *ckey) { return 0; } int32_t mndRetriveAuth(SMnode *pMnode, char *user, char *spi, char *encrypt, char *secret, char *ckey) {
SUserObj *pUser = mndAcquireUser(pMnode, user);
if (pUser == NULL) {
*secret = 0;
mError("user:%s, failed to auth user since %s", user, terrstr());
return -1;
}
*spi = 1;
*encrypt = 0;
*ckey = 0;
memcpy(secret, pUser->pass, TSDB_PASSWORD_LEN);
mndReleaseUser(pMnode, pUser);
mDebug("user:%s, auth info is returned", user);
return 0;
}
static int32_t mndProcessAuthReq(SMnodeMsg *pReq) { static int32_t mndProcessAuthReq(SMnodeMsg *pReq) {
SAuthReq *pAuth = pReq->rpcMsg.pCont; SAuthReq *pAuth = pReq->rpcMsg.pCont;

2
source/dnode/mnode/impl/src/mndDnode.c
View File

@ -234,7 +234,7 @@ int32_t mndGetDnodeSize(SMnode *pMnode) {
} }
bool mndIsDnodeOnline(SMnode *pMnode, SDnodeObj *pDnode, int64_t curMs) { bool mndIsDnodeOnline(SMnode *pMnode, SDnodeObj *pDnode, int64_t curMs) {
int64_t interval = ABS(pDnode->lastAccessTime - curMs); int64_t interval = TABS(pDnode->lastAccessTime - curMs);
if (interval > 3500 * pMnode->cfg.statusInterval) { if (interval > 3500 * pMnode->cfg.statusInterval) {
if (pDnode->rebootTime > 0) { if (pDnode->rebootTime > 0) {
pDnode->offlineReason = DND_REASON_STATUS_MSG_TIMEOUT; pDnode->offlineReason = DND_REASON_STATUS_MSG_TIMEOUT;

2
source/dnode/mnode/impl/src/mndProfile.c
View File

@ -261,7 +261,7 @@ static int32_t mndSaveQueryStreamList(SConnObj *pConn, SHeartBeatReq *pReq) {
pConn->pQueries = calloc(sizeof(SQueryDesc), QUERY_SAVE_SIZE); pConn->pQueries = calloc(sizeof(SQueryDesc), QUERY_SAVE_SIZE);
} }
pConn->numOfQueries = MIN(QUERY_SAVE_SIZE, numOfQueries); pConn->numOfQueries = TMIN(QUERY_SAVE_SIZE, numOfQueries);
int32_t saveSize = pConn->numOfQueries * sizeof(SQueryDesc); int32_t saveSize = pConn->numOfQueries * sizeof(SQueryDesc);
if (saveSize > 0 && pConn->pQueries != NULL) { if (saveSize > 0 && pConn->pQueries != NULL) {

18
source/dnode/mnode/impl/src/mndSubscribe.c
View File

@ -98,7 +98,8 @@ static int32_t mndProcessMqTimerMsg(SMnodeMsg *pMsg) {
// build msg // build msg
SMqSetCVgReq req = { SMqSetCVgReq req = {
.vgId = pCEp->vgId, .vgId = pCEp->vgId,
.consumerId = consumerId, .oldConsumerId = -1,
.newConsumerId = consumerId,
}; };
strcpy(req.cgroup, cgroup); strcpy(req.cgroup, cgroup);
strcpy(req.topicName, topic); strcpy(req.topicName, topic);
@ -152,6 +153,7 @@ static int mndInitUnassignedVg(SMnode *pMnode, SMqTopicObj *pTopic, SArray *unas
//convert dag to msg //convert dag to msg
for (int32_t i = 0; i < sz; i++) { for (int32_t i = 0; i < sz; i++) {
SMqConsumerEp CEp; SMqConsumerEp CEp;
CEp.status = 0;
CEp.lastConsumerHbTs = CEp.lastVgHbTs = -1; CEp.lastConsumerHbTs = CEp.lastVgHbTs = -1;
STaskInfo* pTaskInfo = taosArrayGet(pArray, i); STaskInfo* pTaskInfo = taosArrayGet(pArray, i);
tConvertQueryAddrToEpSet(&CEp.epSet, &pTaskInfo->addr); tConvertQueryAddrToEpSet(&CEp.epSet, &pTaskInfo->addr);
@ -171,7 +173,8 @@ static int mndBuildMqSetConsumerVgReq(SMnode *pMnode, STrans *pTrans, SMqConsume
SVgObj *pVgObj = mndAcquireVgroup(pMnode, vgId); SVgObj *pVgObj = mndAcquireVgroup(pMnode, vgId);
SMqSetCVgReq req = { SMqSetCVgReq req = {
.vgId = vgId, .vgId = vgId,
.consumerId = pConsumer->consumerId, .oldConsumerId = -1,
.newConsumerId = pConsumer->consumerId,
}; };
strcpy(req.cgroup, pConsumer->cgroup); strcpy(req.cgroup, pConsumer->cgroup);
strcpy(req.topicName, pTopic->name); strcpy(req.topicName, pTopic->name);
@ -451,12 +454,13 @@ static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) {
SMqTopicObj *pTopic = mndAcquireTopic(pMnode, newTopicName); SMqTopicObj *pTopic = mndAcquireTopic(pMnode, newTopicName);
if (pTopic == NULL) { if (pTopic == NULL) {
/*terrno = */ mError("topic being subscribed not exist: %s", newTopicName);
continue; continue;
} }
SMqSubscribeObj *pSub = mndAcquireSubscribe(pMnode, consumerGroup, newTopicName); SMqSubscribeObj *pSub = mndAcquireSubscribe(pMnode, consumerGroup, newTopicName);
if (pSub == NULL) { if (pSub == NULL) {
mDebug("create new subscription, group: %s, topic %s", consumerGroup, newTopicName);
pSub = tNewSubscribeObj(); pSub = tNewSubscribeObj();
if (pSub == NULL) { if (pSub == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
@ -464,14 +468,15 @@ static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) {
} }
// set unassigned vg // set unassigned vg
mndInitUnassignedVg(pMnode, pTopic, pSub->unassignedVg); mndInitUnassignedVg(pMnode, pTopic, pSub->unassignedVg);
//TODO: disable alter
} }
taosArrayPush(pSub->availConsumer, &consumerId); taosArrayPush(pSub->availConsumer, &consumerId);
// TODO: no need
SMqConsumerTopic *pConsumerTopic = tNewConsumerTopic(consumerId, pTopic, pSub); SMqConsumerTopic *pConsumerTopic = tNewConsumerTopic(consumerId, pTopic, pSub);
taosArrayPush(pConsumer->topics, pConsumerTopic); taosArrayPush(pConsumer->topics, pConsumerTopic);
if (taosArrayGetSize(pConsumerTopic->pVgInfo) > 0) { if (taosArrayGetSize(pConsumerTopic->pVgInfo) > 0) {
ASSERT(taosArrayGetSize(pConsumerTopic->pVgInfo) == 1);
int32_t vgId = *(int32_t *)taosArrayGetLast(pConsumerTopic->pVgInfo); int32_t vgId = *(int32_t *)taosArrayGetLast(pConsumerTopic->pVgInfo);
// send setmsg to vnode // send setmsg to vnode
if (mndBuildMqSetConsumerVgReq(pMnode, pTrans, pConsumer, pConsumerTopic, pTopic) < 0) { if (mndBuildMqSetConsumerVgReq(pMnode, pTrans, pConsumer, pConsumerTopic, pTopic) < 0) {
@ -479,8 +484,7 @@ static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) {
return -1; return -1;
} }
} }
taosArrayDestroy(pConsumerTopic->pVgInfo);
free(pConsumerTopic);
SSdbRaw *pRaw = mndSubActionEncode(pSub); SSdbRaw *pRaw = mndSubActionEncode(pSub);
/*sdbSetRawStatus(pRaw, SDB_STATUS_READY);*/ /*sdbSetRawStatus(pRaw, SDB_STATUS_READY);*/
mndTransAppendRedolog(pTrans, pRaw); mndTransAppendRedolog(pTrans, pRaw);
@ -533,12 +537,12 @@ static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) {
if (mndTransPrepare(pMnode, pTrans) != 0) { if (mndTransPrepare(pMnode, pTrans) != 0) {
mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
if (newSub) taosArrayDestroy(newSub);
mndTransDrop(pTrans); mndTransDrop(pTrans);
mndReleaseConsumer(pMnode, pConsumer); mndReleaseConsumer(pMnode, pConsumer);
return -1; return -1;
} }
// TODO: free memory
if (newSub) taosArrayDestroy(newSub); if (newSub) taosArrayDestroy(newSub);
mndTransDrop(pTrans); mndTransDrop(pTrans);
mndReleaseConsumer(pMnode, pConsumer); mndReleaseConsumer(pMnode, pConsumer);

2
source/dnode/mnode/impl/src/mndSync.c
View File

@ -56,7 +56,7 @@ static int32_t mndRestoreWal(SMnode *pMnode) {
int64_t last = walGetLastVer(pWal); int64_t last = walGetLastVer(pWal);
mDebug("start to restore sdb wal, sdb ver:%" PRId64 ", wal first:%" PRId64 " last:%" PRId64, lastSdbVer, first, last); mDebug("start to restore sdb wal, sdb ver:%" PRId64 ", wal first:%" PRId64 " last:%" PRId64, lastSdbVer, first, last);
first = MAX(lastSdbVer + 1, first); first = TMAX(lastSdbVer + 1, first);
for (int64_t ver = first; ver >= 0 && ver <= last; ++ver) { for (int64_t ver = first; ver >= 0 && ver <= last; ++ver) {
if (walReadWithHandle(pHandle, ver) < 0) { if (walReadWithHandle(pHandle, ver) < 0) {
mError("failed to read by wal handle since %s, ver:%" PRId64, terrstr(), ver); mError("failed to read by wal handle since %s, ver:%" PRId64, terrstr(), ver);

11
source/dnode/mnode/impl/src/mndUser.c
View File

@ -60,9 +60,9 @@ void mndCleanupUser(SMnode *pMnode) {}
static int32_t mndCreateDefaultUser(SMnode *pMnode, char *acct, char *user, char *pass) { static int32_t mndCreateDefaultUser(SMnode *pMnode, char *acct, char *user, char *pass) {
SUserObj userObj = {0}; SUserObj userObj = {0};
taosEncryptPass_c((uint8_t *)pass, strlen(pass), userObj.pass);
tstrncpy(userObj.user, user, TSDB_USER_LEN); tstrncpy(userObj.user, user, TSDB_USER_LEN);
tstrncpy(userObj.acct, acct, TSDB_USER_LEN); tstrncpy(userObj.acct, acct, TSDB_USER_LEN);
taosEncryptPass((uint8_t *)pass, strlen(pass), userObj.pass);
userObj.createdTime = taosGetTimestampMs(); userObj.createdTime = taosGetTimestampMs();
userObj.updateTime = userObj.createdTime; userObj.updateTime = userObj.createdTime;
@ -202,7 +202,7 @@ SUserObj *mndAcquireUser(SMnode *pMnode, char *userName) {
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
SUserObj *pUser = sdbAcquire(pSdb, SDB_USER, userName); SUserObj *pUser = sdbAcquire(pSdb, SDB_USER, userName);
if (pUser == NULL) { if (pUser == NULL) {
terrno = TSDB_CODE_MND_DB_NOT_EXIST; terrno = TSDB_CODE_MND_USER_NOT_EXIST;
} }
return pUser; return pUser;
} }
@ -214,9 +214,9 @@ void mndReleaseUser(SMnode *pMnode, SUserObj *pUser) {
static int32_t mndCreateUser(SMnode *pMnode, char *acct, char *user, char *pass, SMnodeMsg *pReq) { static int32_t mndCreateUser(SMnode *pMnode, char *acct, char *user, char *pass, SMnodeMsg *pReq) {
SUserObj userObj = {0}; SUserObj userObj = {0};
taosEncryptPass_c((uint8_t *)pass, strlen(pass), userObj.pass);
tstrncpy(userObj.user, user, TSDB_USER_LEN); tstrncpy(userObj.user, user, TSDB_USER_LEN);
tstrncpy(userObj.acct, acct, TSDB_USER_LEN); tstrncpy(userObj.acct, acct, TSDB_USER_LEN);
taosEncryptPass((uint8_t *)pass, strlen(pass), userObj.pass);
userObj.createdTime = taosGetTimestampMs(); userObj.createdTime = taosGetTimestampMs();
userObj.updateTime = userObj.createdTime; userObj.updateTime = userObj.createdTime;
userObj.superUser = 0; userObj.superUser = 0;
@ -351,8 +351,9 @@ static int32_t mndProcessAlterUserReq(SMnodeMsg *pReq) {
SUserObj newUser = {0}; SUserObj newUser = {0};
memcpy(&newUser, pUser, sizeof(SUserObj)); memcpy(&newUser, pUser, sizeof(SUserObj));
memset(pUser->pass, 0, sizeof(pUser->pass)); char pass[TSDB_PASSWORD_LEN + 1] = {0};
taosEncryptPass((uint8_t *)pAlter->pass, strlen(pAlter->pass), pUser->pass); taosEncryptPass_c((uint8_t *)pAlter->pass, strlen(pAlter->pass), pass);
memcpy(pUser->pass, pass, TSDB_PASSWORD_LEN);
newUser.updateTime = taosGetTimestampMs(); newUser.updateTime = taosGetTimestampMs();
int32_t code = mndUpdateUser(pMnode, pUser, &newUser, pReq); int32_t code = mndUpdateUser(pMnode, pUser, &newUser, pReq);

2
source/dnode/mnode/impl/src/mndVgroup.c
View File

@ -459,7 +459,7 @@ static bool mndGetVgroupMaxReplicaFp(SMnode *pMnode, void *pObj, void *p1, void
int32_t *pNumOfVgroups = p3; int32_t *pNumOfVgroups = p3;
if (pVgroup->dbUid == uid) { if (pVgroup->dbUid == uid) {
*pReplica = MAX(*pReplica, pVgroup->replica); *pReplica = TMAX(*pReplica, pVgroup->replica);
(*pNumOfVgroups)++; (*pNumOfVgroups)++;
} }

9
source/dnode/mnode/sdb/src/sdbHash.c
View File

@ -152,6 +152,7 @@ static int32_t sdbInsertRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow *
if (insertFp != NULL) { if (insertFp != NULL) {
code = (*insertFp)(pSdb, pRow->pObj); code = (*insertFp)(pSdb, pRow->pObj);
if (code != 0) { if (code != 0) {
code = terrno;
taosWLockLatch(pLock); taosWLockLatch(pLock);
taosHashRemove(hash, pRow->pObj, keySize); taosHashRemove(hash, pRow->pObj, keySize);
taosWUnLockLatch(pLock); taosWUnLockLatch(pLock);
@ -162,10 +163,10 @@ static int32_t sdbInsertRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow *
} }
if (pSdb->keyTypes[pRow->type] == SDB_KEY_INT32) { if (pSdb->keyTypes[pRow->type] == SDB_KEY_INT32) {
pSdb->maxId[pRow->type] = MAX(pSdb->maxId[pRow->type], *((int32_t *)pRow->pObj)); pSdb->maxId[pRow->type] = TMAX(pSdb->maxId[pRow->type], *((int32_t *)pRow->pObj));
} }
if (pSdb->keyTypes[pRow->type] == SDB_KEY_INT64) { if (pSdb->keyTypes[pRow->type] == SDB_KEY_INT64) {
pSdb->maxId[pRow->type] = MAX(pSdb->maxId[pRow->type], *((int32_t *)pRow->pObj)); pSdb->maxId[pRow->type] = TMAX(pSdb->maxId[pRow->type], *((int32_t *)pRow->pObj));
} }
pSdb->tableVer[pRow->type]++; pSdb->tableVer[pRow->type]++;
@ -420,13 +421,13 @@ int32_t sdbGetMaxId(SSdb *pSdb, ESdbType type) {
while (ppRow != NULL) { while (ppRow != NULL) {
SSdbRow *pRow = *ppRow; SSdbRow *pRow = *ppRow;
int32_t id = *(int32_t *)pRow->pObj; int32_t id = *(int32_t *)pRow->pObj;
maxId = MAX(id, maxId); maxId = TMAX(id, maxId);
ppRow = taosHashIterate(hash, ppRow); ppRow = taosHashIterate(hash, ppRow);
} }
taosRUnLockLatch(pLock); taosRUnLockLatch(pLock);
maxId = MAX(maxId, pSdb->maxId[type]); maxId = TMAX(maxId, pSdb->maxId[type]);
return maxId + 1; return maxId + 1;
} }

7
source/dnode/vnode/inc/tq.h
View File

@ -18,7 +18,6 @@
#include "common.h" #include "common.h"
#include "executor.h" #include "executor.h"
#include "vnode.h"
#include "mallocator.h" #include "mallocator.h"
#include "meta.h" #include "meta.h"
#include "os.h" #include "os.h"
@ -29,6 +28,7 @@
#include "trpc.h" #include "trpc.h"
#include "ttimer.h" #include "ttimer.h"
#include "tutil.h" #include "tutil.h"
#include "vnode.h"
#include "wal.h" #include "wal.h"
#ifdef __cplusplus #ifdef __cplusplus
@ -153,6 +153,7 @@ typedef struct STqTaskItem {
int64_t offset; int64_t offset;
void* dst; void* dst;
qTaskInfo_t task; qTaskInfo_t task;
SSubQueryMsg* pQueryMsg;
} STqTaskItem; } STqTaskItem;
// new version // new version
@ -164,7 +165,6 @@ typedef struct STqBuffer {
typedef struct STqTopicHandle { typedef struct STqTopicHandle {
char topicName[TSDB_TOPIC_FNAME_LEN]; char topicName[TSDB_TOPIC_FNAME_LEN];
char cgroup[TSDB_TOPIC_FNAME_LEN];
char* sql; char* sql;
char* logicalPlan; char* logicalPlan;
char* physicalPlan; char* physicalPlan;
@ -177,6 +177,7 @@ typedef struct STqTopicHandle {
typedef struct STqConsumerHandle { typedef struct STqConsumerHandle {
int64_t consumerId; int64_t consumerId;
int64_t epoch; int64_t epoch;
char cgroup[TSDB_TOPIC_FNAME_LEN];
SArray* topics; // SArray<STqClientTopic> SArray* topics; // SArray<STqClientTopic>
} STqConsumerHandle; } STqConsumerHandle;
@ -318,7 +319,7 @@ int tqSendLaunchQuery(STqMsgItem*, int64_t offset);
#endif #endif
int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg** ppRsp); int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg** ppRsp);
int32_t tqProcessSetConnReq(STQ* pTq, SMqSetCVgReq* pReq); int32_t tqProcessSetConnReq(STQ* pTq, char* msg);
#ifdef __cplusplus #ifdef __cplusplus
} }

6
source/dnode/vnode/src/inc/tqInt.h
View File

@ -44,8 +44,10 @@ extern int32_t tqDebugFlag;
// delete persistent storage for meta info // delete persistent storage for meta info
// int tqDropTCGroup(STQ*, const char* topic, int cgId); // int tqDropTCGroup(STQ*, const char* topic, int cgId);
int tqSerializeGroup(const STqGroup*, STqSerializedHead**); //int tqSerializeGroup(const STqGroup*, STqSerializedHead**);
const void* tqDeserializeGroup(const STqSerializedHead* pHead, STqGroup** ppGroup); //const void* tqDeserializeGroup(const STqSerializedHead* pHead, STqGroup** ppGroup);
int tqSerializeConsumer(const STqConsumerHandle*, STqSerializedHead**);
const void* tqDeserializeConsumer(const STqSerializedHead* pHead, STqConsumerHandle**);
static int FORCE_INLINE tqQueryExecuting(int32_t status) { return status; } static int FORCE_INLINE tqQueryExecuting(int32_t status) { return status; }
#ifdef __cplusplus #ifdef __cplusplus
} }

105
source/dnode/vnode/src/tq/tq.c
View File

@ -68,7 +68,7 @@ STQ* tqOpen(const char* path, SWal* pWal, SMeta* pMeta, STqCfg* tqConfig, SMemAl
// TODO: error code of buffer pool // TODO: error code of buffer pool
} }
#endif #endif
pTq->tqMeta = tqStoreOpen(path, (FTqSerialize)tqSerializeGroup, (FTqDeserialize)tqDeserializeGroup, free, 0); pTq->tqMeta = tqStoreOpen(path, (FTqSerialize)tqSerializeConsumer, (FTqDeserialize)tqDeserializeConsumer, free, 0);
if (pTq->tqMeta == NULL) { if (pTq->tqMeta == NULL) {
free(pTq); free(pTq);
#if 0 #if 0
@ -478,6 +478,59 @@ int tqConsume(STQ* pTq, STqConsumeReq* pMsg) {
} }
#endif #endif
int tqSerializeConsumer(const STqConsumerHandle* pConsumer, STqSerializedHead** ppHead) {
int32_t num = taosArrayGetSize(pConsumer->topics);
int32_t sz = sizeof(STqSerializedHead) + sizeof(int64_t) * 2 + TSDB_TOPIC_FNAME_LEN + num * (sizeof(int64_t) + TSDB_TOPIC_FNAME_LEN);
if (sz > (*ppHead)->ssize) {
void* tmpPtr = realloc(*ppHead, sz);
if (tmpPtr == NULL) {
free(*ppHead);
return -1;
}
*ppHead = tmpPtr;
(*ppHead)->ssize = sz;
}
void* ptr = (*ppHead)->content;
*(int64_t*)ptr = pConsumer->consumerId;
ptr = POINTER_SHIFT(ptr, sizeof(int64_t));
*(int64_t*)ptr = pConsumer->epoch;
ptr = POINTER_SHIFT(ptr, sizeof(int64_t));
memcpy(ptr, pConsumer->topics, TSDB_TOPIC_FNAME_LEN);
ptr = POINTER_SHIFT(ptr, TSDB_TOPIC_FNAME_LEN);
*(int32_t*)ptr = num;
ptr = POINTER_SHIFT(ptr, sizeof(int32_t));
for (int32_t i = 0; i < num; i++) {
STqTopicHandle* pTopic = taosArrayGet(pConsumer->topics, i);
memcpy(ptr, pTopic->topicName, TSDB_TOPIC_FNAME_LEN);
ptr = POINTER_SHIFT(ptr, TSDB_TOPIC_FNAME_LEN);
*(int64_t*)ptr = pTopic->committedOffset;
POINTER_SHIFT(ptr, sizeof(int64_t));
}
return 0;
}
const void* tqDeserializeConsumer(const STqSerializedHead* pHead, STqConsumerHandle** ppConsumer) {
STqConsumerHandle* pConsumer = *ppConsumer;
const void* ptr = pHead->content;
pConsumer->consumerId = *(int64_t*)ptr;
ptr = POINTER_SHIFT(ptr, sizeof(int64_t));
pConsumer->epoch = *(int64_t*)ptr;
ptr = POINTER_SHIFT(ptr, sizeof(int64_t));
memcpy(pConsumer->cgroup, ptr, TSDB_TOPIC_FNAME_LEN);
ptr = POINTER_SHIFT(ptr, TSDB_TOPIC_FNAME_LEN);
int32_t sz = *(int32_t*)ptr;
ptr = POINTER_SHIFT(ptr, sizeof(int32_t));
pConsumer->topics = taosArrayInit(sz, sizeof(STqTopicHandle));
for (int32_t i = 0; i < sz; i++) {
/*STqTopicHandle* topicHandle = */
/*taosArrayPush(pConsumer->topics, );*/
}
return NULL;
}
#if 0
int tqSerializeGroup(const STqGroup* pGroup, STqSerializedHead** ppHead) { int tqSerializeGroup(const STqGroup* pGroup, STqSerializedHead** ppHead) {
// calculate size // calculate size
int sz = tqGroupSSize(pGroup) + sizeof(STqSerializedHead); int sz = tqGroupSSize(pGroup) + sizeof(STqSerializedHead);
@ -608,6 +661,7 @@ int tqItemSSize() {
// mainly for executor // mainly for executor
return 0; return 0;
} }
#endif
int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg** ppRsp) { int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg** ppRsp) {
SMqConsumeReq* pReq = pMsg->pCont; SMqConsumeReq* pReq = pMsg->pCont;
@ -625,7 +679,14 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg** ppRsp) {
for (int i = 0; i < sz; i++) { for (int i = 0; i < sz; i++) {
STqTopicHandle* pTopic = taosArrayGet(pConsumer->topics, i); STqTopicHandle* pTopic = taosArrayGet(pConsumer->topics, i);
//TODO: support multiple topic in one req
if (strcmp(pTopic->topicName, pReq->topic) != 0) {
continue;
}
if (fetchOffset == -1) {
fetchOffset = pTopic->committedOffset + 1;
}
int8_t pos; int8_t pos;
int8_t skip = 0; int8_t skip = 0;
SWalHead* pHead; SWalHead* pHead;
@ -670,6 +731,23 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg** ppRsp) {
break; break;
} }
if (pDataBlock != NULL) { if (pDataBlock != NULL) {
SMqTbData tbData = {
.uid = pDataBlock->info.uid,
.numOfCols = pDataBlock->info.numOfCols,
.numOfRows = pDataBlock->info.rows,
};
for (int i = 0; i < pDataBlock->info.numOfCols; i++) {
SColumnInfoData* pColData = taosArrayGet(pDataBlock->pDataBlock, i);
int32_t sz = pColData->info.bytes * pDataBlock->info.rows;
SMqColData colData = {
.bytes = pColData->info.bytes,
.colId = pColData->info.colId,
.type = pColData->info.type,
};
memcpy(colData.data, pColData->pData, colData.bytes * pDataBlock->info.rows);
memcpy(&tbData.colData[i], &colData, sz);
}
/*pDataBlock->info.*/
taosArrayPush(pRes, pDataBlock); taosArrayPush(pRes, pDataBlock);
} else { } else {
break; break;
@ -692,29 +770,34 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg** ppRsp) {
pTopic->buffer.lastOffset = pReq->offset; pTopic->buffer.lastOffset = pReq->offset;
} }
// put output into rsp // put output into rsp
SMqConsumeRsp rsp = {
.consumerId = consumerId,
.numOfTopics = 1
};
} }
// launch query
// get result
return 0; return 0;
} }
int32_t tqProcessSetConnReq(STQ* pTq, SMqSetCVgReq* pReq) { int32_t tqProcessSetConnReq(STQ* pTq, char* msg) {
SMqSetCVgReq req;
tDecodeSMqSetCVgReq(msg, &req);
STqConsumerHandle* pConsumer = calloc(sizeof(STqConsumerHandle), 1); STqConsumerHandle* pConsumer = calloc(sizeof(STqConsumerHandle), 1);
if (pConsumer == NULL) { if (pConsumer == NULL) {
return -1; return -1;
} }
strcpy(pConsumer->cgroup, req.cgroup);
pConsumer->topics = taosArrayInit(0, sizeof(STqTopicHandle));
STqTopicHandle* pTopic = calloc(sizeof(STqTopicHandle), 1); STqTopicHandle* pTopic = calloc(sizeof(STqTopicHandle), 1);
if (pTopic == NULL) { if (pTopic == NULL) {
free(pConsumer); free(pConsumer);
return -1; return -1;
} }
strcpy(pTopic->topicName, pReq->topicName); strcpy(pTopic->topicName, req.topicName);
strcpy(pTopic->cgroup, pReq->cgroup); strcpy(pTopic->sql, req.sql);
strcpy(pTopic->sql, pReq->sql); strcpy(pTopic->logicalPlan, req.logicalPlan);
strcpy(pTopic->logicalPlan, pReq->logicalPlan); strcpy(pTopic->physicalPlan, req.physicalPlan);
strcpy(pTopic->physicalPlan, pReq->physicalPlan);
pTopic->buffer.firstOffset = -1; pTopic->buffer.firstOffset = -1;
pTopic->buffer.lastOffset = -1; pTopic->buffer.lastOffset = -1;
@ -724,9 +807,9 @@ int32_t tqProcessSetConnReq(STQ* pTq, SMqSetCVgReq* pReq) {
for (int i = 0; i < TQ_BUFFER_SIZE; i++) { for (int i = 0; i < TQ_BUFFER_SIZE; i++) {
pTopic->buffer.output[i].status = 0; pTopic->buffer.output[i].status = 0;
STqReadHandle* pReadHandle = tqInitSubmitMsgScanner(pTq->pMeta); STqReadHandle* pReadHandle = tqInitSubmitMsgScanner(pTq->pMeta);
pTopic->buffer.output[i].task = qCreateStreamExecTaskInfo(&pReq->msg, pReadHandle); pTopic->buffer.output[i].task = qCreateStreamExecTaskInfo(&req.msg, pReadHandle);
} }
// write mq meta taosArrayPush(pConsumer->topics, pTopic);
return 0; return 0;
} }

2
source/dnode/vnode/src/tsdb/tsdbCompact.c
View File

@ -459,7 +459,7 @@ static int tsdbCompactMeta(STsdbRepo *pRepo) {
while (true) { while (true) {
if (pReadh->pDCols[0]->numOfRows - ridx == 0) break; if (pReadh->pDCols[0]->numOfRows - ridx == 0) break;
int rowsToMerge = MIN(pReadh->pDCols[0]->numOfRows - ridx, defaultRows - pComph->pDataCols->numOfRows); int rowsToMerge = TMIN(pReadh->pDCols[0]->numOfRows - ridx, defaultRows - pComph->pDataCols->numOfRows);
tdMergeDataCols(pComph->pDataCols, pReadh->pDCols[0], rowsToMerge, &ridx, pCfg->update != TD_ROW_PARTIAL_UPDATE); tdMergeDataCols(pComph->pDataCols, pReadh->pDCols[0], rowsToMerge, &ridx, pCfg->update != TD_ROW_PARTIAL_UPDATE);

2
source/dnode/vnode/src/tsdb/tsdbFS.c
View File

@ -836,7 +836,7 @@ static int tsdbScanAndTryFixFS(STsdb *pRepo) {
// return -1; // return -1;
// } // }
// maxBufSize = MAX(maxBufSize, rInfo.size); // maxBufSize = TMAX(maxBufSize, rInfo.size);
// if (tsdbSeekMFile(pMFile, rInfo.size, SEEK_CUR) < 0) { // if (tsdbSeekMFile(pMFile, rInfo.size, SEEK_CUR) < 0) {
// tsdbError("vgId:%d failed to lseek file %s since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile), // tsdbError("vgId:%d failed to lseek file %s since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile),

32
source/dnode/vnode/src/tsdb/tsdbMemTable.c
View File

@ -153,8 +153,8 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey
if (fKey == INT64_MAX && rowKey == INT64_MAX) break; if (fKey == INT64_MAX && rowKey == INT64_MAX) break;
if (fKey < rowKey) { if (fKey < rowKey) {
pMergeInfo->keyFirst = MIN(pMergeInfo->keyFirst, fKey); pMergeInfo->keyFirst = TMIN(pMergeInfo->keyFirst, fKey);
pMergeInfo->keyLast = MAX(pMergeInfo->keyLast, fKey); pMergeInfo->keyLast = TMAX(pMergeInfo->keyLast, fKey);
filterIter++; filterIter++;
if (filterIter >= nFilterKeys) { if (filterIter >= nFilterKeys) {
@ -170,8 +170,8 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey
if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break; if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break;
pMergeInfo->rowsInserted++; pMergeInfo->rowsInserted++;
pMergeInfo->nOperations++; pMergeInfo->nOperations++;
pMergeInfo->keyFirst = MIN(pMergeInfo->keyFirst, rowKey); pMergeInfo->keyFirst = TMIN(pMergeInfo->keyFirst, rowKey);
pMergeInfo->keyLast = MAX(pMergeInfo->keyLast, rowKey); pMergeInfo->keyLast = TMAX(pMergeInfo->keyLast, rowKey);
tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row); tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row);
} }
@ -196,12 +196,12 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey
if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break; if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break;
pMergeInfo->rowsUpdated++; pMergeInfo->rowsUpdated++;
pMergeInfo->nOperations++; pMergeInfo->nOperations++;
pMergeInfo->keyFirst = MIN(pMergeInfo->keyFirst, rowKey); pMergeInfo->keyFirst = TMIN(pMergeInfo->keyFirst, rowKey);
pMergeInfo->keyLast = MAX(pMergeInfo->keyLast, rowKey); pMergeInfo->keyLast = TMAX(pMergeInfo->keyLast, rowKey);
tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row); tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row);
} else { } else {
pMergeInfo->keyFirst = MIN(pMergeInfo->keyFirst, fKey); pMergeInfo->keyFirst = TMIN(pMergeInfo->keyFirst, fKey);
pMergeInfo->keyLast = MAX(pMergeInfo->keyLast, fKey); pMergeInfo->keyLast = TMAX(pMergeInfo->keyLast, fKey);
} }
} }
@ -714,8 +714,8 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey
if (fKey == INT64_MAX && rowKey == INT64_MAX) break; if (fKey == INT64_MAX && rowKey == INT64_MAX) break;
if (fKey < rowKey) { if (fKey < rowKey) {
pMergeInfo->keyFirst = MIN(pMergeInfo->keyFirst, fKey); pMergeInfo->keyFirst = TMIN(pMergeInfo->keyFirst, fKey);
pMergeInfo->keyLast = MAX(pMergeInfo->keyLast, fKey); pMergeInfo->keyLast = TMAX(pMergeInfo->keyLast, fKey);
filterIter++; filterIter++;
if (filterIter >= nFilterKeys) { if (filterIter >= nFilterKeys) {
@ -731,8 +731,8 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey
if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break; if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break;
pMergeInfo->rowsInserted++; pMergeInfo->rowsInserted++;
pMergeInfo->nOperations++; pMergeInfo->nOperations++;
pMergeInfo->keyFirst = MIN(pMergeInfo->keyFirst, rowKey); pMergeInfo->keyFirst = TMIN(pMergeInfo->keyFirst, rowKey);
pMergeInfo->keyLast = MAX(pMergeInfo->keyLast, rowKey); pMergeInfo->keyLast = TMAX(pMergeInfo->keyLast, rowKey);
tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row); tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row);
} }
@ -757,12 +757,12 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey
if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break; if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break;
pMergeInfo->rowsUpdated++; pMergeInfo->rowsUpdated++;
pMergeInfo->nOperations++; pMergeInfo->nOperations++;
pMergeInfo->keyFirst = MIN(pMergeInfo->keyFirst, rowKey); pMergeInfo->keyFirst = TMIN(pMergeInfo->keyFirst, rowKey);
pMergeInfo->keyLast = MAX(pMergeInfo->keyLast, rowKey); pMergeInfo->keyLast = TMAX(pMergeInfo->keyLast, rowKey);
tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row); tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row);
} else { } else {
pMergeInfo->keyFirst = MIN(pMergeInfo->keyFirst, fKey); pMergeInfo->keyFirst = TMIN(pMergeInfo->keyFirst, fKey);
pMergeInfo->keyLast = MAX(pMergeInfo->keyLast, fKey); pMergeInfo->keyLast = TMAX(pMergeInfo->keyLast, fKey);
} }
} }

14
source/dnode/vnode/src/tsdb/tsdbRead.c
View File

@ -504,7 +504,7 @@ void tsdbResetQueryHandle(tsdbReadHandleT queryHandle, STsdbQueryCond *pCond) {
if (emptyQueryTimewindow(pTsdbReadHandle)) { if (emptyQueryTimewindow(pTsdbReadHandle)) {
if (pCond->order != pTsdbReadHandle->order) { if (pCond->order != pTsdbReadHandle->order) {
pTsdbReadHandle->order = pCond->order; pTsdbReadHandle->order = pCond->order;
SWAP(pTsdbReadHandle->window.skey, pTsdbReadHandle->window.ekey, int64_t); TSWAP(pTsdbReadHandle->window.skey, pTsdbReadHandle->window.ekey, int64_t);
} }
return; return;
@ -971,7 +971,7 @@ static bool hasMoreDataInCache(STsdbReadHandle* pHandle) {
pHandle->cur.mixBlock = true; pHandle->cur.mixBlock = true;
if (!ASCENDING_TRAVERSE(pHandle->order)) { if (!ASCENDING_TRAVERSE(pHandle->order)) {
SWAP(win->skey, win->ekey, TSKEY); TSWAP(win->skey, win->ekey, TSKEY);
} }
return true; return true;
@ -1074,8 +1074,8 @@ static int32_t loadBlockInfo(STsdbReadHandle * pTsdbReadHandle, int32_t index, i
assert(pCheckInfo->lastKey >= pTsdbReadHandle->window.ekey && pTsdbReadHandle->window.skey >= pTsdbReadHandle->window.ekey); assert(pCheckInfo->lastKey >= pTsdbReadHandle->window.ekey && pTsdbReadHandle->window.skey >= pTsdbReadHandle->window.ekey);
} }
s = MIN(pCheckInfo->lastKey, pTsdbReadHandle->window.ekey); s = TMIN(pCheckInfo->lastKey, pTsdbReadHandle->window.ekey);
e = MAX(pCheckInfo->lastKey, pTsdbReadHandle->window.ekey); e = TMAX(pCheckInfo->lastKey, pTsdbReadHandle->window.ekey);
// discard the unqualified data block based on the query time window // discard the unqualified data block based on the query time window
int32_t start = binarySearchForBlock(pCompInfo->blocks, compIndex->numOfBlocks, s, TSDB_ORDER_ASC); int32_t start = binarySearchForBlock(pCompInfo->blocks, compIndex->numOfBlocks, s, TSDB_ORDER_ASC);
@ -1240,7 +1240,7 @@ static int32_t handleDataMergeIfNeeded(STsdbReadHandle* pTsdbReadHandle, SBlock*
// update the last key value // update the last key value
pCheckInfo->lastKey = cur->win.ekey + step; pCheckInfo->lastKey = cur->win.ekey + step;
if (!ASCENDING_TRAVERSE(pTsdbReadHandle->order)) { if (!ASCENDING_TRAVERSE(pTsdbReadHandle->order)) {
SWAP(cur->win.skey, cur->win.ekey, TSKEY); TSWAP(cur->win.skey, cur->win.ekey, TSKEY);
} }
cur->mixBlock = true; cur->mixBlock = true;
@ -1795,7 +1795,7 @@ static void copyAllRemainRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, STa
int32_t end = endPos; int32_t end = endPos;
if (!ASCENDING_TRAVERSE(pTsdbReadHandle->order)) { if (!ASCENDING_TRAVERSE(pTsdbReadHandle->order)) {
SWAP(start, end, int32_t); TSWAP(start, end, int32_t);
} }
assert(pTsdbReadHandle->outputCapacity >= (end - start + 1)); assert(pTsdbReadHandle->outputCapacity >= (end - start + 1));
@ -2020,7 +2020,7 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf
((pos < endPos || cur->lastKey < pTsdbReadHandle->window.ekey) && !ASCENDING_TRAVERSE(pTsdbReadHandle->order))); ((pos < endPos || cur->lastKey < pTsdbReadHandle->window.ekey) && !ASCENDING_TRAVERSE(pTsdbReadHandle->order)));
if (!ASCENDING_TRAVERSE(pTsdbReadHandle->order)) { if (!ASCENDING_TRAVERSE(pTsdbReadHandle->order)) {
SWAP(cur->win.skey, cur->win.ekey, TSKEY); TSWAP(cur->win.skey, cur->win.ekey, TSKEY);
} }
moveDataToFront(pTsdbReadHandle, numOfRows, numOfCols); moveDataToFront(pTsdbReadHandle, numOfRows, numOfCols);

2
source/dnode/vnode/src/vnd/vnodeArenaMAImpl.c
View File

@ -68,7 +68,7 @@ void *vmaMalloc(SVMemAllocator *pVMA, uint64_t size) {
void * ptr; void * ptr;
if (pNode->size < POINTER_DISTANCE(pNode->ptr, pNode->data) + size) { if (pNode->size < POINTER_DISTANCE(pNode->ptr, pNode->data) + size) {
uint64_t capacity = MAX(pVMA->ssize, size); uint64_t capacity = TMAX(pVMA->ssize, size);
pNode = vArenaNodeNew(capacity); pNode = vArenaNodeNew(capacity);
if (pNode == NULL) { if (pNode == NULL) {
// TODO: handle error // TODO: handle error

4
source/dnode/vnode/src/vnd/vnodeQuery.c
View File

@ -25,7 +25,7 @@ int vnodeQueryOpen(SVnode *pVnode) {
} }
int vnodeProcessQueryReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { int vnodeProcessQueryReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
vTrace("query message is processing"); vTrace("message in query queue is processing");
switch (pMsg->msgType) { switch (pMsg->msgType) {
case TDMT_VND_QUERY: case TDMT_VND_QUERY:
@ -39,7 +39,7 @@ int vnodeProcessQueryReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
} }
int vnodeProcessFetchReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { int vnodeProcessFetchReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
vTrace("fetch message is processed"); vTrace("message in fetch queue is processing");
switch (pMsg->msgType) { switch (pMsg->msgType) {
case TDMT_VND_FETCH: case TDMT_VND_FETCH:
return qWorkerProcessFetchMsg(pVnode, pVnode->pQuery, pMsg); return qWorkerProcessFetchMsg(pVnode, pVnode->pQuery, pMsg);

7
source/dnode/vnode/src/vnd/vnodeWrite.c
View File

@ -96,7 +96,7 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
} }
} }
vDebug("vgId:%d process create %"PRIzu" tables", pVnode->vgId, taosArrayGetSize(vCreateTbBatchReq.pArray)); vTrace("vgId:%d process create %" PRIzu " tables", pVnode->vgId, taosArrayGetSize(vCreateTbBatchReq.pArray));
taosArrayDestroy(vCreateTbBatchReq.pArray); taosArrayDestroy(vCreateTbBatchReq.pArray);
break; break;
@ -112,9 +112,7 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
} }
break; break;
case TDMT_VND_MQ_SET_CONN: { case TDMT_VND_MQ_SET_CONN: {
SMqSetCVgReq req; if (tqProcessSetConnReq(pVnode->pTq, ptr) < 0) {
tDecodeSMqSetCVgReq(ptr, &req);
if (tqProcessSetConnReq(pVnode->pTq, &req) < 0) {
} }
} break; } break;
default: default:
@ -131,6 +129,7 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
// TODO: handle error // TODO: handle error
} }
} }
return 0; return 0;
} }

36
source/libs/executor/src/executorimpl.c
View File

@ -304,7 +304,7 @@ SSDataBlock* createOutputBuf(SExprInfo* pExpr, int32_t numOfOutput, int32_t numO
idata.info.bytes = pExpr[i].base.resSchema.bytes; idata.info.bytes = pExpr[i].base.resSchema.bytes;
idata.info.colId = pExpr[i].base.resSchema.colId; idata.info.colId = pExpr[i].base.resSchema.colId;
int32_t size = MAX(idata.info.bytes * numOfRows, minSize); int32_t size = TMAX(idata.info.bytes * numOfRows, minSize);
idata.pData = calloc(1, size); // at least to hold a pointer on x64 platform idata.pData = calloc(1, size); // at least to hold a pointer on x64 platform
taosArrayPush(res->pDataBlock, &idata); taosArrayPush(res->pDataBlock, &idata);
} }
@ -329,7 +329,7 @@ SSDataBlock* createOutputBuf_rv(SArray* pExprInfo, int32_t numOfRows) {
idata.info.bytes = pExpr->base.resSchema.bytes; idata.info.bytes = pExpr->base.resSchema.bytes;
idata.info.colId = pExpr->base.resSchema.colId; idata.info.colId = pExpr->base.resSchema.colId;
int32_t size = MAX(idata.info.bytes * numOfRows, minSize); int32_t size = TMAX(idata.info.bytes * numOfRows, minSize);
idata.pData = calloc(1, size); // at least to hold a pointer on x64 platform idata.pData = calloc(1, size); // at least to hold a pointer on x64 platform
taosArrayPush(res->pDataBlock, &idata); taosArrayPush(res->pDataBlock, &idata);
} }
@ -2679,7 +2679,7 @@ static void updateDataCheckOrder(SQInfo *pQInfo, SQueryTableReq* pQueryMsg, bool
pQueryAttr->order.order = TSDB_ORDER_ASC; pQueryAttr->order.order = TSDB_ORDER_ASC;
if (pQueryAttr->window.skey > pQueryAttr->window.ekey) { if (pQueryAttr->window.skey > pQueryAttr->window.ekey) {
SWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); TSWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY);
} }
pQueryAttr->needReverseScan = false; pQueryAttr->needReverseScan = false;
@ -2689,7 +2689,7 @@ static void updateDataCheckOrder(SQInfo *pQInfo, SQueryTableReq* pQueryMsg, bool
if (pQueryAttr->groupbyColumn && pQueryAttr->order.order == TSDB_ORDER_DESC) { if (pQueryAttr->groupbyColumn && pQueryAttr->order.order == TSDB_ORDER_DESC) {
pQueryAttr->order.order = TSDB_ORDER_ASC; pQueryAttr->order.order = TSDB_ORDER_ASC;
if (pQueryAttr->window.skey > pQueryAttr->window.ekey) { if (pQueryAttr->window.skey > pQueryAttr->window.ekey) {
SWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); TSWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY);
} }
pQueryAttr->needReverseScan = false; pQueryAttr->needReverseScan = false;
@ -2700,7 +2700,7 @@ static void updateDataCheckOrder(SQInfo *pQInfo, SQueryTableReq* pQueryMsg, bool
if (pQueryAttr->pointInterpQuery && pQueryAttr->interval.interval == 0) { if (pQueryAttr->pointInterpQuery && pQueryAttr->interval.interval == 0) {
if (!QUERY_IS_ASC_QUERY(pQueryAttr)) { if (!QUERY_IS_ASC_QUERY(pQueryAttr)) {
//qDebug(msg, pQInfo->qId, "interp", pQueryAttr->order.order, TSDB_ORDER_ASC, pQueryAttr->window.skey, pQueryAttr->window.ekey, pQueryAttr->window.ekey, pQueryAttr->window.skey); //qDebug(msg, pQInfo->qId, "interp", pQueryAttr->order.order, TSDB_ORDER_ASC, pQueryAttr->window.skey, pQueryAttr->window.ekey, pQueryAttr->window.ekey, pQueryAttr->window.skey);
SWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); TSWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY);
} }
pQueryAttr->order.order = TSDB_ORDER_ASC; pQueryAttr->order.order = TSDB_ORDER_ASC;
@ -2713,7 +2713,7 @@ static void updateDataCheckOrder(SQInfo *pQInfo, SQueryTableReq* pQueryMsg, bool
//qDebug(msg, pQInfo->qId, "only-first", pQueryAttr->order.order, TSDB_ORDER_ASC, pQueryAttr->window.skey, //qDebug(msg, pQInfo->qId, "only-first", pQueryAttr->order.order, TSDB_ORDER_ASC, pQueryAttr->window.skey,
// pQueryAttr->window.ekey, pQueryAttr->window.ekey, pQueryAttr->window.skey); // pQueryAttr->window.ekey, pQueryAttr->window.ekey, pQueryAttr->window.skey);
SWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); TSWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY);
doUpdateLastKey(pQueryAttr); doUpdateLastKey(pQueryAttr);
} }
@ -2724,7 +2724,7 @@ static void updateDataCheckOrder(SQInfo *pQInfo, SQueryTableReq* pQueryMsg, bool
//qDebug(msg, pQInfo->qId, "only-last", pQueryAttr->order.order, TSDB_ORDER_DESC, pQueryAttr->window.skey, //qDebug(msg, pQInfo->qId, "only-last", pQueryAttr->order.order, TSDB_ORDER_DESC, pQueryAttr->window.skey,
// pQueryAttr->window.ekey, pQueryAttr->window.ekey, pQueryAttr->window.skey); // pQueryAttr->window.ekey, pQueryAttr->window.ekey, pQueryAttr->window.skey);
SWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); TSWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY);
doUpdateLastKey(pQueryAttr); doUpdateLastKey(pQueryAttr);
} }
@ -2739,7 +2739,7 @@ static void updateDataCheckOrder(SQInfo *pQInfo, SQueryTableReq* pQueryMsg, bool
//qDebug(msg, pQInfo->qId, "only-first stable", pQueryAttr->order.order, TSDB_ORDER_ASC, //qDebug(msg, pQInfo->qId, "only-first stable", pQueryAttr->order.order, TSDB_ORDER_ASC,
// pQueryAttr->window.skey, pQueryAttr->window.ekey, pQueryAttr->window.ekey, pQueryAttr->window.skey); // pQueryAttr->window.skey, pQueryAttr->window.ekey, pQueryAttr->window.ekey, pQueryAttr->window.skey);
SWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); TSWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY);
doUpdateLastKey(pQueryAttr); doUpdateLastKey(pQueryAttr);
} }
@ -2750,7 +2750,7 @@ static void updateDataCheckOrder(SQInfo *pQInfo, SQueryTableReq* pQueryMsg, bool
//qDebug(msg, pQInfo->qId, "only-last stable", pQueryAttr->order.order, TSDB_ORDER_DESC, //qDebug(msg, pQInfo->qId, "only-last stable", pQueryAttr->order.order, TSDB_ORDER_DESC,
// pQueryAttr->window.skey, pQueryAttr->window.ekey, pQueryAttr->window.ekey, pQueryAttr->window.skey); // pQueryAttr->window.skey, pQueryAttr->window.ekey, pQueryAttr->window.ekey, pQueryAttr->window.skey);
SWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); TSWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY);
doUpdateLastKey(pQueryAttr); doUpdateLastKey(pQueryAttr);
} }
@ -2790,8 +2790,8 @@ static void getIntermediateBufInfo(STaskRuntimeEnv* pRuntimeEnv, int32_t* ps, in
static bool overlapWithTimeWindow(STaskAttr* pQueryAttr, SDataBlockInfo* pBlockInfo) { static bool overlapWithTimeWindow(STaskAttr* pQueryAttr, SDataBlockInfo* pBlockInfo) {
STimeWindow w = {0}; STimeWindow w = {0};
TSKEY sk = MIN(pQueryAttr->window.skey, pQueryAttr->window.ekey); TSKEY sk = TMIN(pQueryAttr->window.skey, pQueryAttr->window.ekey);
TSKEY ek = MAX(pQueryAttr->window.skey, pQueryAttr->window.ekey); TSKEY ek = TMAX(pQueryAttr->window.skey, pQueryAttr->window.ekey);
if (QUERY_IS_ASC_QUERY(pQueryAttr)) { if (QUERY_IS_ASC_QUERY(pQueryAttr)) {
getAlignQueryTimeWindow(pQueryAttr, pBlockInfo->window.skey, sk, ek, &w); getAlignQueryTimeWindow(pQueryAttr, pBlockInfo->window.skey, sk, ek, &w);
@ -3474,7 +3474,7 @@ static void updateTableQueryInfoForReverseScan(STableQueryInfo *pTableQueryInfo)
return; return;
} }
SWAP(pTableQueryInfo->win.skey, pTableQueryInfo->win.ekey, TSKEY); TSWAP(pTableQueryInfo->win.skey, pTableQueryInfo->win.ekey, TSKEY);
pTableQueryInfo->lastKey = pTableQueryInfo->win.skey; pTableQueryInfo->lastKey = pTableQueryInfo->win.skey;
SWITCH_ORDER(pTableQueryInfo->cur.order); SWITCH_ORDER(pTableQueryInfo->cur.order);
@ -4096,8 +4096,8 @@ void setIntervalQueryRange(STaskRuntimeEnv *pRuntimeEnv, TSKEY key) {
*/ */
STimeWindow w = TSWINDOW_INITIALIZER; STimeWindow w = TSWINDOW_INITIALIZER;
TSKEY sk = MIN(win.skey, win.ekey); TSKEY sk = TMIN(win.skey, win.ekey);
TSKEY ek = MAX(win.skey, win.ekey); TSKEY ek = TMAX(win.skey, win.ekey);
getAlignQueryTimeWindow(pQueryAttr, win.skey, sk, ek, &w); getAlignQueryTimeWindow(pQueryAttr, win.skey, sk, ek, &w);
// if (pResultRowInfo->prevSKey == TSKEY_INITIAL_VAL) { // if (pResultRowInfo->prevSKey == TSKEY_INITIAL_VAL) {
@ -4222,7 +4222,7 @@ static void updateNumOfRowsInResultRows(STaskRuntimeEnv* pRuntimeEnv, SQLFunctio
} }
// SResultRowEntryInfo* pCell = getResultCell(pResult, j, rowCellInfoOffset); // SResultRowEntryInfo* pCell = getResultCell(pResult, j, rowCellInfoOffset);
// pResult->numOfRows = (uint16_t)(MAX(pResult->numOfRows, pCell->numOfRes)); // pResult->numOfRows = (uint16_t)(TMAX(pResult->numOfRows, pCell->numOfRes));
} }
} }
} }
@ -5302,7 +5302,7 @@ SOperatorInfo* createExchangeOperatorInfo(const SArray* pSources, const SArray*
rpcInit.user = (char *)"root"; rpcInit.user = (char *)"root";
rpcInit.idleTime = tsShellActivityTimer * 1000; rpcInit.idleTime = tsShellActivityTimer * 1000;
rpcInit.ckey = "key"; rpcInit.ckey = "key";
// rpcInit.spi = 1; rpcInit.spi = 1;
rpcInit.secret = (char *)"dcc5bed04851fec854c035b2e40263b6"; rpcInit.secret = (char *)"dcc5bed04851fec854c035b2e40263b6";
pInfo->pTransporter = rpcOpen(&rpcInit); pInfo->pTransporter = rpcOpen(&rpcInit);
@ -7204,8 +7204,8 @@ SOperatorInfo* createFillOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInf
struct SFillColInfo* pColInfo = createFillColInfo(pExpr, numOfOutput, pQueryAttr->fillVal); struct SFillColInfo* pColInfo = createFillColInfo(pExpr, numOfOutput, pQueryAttr->fillVal);
STimeWindow w = TSWINDOW_INITIALIZER; STimeWindow w = TSWINDOW_INITIALIZER;
TSKEY sk = MIN(pQueryAttr->window.skey, pQueryAttr->window.ekey); TSKEY sk = TMIN(pQueryAttr->window.skey, pQueryAttr->window.ekey);
TSKEY ek = MAX(pQueryAttr->window.skey, pQueryAttr->window.ekey); TSKEY ek = TMAX(pQueryAttr->window.skey, pQueryAttr->window.ekey);
getAlignQueryTimeWindow(pQueryAttr, pQueryAttr->window.skey, sk, ek, &w); getAlignQueryTimeWindow(pQueryAttr, pQueryAttr->window.skey, sk, ek, &w);
pInfo->pFillInfo = pInfo->pFillInfo =

12
source/libs/function/src/taggfunction.c
View File

@ -443,7 +443,7 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI
pInfo->intermediateBytes = (int16_t)sizeof(SPercentileInfo); pInfo->intermediateBytes = (int16_t)sizeof(SPercentileInfo);
} else if (functionId == FUNCTION_LEASTSQR) { } else if (functionId == FUNCTION_LEASTSQR) {
pInfo->type = TSDB_DATA_TYPE_BINARY; pInfo->type = TSDB_DATA_TYPE_BINARY;
pInfo->bytes = MAX(AVG_FUNCTION_INTER_BUFFER_SIZE, sizeof(SLeastsquaresInfo)); // string pInfo->bytes = TMAX(AVG_FUNCTION_INTER_BUFFER_SIZE, sizeof(SLeastsquaresInfo)); // string
pInfo->intermediateBytes = pInfo->bytes; pInfo->intermediateBytes = pInfo->bytes;
} else if (functionId == FUNCTION_FIRST_DST || functionId == FUNCTION_LAST_DST) { } else if (functionId == FUNCTION_FIRST_DST || functionId == FUNCTION_LAST_DST) {
pInfo->type = TSDB_DATA_TYPE_BINARY; pInfo->type = TSDB_DATA_TYPE_BINARY;
@ -1331,7 +1331,7 @@ static void max_func_merge(SQLFunctionCtx *pCtx) {
continue; \ continue; \
} \ } \
(num) += 1; \ (num) += 1; \
(r) += POW2(((type *)d)[i] - (delta)); \ (r) += TPOW2(((type *)d)[i] - (delta)); \
} }
static void stddev_function(SQLFunctionCtx *pCtx) { static void stddev_function(SQLFunctionCtx *pCtx) {
@ -1370,7 +1370,7 @@ static void stddev_function(SQLFunctionCtx *pCtx) {
continue; continue;
} }
num += 1; num += 1;
*retVal += POW2(((int32_t *)pData)[i] - avg); *retVal += TPOW2(((int32_t *)pData)[i] - avg);
} }
break; break;
} }
@ -1484,7 +1484,7 @@ static void stddev_dst_function(SQLFunctionCtx *pCtx) {
continue; continue;
} }
num += 1; num += 1;
*retVal += POW2(((int32_t *)pData)[i] - avg); *retVal += TPOW2(((int32_t *)pData)[i] - avg);
} }
break; break;
} }
@ -4210,8 +4210,8 @@ static void mergeTableBlockDist(SResultRowEntryInfo* pResInfo, const STableBlock
pDist->totalRows += pSrc->totalRows; pDist->totalRows += pSrc->totalRows;
if (pResInfo->hasResult == DATA_SET_FLAG) { if (pResInfo->hasResult == DATA_SET_FLAG) {
pDist->maxRows = MAX(pDist->maxRows, pSrc->maxRows); pDist->maxRows = TMAX(pDist->maxRows, pSrc->maxRows);
pDist->minRows = MIN(pDist->minRows, pSrc->minRows); pDist->minRows = TMIN(pDist->minRows, pSrc->minRows);
} else { } else {
pDist->maxRows = pSrc->maxRows; pDist->maxRows = pSrc->maxRows;
pDist->minRows = pSrc->minRows; pDist->minRows = pSrc->minRows;

14
source/libs/function/src/tbinoperator.c
View File

@ -26,7 +26,7 @@ void calc_i32_i32_add(void *left, void *right, int32_t numLeft, int32_t numRight
int32_t *pRight = (int32_t *)right; int32_t *pRight = (int32_t *)right;
double * pOutput = (double *)output; double * pOutput = (double *)output;
int32_t i = (order == TSDB_ORDER_ASC) ? 0 : MAX(numLeft, numRight) - 1; int32_t i = (order == TSDB_ORDER_ASC) ? 0 : TMAX(numLeft, numRight) - 1;
int32_t step = (order == TSDB_ORDER_ASC) ? 1 : -1; int32_t step = (order == TSDB_ORDER_ASC) ? 1 : -1;
if (numLeft == numRight) { if (numLeft == numRight) {
@ -181,7 +181,7 @@ _getValueAddr_fn_t getVectorValueAddrFn(int32_t srcType) {
} }
void vectorAdd(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int32_t _ord) { void vectorAdd(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int32_t _ord) {
int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : MAX(pLeft->num, pRight->num) - 1; int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1;
int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1;
double *output=(double*)out; double *output=(double*)out;
@ -220,7 +220,7 @@ void vectorAdd(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int
} }
void vectorSub(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int32_t _ord) { void vectorSub(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int32_t _ord) {
int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : MAX(pLeft->num, pRight->num) - 1; int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1;
int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1;
double *output=(double*)out; double *output=(double*)out;
@ -257,7 +257,7 @@ void vectorSub(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int
} }
} }
void vectorMultiply(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int32_t _ord) { void vectorMultiply(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int32_t _ord) {
int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : MAX(pLeft->num, pRight->num) - 1; int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1;
int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1;
double *output=(double*)out; double *output=(double*)out;
@ -296,7 +296,7 @@ void vectorMultiply(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out
} }
void vectorDivide(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int32_t _ord) { void vectorDivide(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int32_t _ord) {
int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : MAX(pLeft->num, pRight->num) - 1; int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1;
int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1;
double *output=(double*)out; double *output=(double*)out;
@ -342,7 +342,7 @@ void vectorDivide(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out,
} }
void vectorRemainder(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int32_t _ord) { void vectorRemainder(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int32_t _ord) {
int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : MAX(pLeft->num, pRight->num) - 1; int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1;
int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1;
double * output = (double *)out; double * output = (double *)out;
@ -416,7 +416,7 @@ void vectorRemainder(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *ou
void vectorConcat(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int32_t _ord) { void vectorConcat(SScalarFuncParam* pLeft, SScalarFuncParam* pRight, void *out, int32_t _ord) {
int32_t len = pLeft->bytes + pRight->bytes; int32_t len = pLeft->bytes + pRight->bytes;
int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : MAX(pLeft->num, pRight->num) - 1; int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1;
int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1;
char *output = (char *)out; char *output = (char *)out;

6
source/libs/index/src/index_fst.c
View File

@ -152,7 +152,7 @@ uint64_t fstUnFinishedNodesFindCommPrefixAndSetOutput(FstUnFinishedNodes* node,
uint64_t addPrefix = 0; uint64_t addPrefix = 0;
uint8_t* data = fstSliceData(s, NULL); uint8_t* data = fstSliceData(s, NULL);
if (t && t->inp == data[i]) { if (t && t->inp == data[i]) {
uint64_t commPrefix = MIN(t->out, *out); uint64_t commPrefix = TMIN(t->out, *out);
uint64_t tAddPrefix = t->out - commPrefix; uint64_t tAddPrefix = t->out - commPrefix;
(*out) = (*out) - commPrefix; (*out) = (*out) - commPrefix;
t->out = commPrefix; t->out = commPrefix;
@ -244,8 +244,8 @@ void fstStateCompileForAnyTrans(FstCountingWriter* w, CompiledAddr addr, FstBuil
bool anyOuts = (node->finalOutput != 0); bool anyOuts = (node->finalOutput != 0);
for (size_t i = 0; i < sz; i++) { for (size_t i = 0; i < sz; i++) {
FstTransition* t = taosArrayGet(node->trans, i); FstTransition* t = taosArrayGet(node->trans, i);
tSize = MAX(tSize, packDeltaSize(addr, t->addr)); tSize = TMAX(tSize, packDeltaSize(addr, t->addr));
oSize = MAX(oSize, packSize(t->out)); oSize = TMAX(oSize, packSize(t->out));
anyOuts = anyOuts || (t->out != 0); anyOuts = anyOuts || (t->out != 0);
} }

12
source/libs/parser/src/astValidate.c
View File

@ -1267,7 +1267,7 @@ static int32_t checkFillQueryRange(SQueryStmtInfo* pQueryInfo, SMsgBuf* pMsgBuf)
// return buildInvalidOperationMsg(pMsgBuf, msg1); // return buildInvalidOperationMsg(pMsgBuf, msg1);
// } // }
int64_t timeRange = ABS(pQueryInfo->window.skey - pQueryInfo->window.ekey); int64_t timeRange = TABS(pQueryInfo->window.skey - pQueryInfo->window.ekey);
int64_t intervalRange = 0; int64_t intervalRange = 0;
if (!TIME_IS_VAR_DURATION(pQueryInfo->interval.intervalUnit)) { if (!TIME_IS_VAR_DURATION(pQueryInfo->interval.intervalUnit)) {
@ -1353,7 +1353,7 @@ int32_t validateFillNode(SQueryStmtInfo *pQueryInfo, SSqlNode* pSqlNode, SMsgBuf
numOfFillVal = numOfFields; numOfFillVal = numOfFields;
} }
} else { } else {
numOfFillVal = MIN(num, numOfFields); numOfFillVal = TMIN(num, numOfFields);
} }
int32_t j = 1; int32_t j = 1;
@ -1928,7 +1928,7 @@ void setResultColName(char* name, tSqlExprItem* pItem, SToken* pToken, SToken* f
tstrncpy(name, pItem->aliasName, TSDB_COL_NAME_LEN); tstrncpy(name, pItem->aliasName, TSDB_COL_NAME_LEN);
} else if (multiCols) { } else if (multiCols) {
char uname[TSDB_COL_NAME_LEN] = {0}; char uname[TSDB_COL_NAME_LEN] = {0};
int32_t len = MIN(pToken->n + 1, TSDB_COL_NAME_LEN); int32_t len = TMIN(pToken->n + 1, TSDB_COL_NAME_LEN);
tstrncpy(uname, pToken->z, len); tstrncpy(uname, pToken->z, len);
if (tsKeepOriginalColumnName) { // keep the original column name if (tsKeepOriginalColumnName) { // keep the original column name
@ -1944,7 +1944,7 @@ void setResultColName(char* name, tSqlExprItem* pItem, SToken* pToken, SToken* f
tstrncpy(name, tmp, TSDB_COL_NAME_LEN); tstrncpy(name, tmp, TSDB_COL_NAME_LEN);
} }
} else { // use the user-input result column name } else { // use the user-input result column name
int32_t len = MIN(pItem->pNode->exprToken.n + 1, TSDB_COL_NAME_LEN); int32_t len = TMIN(pItem->pNode->exprToken.n + 1, TSDB_COL_NAME_LEN);
tstrncpy(name, pItem->pNode->exprToken.z, len); tstrncpy(name, pItem->pNode->exprToken.z, len);
} }
} }
@ -2948,7 +2948,7 @@ static SSchema createConstantColumnSchema(SVariant* pVal, const SToken* exprStr,
if (name != NULL) { if (name != NULL) {
tstrncpy(s.name, name, sizeof(s.name)); tstrncpy(s.name, name, sizeof(s.name));
} else { } else {
size_t tlen = MIN(sizeof(s.name), exprStr->n + 1); size_t tlen = TMIN(sizeof(s.name), exprStr->n + 1);
tstrncpy(s.name, exprStr->z, tlen); tstrncpy(s.name, exprStr->z, tlen);
strdequote(s.name); strdequote(s.name);
} }
@ -3026,7 +3026,7 @@ int32_t addProjectionExprAndResColumn(SQueryStmtInfo* pQueryInfo, tSqlExprItem*
SSchema colSchema = createConstantColumnSchema(&pItem->pNode->value, &pItem->pNode->exprToken, pItem->aliasName); SSchema colSchema = createConstantColumnSchema(&pItem->pNode->value, &pItem->pNode->exprToken, pItem->aliasName);
char token[TSDB_COL_NAME_LEN] = {0}; char token[TSDB_COL_NAME_LEN] = {0};
tstrncpy(token, pItem->pNode->exprToken.z, MIN(TSDB_COL_NAME_LEN, TSDB_COL_NAME_LEN)); tstrncpy(token, pItem->pNode->exprToken.z, TMIN(TSDB_COL_NAME_LEN, TSDB_COL_NAME_LEN));
STableMetaInfo* pTableMetaInfo = getMetaInfo(pQueryInfo, index.tableIndex); STableMetaInfo* pTableMetaInfo = getMetaInfo(pQueryInfo, index.tableIndex);
SColumn c = createColumn(pTableMetaInfo->pTableMeta->uid, pTableMetaInfo->aliasName, index.type, &colSchema); SColumn c = createColumn(pTableMetaInfo->pTableMeta->uid, pTableMetaInfo->aliasName, index.type, &colSchema);

2
source/libs/parser/src/insertParser.c
View File

@ -156,7 +156,7 @@ static int32_t buildOutput(SInsertParseContext* pCxt) {
taosHashGetClone(pCxt->pVgroupsHashObj, (const char*)&src->vgId, sizeof(src->vgId), &dst->vg); taosHashGetClone(pCxt->pVgroupsHashObj, (const char*)&src->vgId, sizeof(src->vgId), &dst->vg);
dst->numOfTables = src->numOfTables; dst->numOfTables = src->numOfTables;
dst->size = src->size; dst->size = src->size;
SWAP(dst->pData, src->pData, char*); TSWAP(dst->pData, src->pData, char*);
buildMsgHeader(dst); buildMsgHeader(dst);
taosArrayPush(pCxt->pOutput->pDataBlocks, &dst); taosArrayPush(pCxt->pOutput->pDataBlocks, &dst);
} }

2
source/libs/planner/src/physicalPlan.c
View File

@ -135,7 +135,7 @@ static SDataSink* createDataInserter(SPlanContext* pCxt, SVgDataBlocks* pBlocks,
SDataInserter* inserter = (SDataInserter*)initDataSink(DSINK_Insert, sizeof(SDataInserter), pRoot); SDataInserter* inserter = (SDataInserter*)initDataSink(DSINK_Insert, sizeof(SDataInserter), pRoot);
inserter->numOfTables = pBlocks->numOfTables; inserter->numOfTables = pBlocks->numOfTables;
inserter->size = pBlocks->size; inserter->size = pBlocks->size;
SWAP(inserter->pData, pBlocks->pData, char*); TSWAP(inserter->pData, pBlocks->pData, char*);
return (SDataSink*)inserter; return (SDataSink*)inserter;
} }

2
source/libs/qcom/src/queryUtil.c
View File

@ -85,7 +85,7 @@ static void* pTaskQueue = NULL;
int32_t initTaskQueue() { int32_t initTaskQueue() {
double factor = 4.0; double factor = 4.0;
int32_t numOfThreads = MAX((int)(tsNumOfCores * tsNumOfThreadsPerCore / factor), 2); int32_t numOfThreads = TMAX((int)(tsNumOfCores * tsNumOfThreadsPerCore / factor), 2);
int32_t queueSize = tsMaxConnections * 2; int32_t queueSize = tsMaxConnections * 2;
pTaskQueue = taosInitScheduler(queueSize, numOfThreads, "tsc"); pTaskQueue = taosInitScheduler(queueSize, numOfThreads, "tsc");

26
source/libs/qworker/inc/qworkerInt.h
View File

@ -22,17 +22,17 @@ extern "C" {
#include "tlockfree.h" #include "tlockfree.h"
#define QWORKER_DEFAULT_SCHEDULER_NUMBER 10000 #define QW_DEFAULT_SCHEDULER_NUMBER 10000
#define QWORKER_DEFAULT_TASK_NUMBER 10000 #define QW_DEFAULT_TASK_NUMBER 10000
#define QWORKER_DEFAULT_SCH_TASK_NUMBER 10000 #define QW_DEFAULT_SCH_TASK_NUMBER 10000
#define QW_DEFAULT_SHORT_RUN_TIMES 2
enum { enum {
QW_PHASE_PRE_QUERY = 1, QW_PHASE_PRE_QUERY = 1,
QW_PHASE_POST_QUERY, QW_PHASE_POST_QUERY,
QW_PHASE_PRE_CQUERY,
QW_PHASE_POST_CQUERY,
QW_PHASE_PRE_FETCH, QW_PHASE_PRE_FETCH,
QW_PHASE_POST_FETCH, QW_PHASE_POST_FETCH,
QW_PHASE_PRE_CQUERY,
QW_PHASE_POST_CQUERY,
}; };
enum { enum {
@ -83,11 +83,9 @@ typedef struct SQWMsg {
} SQWMsg; } SQWMsg;
typedef struct SQWPhaseInput { typedef struct SQWPhaseInput {
int8_t status; int8_t taskStatus;
int8_t taskType; int8_t taskType;
int32_t code; int32_t code;
qTaskInfo_t taskHandle;
DataSinkHandle sinkHandle;
} SQWPhaseInput; } SQWPhaseInput;
typedef struct SQWPhaseOutput { typedef struct SQWPhaseOutput {
@ -111,6 +109,7 @@ typedef struct SQWTaskCtx {
void *cancelConnection; void *cancelConnection;
bool emptyRes; bool emptyRes;
bool multiExec;
int8_t queryContinue; int8_t queryContinue;
int8_t queryInQueue; int8_t queryInQueue;
int32_t rspCode; int32_t rspCode;
@ -133,7 +132,7 @@ typedef struct SQWorkerMgmt {
int8_t nodeType; int8_t nodeType;
int32_t nodeId; int32_t nodeId;
SRWLatch schLock; SRWLatch schLock;
SRWLatch ctxLock; //SRWLatch ctxLock;
SHashObj *schHash; //key: schedulerId, value: SQWSchStatus SHashObj *schHash; //key: schedulerId, value: SQWSchStatus
SHashObj *ctxHash; //key: queryId+taskId, value: SQWTaskCtx SHashObj *ctxHash; //key: queryId+taskId, value: SQWTaskCtx
void *nodeObj; void *nodeObj;
@ -144,6 +143,8 @@ typedef struct SQWorkerMgmt {
#define QW_IDS() sId, qId, tId #define QW_IDS() sId, qId, tId
#define QW_FPARAMS() mgmt, QW_IDS() #define QW_FPARAMS() mgmt, QW_IDS()
#define QW_GET_EVENT_VALUE(ctx, event) atomic_load_8(&(ctx)->events[event])
#define QW_IS_EVENT_RECEIVED(ctx, event) (atomic_load_8(&(ctx)->events[event]) == QW_EVENT_RECEIVED) #define QW_IS_EVENT_RECEIVED(ctx, event) (atomic_load_8(&(ctx)->events[event]) == QW_EVENT_RECEIVED)
#define QW_IS_EVENT_PROCESSED(ctx, event) (atomic_load_8(&(ctx)->events[event]) == QW_EVENT_PROCESSED) #define QW_IS_EVENT_PROCESSED(ctx, event) (atomic_load_8(&(ctx)->events[event]) == QW_EVENT_PROCESSED)
#define QW_SET_EVENT_RECEIVED(ctx, event) atomic_store_8(&(ctx)->events[event], QW_EVENT_RECEIVED) #define QW_SET_EVENT_RECEIVED(ctx, event) atomic_store_8(&(ctx)->events[event], QW_EVENT_RECEIVED)
@ -151,9 +152,10 @@ typedef struct SQWorkerMgmt {
#define QW_GET_PHASE(ctx) atomic_load_8(&(ctx)->phase) #define QW_GET_PHASE(ctx) atomic_load_8(&(ctx)->phase)
#define QW_SET_RSP_CODE(ctx, code) atomic_val_compare_exchange_32(&(ctx)->rspCode, 0, code) #define QW_SET_RSP_CODE(ctx, code) atomic_store_32(&(ctx)->rspCode, code)
#define QW_UPDATE_RSP_CODE(ctx, code) atomic_val_compare_exchange_32(&(ctx)->rspCode, 0, code)
#define QW_IN_EXECUTOR(ctx) (QW_GET_PHASE(ctx) == QW_PHASE_PRE_QUERY || QW_GET_PHASE(ctx) == QW_PHASE_PRE_CQUERY || QW_GET_PHASE(ctx) == QW_PHASE_PRE_FETCH) #define QW_IS_QUERY_RUNNING(ctx) (QW_GET_PHASE(ctx) == QW_PHASE_PRE_QUERY || QW_GET_PHASE(ctx) == QW_PHASE_PRE_CQUERY)
#define QW_TASK_NOT_EXIST(code) (TSDB_CODE_QRY_SCH_NOT_EXIST == (code) || TSDB_CODE_QRY_TASK_NOT_EXIST == (code)) #define QW_TASK_NOT_EXIST(code) (TSDB_CODE_QRY_SCH_NOT_EXIST == (code) || TSDB_CODE_QRY_TASK_NOT_EXIST == (code))
#define QW_TASK_ALREADY_EXIST(code) (TSDB_CODE_QRY_TASK_ALREADY_EXIST == (code)) #define QW_TASK_ALREADY_EXIST(code) (TSDB_CODE_QRY_TASK_ALREADY_EXIST == (code))

1
source/libs/qworker/inc/qworkerMsg.h
View File

@ -30,6 +30,7 @@ int32_t qwProcessFetch(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t
int32_t qwProcessDrop(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, SQWMsg *qwMsg); int32_t qwProcessDrop(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, SQWMsg *qwMsg);
int32_t qwBuildAndSendDropRsp(void *connection, int32_t code); int32_t qwBuildAndSendDropRsp(void *connection, int32_t code);
int32_t qwBuildAndSendCancelRsp(SRpcMsg *pMsg, int32_t code);
int32_t qwBuildAndSendFetchRsp(void *connection, SRetrieveTableRsp *pRsp, int32_t dataLength, int32_t code); int32_t qwBuildAndSendFetchRsp(void *connection, SRetrieveTableRsp *pRsp, int32_t dataLength, int32_t code);
void qwBuildFetchRsp(void *msg, SOutputData *input, int32_t len); void qwBuildFetchRsp(void *msg, SOutputData *input, int32_t len);
int32_t qwBuildAndSendCQueryMsg(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, void *connection); int32_t qwBuildAndSendCQueryMsg(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, void *connection);

632
source/libs/qworker/src/qworker.c
View File

File diff suppressed because it is too large Load Diff

23
source/libs/qworker/src/qworkerMsg.c
View File

@ -50,6 +50,7 @@ int32_t qwBuildAndSendQueryRsp(void *connection, int32_t code) {
pRsp->code = code; pRsp->code = code;
SRpcMsg rpcRsp = { SRpcMsg rpcRsp = {
.msgType = TDMT_VND_QUERY_RSP,
.handle = pMsg->handle, .handle = pMsg->handle,
.ahandle = pMsg->ahandle, .ahandle = pMsg->ahandle,
.pCont = pRsp, .pCont = pRsp,
@ -68,6 +69,7 @@ int32_t qwBuildAndSendReadyRsp(void *connection, int32_t code) {
pRsp->code = code; pRsp->code = code;
SRpcMsg rpcRsp = { SRpcMsg rpcRsp = {
.msgType = TDMT_VND_RES_READY_RSP,
.handle = pMsg->handle, .handle = pMsg->handle,
.ahandle = pMsg->ahandle, .ahandle = pMsg->ahandle,
.pCont = pRsp, .pCont = pRsp,
@ -98,7 +100,7 @@ int32_t qwBuildAndSendStatusRsp(SRpcMsg *pMsg, SSchedulerStatusRsp *sStatus) {
} }
SRpcMsg rpcRsp = { SRpcMsg rpcRsp = {
.msgType = pMsg->msgType + 1, .msgType = TDMT_VND_TASKS_STATUS_RSP,
.handle = pMsg->handle, .handle = pMsg->handle,
.ahandle = pMsg->ahandle, .ahandle = pMsg->ahandle,
.pCont = pRsp, .pCont = pRsp,
@ -121,6 +123,7 @@ int32_t qwBuildAndSendFetchRsp(void *connection, SRetrieveTableRsp *pRsp, int32_
} }
SRpcMsg rpcRsp = { SRpcMsg rpcRsp = {
.msgType = TDMT_VND_FETCH_RSP,
.handle = pMsg->handle, .handle = pMsg->handle,
.ahandle = pMsg->ahandle, .ahandle = pMsg->ahandle,
.pCont = pRsp, .pCont = pRsp,
@ -138,6 +141,7 @@ int32_t qwBuildAndSendCancelRsp(SRpcMsg *pMsg, int32_t code) {
pRsp->code = code; pRsp->code = code;
SRpcMsg rpcRsp = { SRpcMsg rpcRsp = {
.msgType = TDMT_VND_CANCEL_TASK_RSP,
.handle = pMsg->handle, .handle = pMsg->handle,
.ahandle = pMsg->ahandle, .ahandle = pMsg->ahandle,
.pCont = pRsp, .pCont = pRsp,
@ -155,6 +159,7 @@ int32_t qwBuildAndSendDropRsp(void *connection, int32_t code) {
pRsp->code = code; pRsp->code = code;
SRpcMsg rpcRsp = { SRpcMsg rpcRsp = {
.msgType = TDMT_VND_DROP_TASK_RSP,
.handle = pMsg->handle, .handle = pMsg->handle,
.ahandle = pMsg->ahandle, .ahandle = pMsg->ahandle,
.pCont = pRsp, .pCont = pRsp,
@ -273,7 +278,7 @@ int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) {
SQWorkerMgmt *mgmt = (SQWorkerMgmt *)qWorkerMgmt; SQWorkerMgmt *mgmt = (SQWorkerMgmt *)qWorkerMgmt;
if (NULL == msg || pMsg->contLen <= sizeof(*msg)) { if (NULL == msg || pMsg->contLen <= sizeof(*msg)) {
QW_ELOG("invalid query msg, contLen:%d", pMsg->contLen); QW_ELOG("invalid query msg, msg:%p, msgLen:%d", msg, pMsg->contLen);
QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
} }
@ -306,15 +311,11 @@ int32_t qWorkerProcessCQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) {
SQWTaskCtx *handles = NULL; SQWTaskCtx *handles = NULL;
SQWorkerMgmt *mgmt = (SQWorkerMgmt *)qWorkerMgmt; SQWorkerMgmt *mgmt = (SQWorkerMgmt *)qWorkerMgmt;
if (NULL == msg || pMsg->contLen <= sizeof(*msg)) { if (NULL == msg || pMsg->contLen < sizeof(*msg)) {
QW_ELOG("invalid cquery msg, contLen:%d", pMsg->contLen); QW_ELOG("invalid cquery msg, msg:%p, msgLen:%d", msg, pMsg->contLen);
QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
} }
msg->sId = be64toh(msg->sId);
msg->queryId = be64toh(msg->queryId);
msg->taskId = be64toh(msg->taskId);
uint64_t sId = msg->sId; uint64_t sId = msg->sId;
uint64_t qId = msg->queryId; uint64_t qId = msg->queryId;
uint64_t tId = msg->taskId; uint64_t tId = msg->taskId;
@ -335,14 +336,13 @@ int32_t qWorkerProcessReadyMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg){
return TSDB_CODE_QRY_INVALID_INPUT; return TSDB_CODE_QRY_INVALID_INPUT;
} }
SQWorkerMgmt *mgmt = (SQWorkerMgmt *)qWorkerMgmt;
SResReadyReq *msg = pMsg->pCont; SResReadyReq *msg = pMsg->pCont;
if (NULL == msg || pMsg->contLen < sizeof(*msg)) { if (NULL == msg || pMsg->contLen < sizeof(*msg)) {
qError("invalid task status msg"); QW_ELOG("invalid task ready msg, msg:%p, msgLen:%d", msg, pMsg->contLen);
QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
} }
SQWorkerMgmt *mgmt = (SQWorkerMgmt *)qWorkerMgmt;
msg->sId = be64toh(msg->sId); msg->sId = be64toh(msg->sId);
msg->queryId = be64toh(msg->queryId); msg->queryId = be64toh(msg->queryId);
msg->taskId = be64toh(msg->taskId); msg->taskId = be64toh(msg->taskId);
@ -398,6 +398,7 @@ int32_t qWorkerProcessFetchMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) {
SQWorkerMgmt *mgmt = (SQWorkerMgmt *)qWorkerMgmt; SQWorkerMgmt *mgmt = (SQWorkerMgmt *)qWorkerMgmt;
if (NULL == msg || pMsg->contLen < sizeof(*msg)) { if (NULL == msg || pMsg->contLen < sizeof(*msg)) {
QW_ELOG("invalid fetch msg, msg:%p, msgLen:%d", msg, pMsg->contLen);
QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
} }

803
source/libs/qworker/test/qworkerTests.cpp
View File

File diff suppressed because it is too large Load Diff

53
source/libs/scheduler/src/scheduler.c
View File

@ -275,10 +275,13 @@ int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) {
} }
int32_t schRecordTaskSucceedNode(SSchTask *pTask) { int32_t schRecordTaskSucceedNode(SSchJob *pJob, SSchTask *pTask) {
SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, atomic_load_8(&pTask->candidateIdx)); int32_t idx = atomic_load_8(&pTask->candidateIdx);
SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, idx);
assert(NULL != addr); if (NULL == addr) {
SCH_TASK_ELOG("taosArrayGet candidate addr failed, idx:%d, size:%d", idx, (int32_t)taosArrayGetSize(pTask->candidateAddrs));
SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR);
}
pTask->succeedAddr = *addr; pTask->succeedAddr = *addr;
@ -578,9 +581,10 @@ int32_t schProcessOnJobFailureImpl(SSchJob *pJob, int32_t status, int32_t errCod
tsem_post(&pJob->rspSem); tsem_post(&pJob->rspSem);
} }
SCH_ERR_RET(atomic_load_32(&pJob->errCode)); int32_t code = atomic_load_32(&pJob->errCode);
SCH_ERR_RET(code);
assert(0); SCH_JOB_ELOG("job errCode is invalid, errCode:%d", code);
} }
@ -721,7 +725,7 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) {
SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_PARTIAL_SUCCEED); SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_PARTIAL_SUCCEED);
SCH_ERR_JRET(schRecordTaskSucceedNode(pTask)); SCH_ERR_JRET(schRecordTaskSucceedNode(pJob, pTask));
int32_t parentNum = pTask->parents ? (int32_t)taosArrayGetSize(pTask->parents) : 0; int32_t parentNum = pTask->parents ? (int32_t)taosArrayGetSize(pTask->parents) : 0;
if (parentNum == 0) { if (parentNum == 0) {
@ -734,11 +738,11 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) {
SCH_UNLOCK(SCH_WRITE, &pTask->level->lock); SCH_UNLOCK(SCH_WRITE, &pTask->level->lock);
if (taskDone < pTask->level->taskNum) { if (taskDone < pTask->level->taskNum) {
SCH_TASK_ELOG("wait all tasks, done:%d, all:%d", taskDone, pTask->level->taskNum); SCH_TASK_DLOG("wait all tasks, done:%d, all:%d", taskDone, pTask->level->taskNum);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} else if (taskDone > pTask->level->taskNum) { } else if (taskDone > pTask->level->taskNum) {
assert(0); SCH_TASK_ELOG("taskDone number invalid, done:%d, total:%d", taskDone, pTask->level->taskNum);
} }
if (pTask->level->taskFailed > 0) { if (pTask->level->taskFailed > 0) {
@ -871,18 +875,21 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch
} }
atomic_store_ptr(&pJob->res, rsp); atomic_store_ptr(&pJob->res, rsp);
atomic_store_32(&pJob->resNumOfRows, rsp->numOfRows); atomic_add_fetch_32(&pJob->resNumOfRows, htonl(rsp->numOfRows));
if (rsp->completed) { if (rsp->completed) {
SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_SUCCEED); SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_SUCCEED);
} }
SCH_TASK_DLOG("got fetch rsp, rows:%d, complete:%d", htonl(rsp->numOfRows), rsp->completed);
SCH_ERR_JRET(schProcessOnDataFetched(pJob)); SCH_ERR_JRET(schProcessOnDataFetched(pJob));
break; break;
} }
case TDMT_VND_DROP_TASK: { case TDMT_VND_DROP_TASK_RSP: {
// SHOULD NEVER REACH HERE // SHOULD NEVER REACH HERE
assert(0); SCH_TASK_ELOG("invalid status to handle drop task rsp, ref:%d", atomic_load_32(&pJob->ref));
SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR);
break; break;
} }
default: default:
@ -1031,6 +1038,8 @@ int32_t schAsyncSendMsg(void *transport, SEpSet* epSet, uint64_t qId, uint64_t t
SCH_ERR_JRET(code); SCH_ERR_JRET(code);
} }
qDebug("QID:%"PRIx64 ",TID:%"PRIx64 " req msg sent, type:%d, %s", qId, tId, msgType, TMSG_INFO(msgType));
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
_return: _return:
@ -1289,8 +1298,10 @@ void schDropJobAllTasks(SSchJob *pJob) {
} }
int32_t schExecJobImpl(void *transport, SArray *pNodeList, SQueryDag* pDag, struct SSchJob** job, bool syncSchedule) { int32_t schExecJobImpl(void *transport, SArray *pNodeList, SQueryDag* pDag, struct SSchJob** job, bool syncSchedule) {
qDebug("QID:0x%"PRIx64" job started", pDag->queryId);
if (pNodeList && taosArrayGetSize(pNodeList) <= 0) { if (pNodeList && taosArrayGetSize(pNodeList) <= 0) {
qDebug("QID:%"PRIx64" input exec nodeList is empty", pDag->queryId); qDebug("QID:0x%"PRIx64" input exec nodeList is empty", pDag->queryId);
} }
int32_t code = 0; int32_t code = 0;
@ -1359,7 +1370,7 @@ _return:
*(SSchJob **)job = NULL; *(SSchJob **)job = NULL;
scheduleFreeJob(pJob); schedulerFreeJob(pJob);
SCH_RET(code); SCH_RET(code);
} }
@ -1404,7 +1415,7 @@ int32_t schedulerInit(SSchedulerCfg *cfg) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t scheduleExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, struct SSchJob** pJob, SQueryResult *pRes) { int32_t schedulerExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, struct SSchJob** pJob, SQueryResult *pRes) {
if (NULL == transport || NULL == pDag || NULL == pDag->pSubplans || NULL == pJob || NULL == pRes) { if (NULL == transport || NULL == pDag || NULL == pDag->pSubplans || NULL == pJob || NULL == pRes) {
SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
} }
@ -1421,7 +1432,7 @@ int32_t scheduleExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, stru
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t scheduleAsyncExecJob(void *transport, SArray *pNodeList, SQueryDag* pDag, struct SSchJob** pJob) { int32_t schedulerAsyncExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, struct SSchJob** pJob) {
if (NULL == transport || NULL == pDag || NULL == pDag->pSubplans || NULL == pJob) { if (NULL == transport || NULL == pDag || NULL == pDag->pSubplans || NULL == pJob) {
SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
} }
@ -1554,7 +1565,7 @@ _return:
} }
int32_t scheduleFetchRows(SSchJob *pJob, void** pData) { int32_t schedulerFetchRows(SSchJob *pJob, void** pData) {
if (NULL == pJob || NULL == pData) { if (NULL == pJob || NULL == pData) {
SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
} }
@ -1619,11 +1630,12 @@ _return:
} }
*pData = rsp; *pData = rsp;
SCH_JOB_DLOG("empty res and set query complete, code:%x", code);
} }
atomic_val_compare_exchange_8(&pJob->userFetch, 1, 0); atomic_val_compare_exchange_8(&pJob->userFetch, 1, 0);
SCH_JOB_DLOG("fetch done, code:%s", tstrerror(code)); SCH_JOB_DLOG("fetch done, totalRows:%d, code:%s", pJob->resNumOfRows, tstrerror(code));
atomic_sub_fetch_32(&pJob->ref, 1); atomic_sub_fetch_32(&pJob->ref, 1);
SCH_RET(code); SCH_RET(code);
@ -1641,7 +1653,7 @@ int32_t scheduleCancelJob(void *job) {
SCH_RET(code); SCH_RET(code);
} }
void scheduleFreeJob(void *job) { void schedulerFreeJob(void *job) {
if (NULL == job) { if (NULL == job) {
return; return;
} }
@ -1670,7 +1682,8 @@ void scheduleFreeJob(void *job) {
usleep(1); usleep(1);
} else { } else {
assert(0); SCH_JOB_ELOG("invalid job ref number, ref:%d", ref);
break;
} }
} }

20
source/libs/scheduler/test/schedulerTests.cpp
View File

@ -334,7 +334,7 @@ void schtFreeQueryJob(int32_t freeThread) {
SSchJob *job = atomic_load_ptr(&pQueryJob); SSchJob *job = atomic_load_ptr(&pQueryJob);
if (job && atomic_val_compare_exchange_ptr(&pQueryJob, job, NULL)) { if (job && atomic_val_compare_exchange_ptr(&pQueryJob, job, NULL)) {
scheduleFreeJob(job); schedulerFreeJob(job);
if (freeThread) { if (freeThread) {
if (++freeNum % schtTestPrintNum == 0) { if (++freeNum % schtTestPrintNum == 0) {
printf("FreeNum:%d\n", freeNum); printf("FreeNum:%d\n", freeNum);
@ -378,7 +378,7 @@ void* schtRunJobThread(void *aa) {
qnodeAddr.port = 6031; qnodeAddr.port = 6031;
taosArrayPush(qnodeList, &qnodeAddr); taosArrayPush(qnodeList, &qnodeAddr);
code = scheduleAsyncExecJob(mockPointer, qnodeList, &dag, &job); code = schedulerAsyncExecJob(mockPointer, qnodeList, &dag, &job);
assert(code == 0); assert(code == 0);
execTasks = taosHashInit(5, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_ENTRY_LOCK); execTasks = taosHashInit(5, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_ENTRY_LOCK);
@ -472,7 +472,7 @@ void* schtRunJobThread(void *aa) {
atomic_store_32(&schtStartFetch, 1); atomic_store_32(&schtStartFetch, 1);
void *data = NULL; void *data = NULL;
code = scheduleFetchRows(pQueryJob, &data); code = schedulerFetchRows(pQueryJob, &data);
assert(code == 0 || code); assert(code == 0 || code);
if (0 == code) { if (0 == code) {
@ -482,7 +482,7 @@ void* schtRunJobThread(void *aa) {
} }
data = NULL; data = NULL;
code = scheduleFetchRows(pQueryJob, &data); code = schedulerFetchRows(pQueryJob, &data);
assert(code == 0 || code); assert(code == 0 || code);
schtFreeQueryJob(0); schtFreeQueryJob(0);
@ -539,7 +539,7 @@ TEST(queryTest, normalCase) {
schtSetExecNode(); schtSetExecNode();
schtSetAsyncSendMsgToServer(); schtSetAsyncSendMsgToServer();
code = scheduleAsyncExecJob(mockPointer, qnodeList, &dag, &pJob); code = schedulerAsyncExecJob(mockPointer, qnodeList, &dag, &pJob);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
SSchJob *job = (SSchJob *)pJob; SSchJob *job = (SSchJob *)pJob;
@ -594,7 +594,7 @@ TEST(queryTest, normalCase) {
pthread_create(&(thread1), &thattr, schtCreateFetchRspThread, job); pthread_create(&(thread1), &thattr, schtCreateFetchRspThread, job);
void *data = NULL; void *data = NULL;
code = scheduleFetchRows(job, &data); code = schedulerFetchRows(job, &data);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
SRetrieveTableRsp *pRsp = (SRetrieveTableRsp *)data; SRetrieveTableRsp *pRsp = (SRetrieveTableRsp *)data;
@ -603,11 +603,11 @@ TEST(queryTest, normalCase) {
tfree(data); tfree(data);
data = NULL; data = NULL;
code = scheduleFetchRows(job, &data); code = schedulerFetchRows(job, &data);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
ASSERT_TRUE(data); ASSERT_TRUE(data);
scheduleFreeJob(pJob); schedulerFreeJob(pJob);
schtFreeQueryDag(&dag); schtFreeQueryDag(&dag);
@ -649,11 +649,11 @@ TEST(insertTest, normalCase) {
pthread_create(&(thread1), &thattr, schtSendRsp, &pInsertJob); pthread_create(&(thread1), &thattr, schtSendRsp, &pInsertJob);
SQueryResult res = {0}; SQueryResult res = {0};
code = scheduleExecJob(mockPointer, qnodeList, &dag, &pInsertJob, &res); code = schedulerExecJob(mockPointer, qnodeList, &dag, &pInsertJob, &res);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
ASSERT_EQ(res.numOfRows, 20); ASSERT_EQ(res.numOfRows, 20);
scheduleFreeJob(pInsertJob); schedulerFreeJob(pInsertJob);
schedulerDestroy(); schedulerDestroy();
} }

8
source/libs/sync/inc/sync_type.h
View File

@ -43,12 +43,12 @@ typedef struct SSyncRaftLog SSyncRaftLog;
typedef struct SSyncRaftEntry SSyncRaftEntry; typedef struct SSyncRaftEntry SSyncRaftEntry;
#if 0 #if 0
#ifndef MIN #ifndef TMIN
#define MIN(x, y) (((x) < (y)) ? (x) : (y)) #define TMIN(x, y) (((x) < (y)) ? (x) : (y))
#endif #endif
#ifndef MAX #ifndef TMAX
#define MAX(x, y) (((x) > (y)) ? (x) : (y)) #define TMAX(x, y) (((x) > (y)) ? (x) : (y))
#endif #endif
#endif #endif

6
source/libs/sync/src/sync_raft_progress.c
View File

@ -59,7 +59,7 @@ bool syncRaftProgressMaybeUpdate(SSyncRaftProgress* progress, SyncIndex lastInde
probeAcked(progress); probeAcked(progress);
} }
progress->nextIndex = MAX(progress->nextIndex, lastIndex + 1); progress->nextIndex = TMAX(progress->nextIndex, lastIndex + 1);
return updated; return updated;
} }
@ -100,7 +100,7 @@ bool syncRaftProgressMaybeDecrTo(SSyncRaftProgress* progress,
return false; return false;
} }
progress->nextIndex = MAX(MIN(rejected, matchHint + 1), 1); progress->nextIndex = TMAX(TMIN(rejected, matchHint + 1), 1);
progress->probeSent = false; progress->probeSent = false;
return true; return true;
@ -166,7 +166,7 @@ void syncRaftProgressBecomeProbe(SSyncRaftProgress* progress) {
if (progress->state == PROGRESS_STATE_SNAPSHOT) { if (progress->state == PROGRESS_STATE_SNAPSHOT) {
SyncIndex pendingSnapshotIndex = progress->pendingSnapshotIndex; SyncIndex pendingSnapshotIndex = progress->pendingSnapshotIndex;
resetProgressState(progress, PROGRESS_STATE_PROBE); resetProgressState(progress, PROGRESS_STATE_PROBE);
progress->nextIndex = MAX(progress->matchIndex + 1, pendingSnapshotIndex + 1); progress->nextIndex = TMAX(progress->matchIndex + 1, pendingSnapshotIndex + 1);
} else { } else {
resetProgressState(progress, PROGRESS_STATE_PROBE); resetProgressState(progress, PROGRESS_STATE_PROBE);
progress->nextIndex = progress->matchIndex + 1; progress->nextIndex = progress->matchIndex + 1;

1
source/libs/transport/src/trans.c
View File

@ -32,6 +32,7 @@ void* rpcOpen(const SRpcInit* pInit) {
pRpc->cfp = pInit->cfp; pRpc->cfp = pInit->cfp;
pRpc->numOfThreads = pInit->numOfThreads > TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS : pInit->numOfThreads; pRpc->numOfThreads = pInit->numOfThreads > TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS : pInit->numOfThreads;
pRpc->connType = pInit->connType; pRpc->connType = pInit->connType;
pRpc->idleTime = pInit->idleTime;
pRpc->tcphandle = (*taosInitHandle[pRpc->connType])(0, pInit->localPort, pRpc->label, pRpc->numOfThreads, NULL, pRpc); pRpc->tcphandle = (*taosInitHandle[pRpc->connType])(0, pInit->localPort, pRpc->label, pRpc->numOfThreads, NULL, pRpc);
return pRpc; return pRpc;

176
source/libs/transport/src/transCli.c
View File

@ -17,6 +17,8 @@
#include "transComm.h" #include "transComm.h"
#define CONN_PERSIST_TIME(para) (para * 1000 * 10)
typedef struct SCliConn { typedef struct SCliConn {
uv_connect_t connReq; uv_connect_t connReq;
uv_stream_t* stream; uv_stream_t* stream;
@ -42,7 +44,7 @@ typedef struct SCliThrdObj {
uv_loop_t* loop; uv_loop_t* loop;
uv_async_t* cliAsync; // uv_async_t* cliAsync; //
uv_timer_t* pTimer; uv_timer_t* pTimer;
void* cache; // conn pool void* pool; // conn pool
queue msg; queue msg;
pthread_mutex_t msgMtx; pthread_mutex_t msgMtx;
uint64_t nextTimeout; // next timeout uint64_t nextTimeout; // next timeout
@ -63,15 +65,15 @@ typedef struct SConnList {
// conn pool // conn pool
// add expire timeout and capacity limit // add expire timeout and capacity limit
static void* connCacheCreate(int size); static void* creatConnPool(int size);
static void* connCacheDestroy(void* cache); static void* destroyConnPool(void* pool);
static SCliConn* getConnFromCache(void* cache, char* ip, uint32_t port); static SCliConn* getConnFromPool(void* pool, char* ip, uint32_t port);
static void addConnToCache(void* cache, char* ip, uint32_t port, SCliConn* conn); static void addConnToPool(void* pool, char* ip, uint32_t port, SCliConn* conn);
// register timer in each thread to clear expire conn // register timer in each thread to clear expire conn
static void clientTimeoutCb(uv_timer_t* handle); static void clientTimeoutCb(uv_timer_t* handle);
// process data read from server, auth/decompress etc later // process data read from server, auth/decompress etc later
static void clientProcessData(SCliConn* conn); static void clientHandleResp(SCliConn* conn);
// check whether already read complete packet from server // check whether already read complete packet from server
static bool clientReadComplete(SConnBuffer* pBuf); static bool clientReadComplete(SConnBuffer* pBuf);
// alloc buf for read // alloc buf for read
@ -84,13 +86,19 @@ static void clientWriteCb(uv_write_t* req, int status);
static void clientConnCb(uv_connect_t* req, int status); static void clientConnCb(uv_connect_t* req, int status);
static void clientAsyncCb(uv_async_t* handle); static void clientAsyncCb(uv_async_t* handle);
static void clientDestroy(uv_handle_t* handle); static void clientDestroy(uv_handle_t* handle);
static void clientConnDestroy(SCliConn* pConn); static void clientConnDestroy(SCliConn* pConn, bool clear /*clear tcp handle or not*/);
static void clientMsgDestroy(SCliMsg* pMsg); static void clientMsgDestroy(SCliMsg* pMsg);
// handle req from app
static void clientHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd);
// thread obj
static SCliThrdObj* createThrdObj();
static void destroyThrdObj(SCliThrdObj* pThrd);
// thread
static void* clientThread(void* arg); static void* clientThread(void* arg);
static void clientProcessData(SCliConn* conn) { static void clientHandleResp(SCliConn* conn) {
STransConnCtx* pCtx = ((SCliMsg*)conn->data)->ctx; STransConnCtx* pCtx = ((SCliMsg*)conn->data)->ctx;
SRpcInfo* pRpc = pCtx->pRpc; SRpcInfo* pRpc = pCtx->pRpc;
SRpcMsg rpcMsg; SRpcMsg rpcMsg;
@ -101,66 +109,70 @@ static void clientProcessData(SCliConn* conn) {
(pRpc->cfp)(NULL, &rpcMsg, NULL); (pRpc->cfp)(NULL, &rpcMsg, NULL);
SCliThrdObj* pThrd = conn->hostThrd; SCliThrdObj* pThrd = conn->hostThrd;
addConnToCache(pThrd->cache, pCtx->ip, pCtx->port, conn); addConnToPool(pThrd->pool, pCtx->ip, pCtx->port, conn);
if (!uv_is_active((uv_handle_t*)pThrd->pTimer) && pRpc->idleTime > 0) {
uv_timer_start((uv_timer_t*)pThrd->pTimer, clientTimeoutCb, CONN_PERSIST_TIME(pRpc->idleTime) / 2, 0);
}
free(pCtx->ip); free(pCtx->ip);
free(pCtx); free(pCtx);
// impl // impl
} }
static void clientHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd);
static void clientTimeoutCb(uv_timer_t* handle) { static void clientTimeoutCb(uv_timer_t* handle) {
SCliThrdObj* pThrd = handle->data; SCliThrdObj* pThrd = handle->data;
SRpcInfo* pRpc = pThrd->pTransInst; SRpcInfo* pRpc = pThrd->pTransInst;
int64_t currentTime = pThrd->nextTimeout; int64_t currentTime = pThrd->nextTimeout;
tDebug("timeout, try to remove expire conn from conn pool");
SConnList* p = taosHashIterate((SHashObj*)pThrd->cache, NULL); SConnList* p = taosHashIterate((SHashObj*)pThrd->pool, NULL);
while (p != NULL) { while (p != NULL) {
while (!QUEUE_IS_EMPTY(&p->conn)) { while (!QUEUE_IS_EMPTY(&p->conn)) {
queue* h = QUEUE_HEAD(&p->conn); queue* h = QUEUE_HEAD(&p->conn);
SCliConn* c = QUEUE_DATA(h, SCliConn, conn); SCliConn* c = QUEUE_DATA(h, SCliConn, conn);
if (c->expireTime < currentTime) { if (c->expireTime < currentTime) {
QUEUE_REMOVE(h); QUEUE_REMOVE(h);
clientConnDestroy(c); // uv_stream_t stm = *(c->stream);
// uv_close((uv_handle_t*)&stm, clientDestroy);
clientConnDestroy(c, true);
} else { } else {
break; break;
} }
} }
p = taosHashIterate((SHashObj*)pThrd->cache, p); p = taosHashIterate((SHashObj*)pThrd->pool, p);
} }
pThrd->nextTimeout = taosGetTimestampMs() + pRpc->idleTime * 1000 * 10; pThrd->nextTimeout = taosGetTimestampMs() + CONN_PERSIST_TIME(pRpc->idleTime);
uv_timer_start(handle, clientTimeoutCb, pRpc->idleTime * 10, 0); uv_timer_start(handle, clientTimeoutCb, CONN_PERSIST_TIME(pRpc->idleTime) / 2, 0);
} }
static void* connCacheCreate(int size) { static void* creatConnPool(int size) {
SHashObj* cache = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); // thread local, no lock
return cache; return taosHashInit(size, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
} }
static void* connCacheDestroy(void* cache) { static void* destroyConnPool(void* pool) {
SConnList* connList = taosHashIterate((SHashObj*)cache, NULL); SConnList* connList = taosHashIterate((SHashObj*)pool, NULL);
while (connList != NULL) { while (connList != NULL) {
while (!QUEUE_IS_EMPTY(&connList->conn)) { while (!QUEUE_IS_EMPTY(&connList->conn)) {
queue* h = QUEUE_HEAD(&connList->conn); queue* h = QUEUE_HEAD(&connList->conn);
QUEUE_REMOVE(h); QUEUE_REMOVE(h);
SCliConn* c = QUEUE_DATA(h, SCliConn, conn); SCliConn* c = QUEUE_DATA(h, SCliConn, conn);
clientConnDestroy(c); clientConnDestroy(c, true);
} }
connList = taosHashIterate((SHashObj*)cache, connList); connList = taosHashIterate((SHashObj*)pool, connList);
} }
taosHashClear(cache); taosHashClear(pool);
} }
static SCliConn* getConnFromCache(void* cache, char* ip, uint32_t port) { static SCliConn* getConnFromPool(void* pool, char* ip, uint32_t port) {
char key[128] = {0}; char key[128] = {0};
tstrncpy(key, ip, strlen(ip)); tstrncpy(key, ip, strlen(ip));
tstrncpy(key + strlen(key), (char*)(&port), sizeof(port)); tstrncpy(key + strlen(key), (char*)(&port), sizeof(port));
SHashObj* pCache = cache; SHashObj* pPool = pool;
SConnList* plist = taosHashGet(pCache, key, strlen(key)); SConnList* plist = taosHashGet(pPool, key, strlen(key));
if (plist == NULL) { if (plist == NULL) {
SConnList list; SConnList list;
plist = &list; taosHashPut(pPool, key, strlen(key), (void*)&list, sizeof(list));
taosHashPut(pCache, key, strlen(key), plist, sizeof(*plist)); plist = taosHashGet(pPool, key, strlen(key));
plist = taosHashGet(pCache, key, strlen(key));
QUEUE_INIT(&plist->conn); QUEUE_INIT(&plist->conn);
} }
@ -171,14 +183,14 @@ static SCliConn* getConnFromCache(void* cache, char* ip, uint32_t port) {
QUEUE_REMOVE(h); QUEUE_REMOVE(h);
return QUEUE_DATA(h, SCliConn, conn); return QUEUE_DATA(h, SCliConn, conn);
} }
static void addConnToCache(void* cache, char* ip, uint32_t port, SCliConn* conn) { static void addConnToPool(void* pool, char* ip, uint32_t port, SCliConn* conn) {
char key[128] = {0}; char key[128] = {0};
tstrncpy(key, ip, strlen(ip)); tstrncpy(key, ip, strlen(ip));
tstrncpy(key + strlen(key), (char*)(&port), sizeof(port)); tstrncpy(key + strlen(key), (char*)(&port), sizeof(port));
SRpcInfo* pRpc = ((SCliThrdObj*)conn->hostThrd)->pTransInst; SRpcInfo* pRpc = ((SCliThrdObj*)conn->hostThrd)->pTransInst;
conn->expireTime = taosGetTimestampMs() + pRpc->idleTime * 1000 * 10; conn->expireTime = taosGetTimestampMs() + CONN_PERSIST_TIME(pRpc->idleTime);
SConnList* plist = taosHashGet((SHashObj*)cache, key, strlen(key)); SConnList* plist = taosHashGet((SHashObj*)pool, key, strlen(key));
// list already create before // list already create before
assert(plist != NULL); assert(plist != NULL);
QUEUE_PUSH(&plist->conn, &conn->conn); QUEUE_PUSH(&plist->conn, &conn->conn);
@ -235,28 +247,37 @@ static void clientReadCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf
pBuf->len += nread; pBuf->len += nread;
if (clientReadComplete(pBuf)) { if (clientReadComplete(pBuf)) {
tDebug("alread read complete"); tDebug("alread read complete");
clientProcessData(conn); clientHandleResp(conn);
} else { } else {
tDebug("read halp packet, continue to read"); tDebug("read half packet, continue to read");
} }
return; return;
} }
assert(nread <= 0);
if (nread == 0) {
return;
}
if (nread != UV_EOF) { if (nread != UV_EOF) {
tDebug("Read error %s\n", uv_err_name(nread)); tDebug("read error %s", uv_err_name(nread));
} }
// // tDebug("Read error %s\n", uv_err_name(nread));
uv_close((uv_handle_t*)handle, clientDestroy); // uv_close((uv_handle_t*)handle, clientDestroy);
} }
static void clientConnDestroy(SCliConn* conn) { static void clientConnDestroy(SCliConn* conn, bool clear) {
// impl later tDebug("conn %p destroy", conn);
// if (clear) {
uv_close((uv_handle_t*)conn->stream, NULL);
}
free(conn->stream);
free(conn->readBuf.buf);
free(conn->writeReq);
free(conn);
} }
static void clientDestroy(uv_handle_t* handle) { static void clientDestroy(uv_handle_t* handle) {
SCliConn* conn = handle->data; SCliConn* conn = handle->data;
QUEUE_REMOVE(&conn->conn); // QUEUE_REMOVE(&conn->conn);
clientConnDestroy(conn); clientConnDestroy(conn, false);
} }
static void clientWriteCb(uv_write_t* req, int status) { static void clientWriteCb(uv_write_t* req, int status) {
@ -264,7 +285,8 @@ static void clientWriteCb(uv_write_t* req, int status) {
if (status == 0) { if (status == 0) {
tDebug("data already was written on stream"); tDebug("data already was written on stream");
} else { } else {
uv_close((uv_handle_t*)pConn->stream, clientDestroy); tError("failed to write: %s", uv_err_name(status));
clientConnDestroy(pConn, true);
return; return;
} }
SCliThrdObj* pThrd = pConn->hostThrd; SCliThrdObj* pThrd = pConn->hostThrd;
@ -307,7 +329,9 @@ static void clientConnCb(uv_connect_t* req, int status) {
rpcMsg.ahandle = pCtx->ahandle; rpcMsg.ahandle = pCtx->ahandle;
// SRpcInfo* pRpc = pMsg->ctx->pRpc; // SRpcInfo* pRpc = pMsg->ctx->pRpc;
(pRpc->cfp)(NULL, &rpcMsg, NULL); (pRpc->cfp)(NULL, &rpcMsg, NULL);
uv_close((uv_handle_t*)req->handle, clientDestroy);
clientConnDestroy(pConn, true);
// uv_close((uv_handle_t*)req->handle, clientDestroy);
return; return;
} }
@ -322,7 +346,7 @@ static void clientHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) {
et = taosGetTimestampUs(); et = taosGetTimestampUs();
STransConnCtx* pCtx = pMsg->ctx; STransConnCtx* pCtx = pMsg->ctx;
SCliConn* conn = getConnFromCache(pThrd->cache, pCtx->ip, pCtx->port); SCliConn* conn = getConnFromPool(pThrd->pool, pCtx->ip, pCtx->port);
if (conn != NULL) { if (conn != NULL) {
// impl later // impl later
conn->data = pMsg; conn->data = pMsg;
@ -373,15 +397,14 @@ static void clientAsyncCb(uv_async_t* handle) {
SCliMsg* pMsg = QUEUE_DATA(h, SCliMsg, q); SCliMsg* pMsg = QUEUE_DATA(h, SCliMsg, q);
clientHandleReq(pMsg, pThrd); clientHandleReq(pMsg, pThrd);
count++; count++;
if (count >= 2) {
tError("send batch size: %d", count);
} }
if (count >= 2) {
tDebug("already process batch size: %d", count);
} }
} }
static void* clientThread(void* arg) { static void* clientThread(void* arg) {
SCliThrdObj* pThrd = (SCliThrdObj*)arg; SCliThrdObj* pThrd = (SCliThrdObj*)arg;
uv_run(pThrd->loop, UV_RUN_DEFAULT); uv_run(pThrd->loop, UV_RUN_DEFAULT);
} }
@ -394,24 +417,8 @@ void* taosInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads,
cli->pThreadObj = (SCliThrdObj**)calloc(cli->numOfThreads, sizeof(SCliThrdObj*)); cli->pThreadObj = (SCliThrdObj**)calloc(cli->numOfThreads, sizeof(SCliThrdObj*));
for (int i = 0; i < cli->numOfThreads; i++) { for (int i = 0; i < cli->numOfThreads; i++) {
SCliThrdObj* pThrd = (SCliThrdObj*)calloc(1, sizeof(SCliThrdObj)); SCliThrdObj* pThrd = createThrdObj();
pThrd->nextTimeout = taosGetTimestampMs() + CONN_PERSIST_TIME(pRpc->idleTime);
QUEUE_INIT(&pThrd->msg);
pthread_mutex_init(&pThrd->msgMtx, NULL);
pThrd->loop = (uv_loop_t*)malloc(sizeof(uv_loop_t));
uv_loop_init(pThrd->loop);
pThrd->cliAsync = malloc(sizeof(uv_async_t));
uv_async_init(pThrd->loop, pThrd->cliAsync, clientAsyncCb);
pThrd->cliAsync->data = pThrd;
pThrd->pTimer = malloc(sizeof(uv_timer_t));
uv_timer_init(pThrd->loop, pThrd->pTimer);
pThrd->pTimer->data = pThrd;
pThrd->nextTimeout = taosGetTimestampMs() + pRpc->idleTime * 1000 * 10;
pThrd->cache = connCacheCreate(1);
pThrd->pTransInst = shandle; pThrd->pTransInst = shandle;
int err = pthread_create(&pThrd->thread, NULL, clientThread, (void*)(pThrd)); int err = pthread_create(&pThrd->thread, NULL, clientThread, (void*)(pThrd));
@ -426,17 +433,42 @@ static void clientMsgDestroy(SCliMsg* pMsg) {
// impl later // impl later
free(pMsg); free(pMsg);
} }
void taosCloseClient(void* arg) { static SCliThrdObj* createThrdObj() {
// impl later SCliThrdObj* pThrd = (SCliThrdObj*)calloc(1, sizeof(SCliThrdObj));
SClientObj* cli = arg; QUEUE_INIT(&pThrd->msg);
for (int i = 0; i < cli->numOfThreads; i++) { pthread_mutex_init(&pThrd->msgMtx, NULL);
SCliThrdObj* pThrd = cli->pThreadObj[i];
pThrd->loop = (uv_loop_t*)malloc(sizeof(uv_loop_t));
uv_loop_init(pThrd->loop);
pThrd->cliAsync = malloc(sizeof(uv_async_t));
uv_async_init(pThrd->loop, pThrd->cliAsync, clientAsyncCb);
pThrd->cliAsync->data = pThrd;
pThrd->pTimer = malloc(sizeof(uv_timer_t));
uv_timer_init(pThrd->loop, pThrd->pTimer);
pThrd->pTimer->data = pThrd;
pThrd->pool = creatConnPool(1);
return pThrd;
}
static void destroyThrdObj(SCliThrdObj* pThrd) {
if (pThrd == NULL) {
return;
}
pthread_join(pThrd->thread, NULL); pthread_join(pThrd->thread, NULL);
pthread_mutex_destroy(&pThrd->msgMtx); pthread_mutex_destroy(&pThrd->msgMtx);
free(pThrd->cliAsync); free(pThrd->cliAsync);
free(pThrd->loop); free(pThrd->loop);
free(pThrd); free(pThrd);
} }
//
void taosCloseClient(void* arg) {
// impl later
SClientObj* cli = arg;
for (int i = 0; i < cli->numOfThreads; i++) {
destroyThrdObj(cli->pThreadObj[i]);
}
free(cli->pThreadObj); free(cli->pThreadObj);
free(cli); free(cli);
} }

128
source/libs/transport/src/transSrv.c
View File

@ -14,8 +14,8 @@
*/ */
#ifdef USE_UV #ifdef USE_UV
#include "transComm.h"
#include "transComm.h"
typedef struct SConn { typedef struct SConn {
uv_tcp_t* pTcp; uv_tcp_t* pTcp;
uv_write_t* pWriter; uv_write_t* pWriter;
@ -84,17 +84,21 @@ static void uvWorkerAsyncCb(uv_async_t* handle);
static void uvPrepareSendData(SConn* conn, uv_buf_t* wb); static void uvPrepareSendData(SConn* conn, uv_buf_t* wb);
// already read complete packet // check whether already read complete packet
static bool readComplete(SConnBuffer* buf); static bool readComplete(SConnBuffer* buf);
static SConn* createConn();
static void destroyConn(SConn* conn, bool clear /*clear handle or not*/);
static SConn* connCreate(); static void uvDestroyConn(uv_handle_t* handle);
static void connDestroy(SConn* conn);
static void uvConnDestroy(uv_handle_t* handle);
// server worke thread // server and worker thread
static void* workerThread(void* arg); static void* workerThread(void* arg);
static void* acceptThread(void* arg); static void* acceptThread(void* arg);
// add handle loop
static bool addHandleToWorkloop(void* arg);
static bool addHandleToAcceptloop(void* arg);
void uvAllocReadBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) { void uvAllocReadBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) {
/* /*
* formate of data buffer: * formate of data buffer:
@ -268,7 +272,7 @@ static void uvProcessData(SConn* pConn) {
rpcMsg.handle = pConn; rpcMsg.handle = pConn;
(*(pRpc->cfp))(pRpc->parent, &rpcMsg, NULL); (*(pRpc->cfp))(pRpc->parent, &rpcMsg, NULL);
uv_timer_start(pConn->pTimer, uvHandleActivityTimeout, pRpc->idleTime * 10000, 0); // uv_timer_start(pConn->pTimer, uvHandleActivityTimeout, pRpc->idleTime * 10000, 0);
// auth // auth
// validate msg type // validate msg type
} }
@ -279,6 +283,7 @@ void uvOnReadCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) {
SConnBuffer* pBuf = &conn->connBuf; SConnBuffer* pBuf = &conn->connBuf;
if (nread > 0) { if (nread > 0) {
pBuf->len += nread; pBuf->len += nread;
tDebug("on read %p, total read: %d, current read: %d", cli, pBuf->len, (int)nread);
if (readComplete(pBuf)) { if (readComplete(pBuf)) {
tDebug("alread read complete packet"); tDebug("alread read complete packet");
uvProcessData(conn); uvProcessData(conn);
@ -287,11 +292,12 @@ void uvOnReadCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) {
} }
return; return;
} }
if (nread != UV_EOF) { if (nread == 0) {
tDebug("Read error %s", uv_err_name(nread)); return;
} }
if (nread != UV_EOF) {
tDebug("read error %s", uv_err_name(nread)); tDebug("read error %s", uv_err_name(nread));
uv_close((uv_handle_t*)cli, uvConnDestroy); }
} }
void uvAllocConnBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) { void uvAllocConnBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) {
buf->base = malloc(sizeof(char)); buf->base = malloc(sizeof(char));
@ -314,7 +320,7 @@ void uvOnWriteCb(uv_write_t* req, int status) {
tDebug("data already was written on stream"); tDebug("data already was written on stream");
} else { } else {
tDebug("failed to write data, %s", uv_err_name(status)); tDebug("failed to write data, %s", uv_err_name(status));
connDestroy(conn); destroyConn(conn, true);
} }
// opt // opt
} }
@ -327,7 +333,8 @@ static void uvOnPipeWriteCb(uv_write_t* req, int status) {
} }
static void uvPrepareSendData(SConn* conn, uv_buf_t* wb) { static void uvPrepareSendData(SConn* conn, uv_buf_t* wb) {
// impl later // impl later;
tDebug("prepare to send back");
SRpcMsg* pMsg = &conn->sendMsg; SRpcMsg* pMsg = &conn->sendMsg;
if (pMsg->pCont == 0) { if (pMsg->pCont == 0) {
pMsg->pCont = (void*)rpcMallocCont(0); pMsg->pCont = (void*)rpcMallocCont(0);
@ -390,6 +397,7 @@ void uvOnAcceptCb(uv_stream_t* stream, int status) {
uv_write2(wr, (uv_stream_t*)&(pObj->pipe[pObj->workerIdx][0]), &buf, 1, (uv_stream_t*)cli, uvOnPipeWriteCb); uv_write2(wr, (uv_stream_t*)&(pObj->pipe[pObj->workerIdx][0]), &buf, 1, (uv_stream_t*)cli, uvOnPipeWriteCb);
} else { } else {
uv_close((uv_handle_t*)cli, NULL); uv_close((uv_handle_t*)cli, NULL);
free(cli);
} }
} }
void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) { void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) {
@ -399,7 +407,7 @@ void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) {
tError("read error %s", uv_err_name(nread)); tError("read error %s", uv_err_name(nread));
} }
// TODO(log other failure reason) // TODO(log other failure reason)
uv_close((uv_handle_t*)q, NULL); // uv_close((uv_handle_t*)q, NULL);
return; return;
} }
// free memory allocated by // free memory allocated by
@ -418,7 +426,7 @@ void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) {
uv_handle_type pending = uv_pipe_pending_type(pipe); uv_handle_type pending = uv_pipe_pending_type(pipe);
assert(pending == UV_TCP); assert(pending == UV_TCP);
SConn* pConn = connCreate(); SConn* pConn = createConn();
pConn->pTransInst = pThrd->pTransInst; pConn->pTransInst = pThrd->pTransInst;
/* init conn timer*/ /* init conn timer*/
pConn->pTimer = malloc(sizeof(uv_timer_t)); pConn->pTimer = malloc(sizeof(uv_timer_t));
@ -444,29 +452,21 @@ void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) {
uv_read_start((uv_stream_t*)(pConn->pTcp), uvAllocReadBufferCb, uvOnReadCb); uv_read_start((uv_stream_t*)(pConn->pTcp), uvAllocReadBufferCb, uvOnReadCb);
} else { } else {
tDebug("failed to create new connection"); tDebug("failed to create new connection");
connDestroy(pConn); destroyConn(pConn, true);
} }
} }
void* acceptThread(void* arg) { void* acceptThread(void* arg) {
// opt // opt
SServerObj* srv = (SServerObj*)arg; SServerObj* srv = (SServerObj*)arg;
uv_tcp_init(srv->loop, &srv->server);
struct sockaddr_in bind_addr;
uv_ip4_addr("0.0.0.0", srv->port, &bind_addr);
uv_tcp_bind(&srv->server, (const struct sockaddr*)&bind_addr, 0);
int err = 0;
if ((err = uv_listen((uv_stream_t*)&srv->server, 128, uvOnAcceptCb)) != 0) {
tError("Listen error %s\n", uv_err_name(err));
return NULL;
}
uv_run(srv->loop, UV_RUN_DEFAULT); uv_run(srv->loop, UV_RUN_DEFAULT);
} }
static void initWorkThrdObj(SWorkThrdObj* pThrd) { static bool addHandleToWorkloop(void* arg) {
SWorkThrdObj* pThrd = arg;
pThrd->loop = (uv_loop_t*)malloc(sizeof(uv_loop_t)); pThrd->loop = (uv_loop_t*)malloc(sizeof(uv_loop_t));
uv_loop_init(pThrd->loop); if (0 != uv_loop_init(pThrd->loop)) {
return false;
}
// SRpcInfo* pRpc = pThrd->shandle; // SRpcInfo* pRpc = pThrd->shandle;
uv_pipe_init(pThrd->loop, pThrd->pipe, 1); uv_pipe_init(pThrd->loop, pThrd->pipe, 1);
@ -482,13 +482,38 @@ static void initWorkThrdObj(SWorkThrdObj* pThrd) {
pThrd->workerAsync->data = pThrd; pThrd->workerAsync->data = pThrd;
uv_read_start((uv_stream_t*)pThrd->pipe, uvAllocConnBufferCb, uvOnConnectionCb); uv_read_start((uv_stream_t*)pThrd->pipe, uvAllocConnBufferCb, uvOnConnectionCb);
return true;
}
static bool addHandleToAcceptloop(void* arg) {
// impl later
SServerObj* srv = arg;
int err = 0;
if ((err = uv_tcp_init(srv->loop, &srv->server)) != 0) {
tError("failed to init accept server: %s", uv_err_name(err));
return false;
}
struct sockaddr_in bind_addr;
uv_ip4_addr("0.0.0.0", srv->port, &bind_addr);
if ((err = uv_tcp_bind(&srv->server, (const struct sockaddr*)&bind_addr, 0)) != 0) {
tError("failed to bind: %s", uv_err_name(err));
return false;
}
if ((err = uv_listen((uv_stream_t*)&srv->server, 128, uvOnAcceptCb)) != 0) {
tError("failed to listen: %s", uv_err_name(err));
return false;
}
return true;
} }
void* workerThread(void* arg) { void* workerThread(void* arg) {
SWorkThrdObj* pThrd = (SWorkThrdObj*)arg; SWorkThrdObj* pThrd = (SWorkThrdObj*)arg;
uv_run(pThrd->loop, UV_RUN_DEFAULT); uv_run(pThrd->loop, UV_RUN_DEFAULT);
} }
static SConn* connCreate() { static SConn* createConn() {
SConn* pConn = (SConn*)calloc(1, sizeof(SConn)); SConn* pConn = (SConn*)calloc(1, sizeof(SConn));
return pConn; return pConn;
} }
@ -496,22 +521,24 @@ static void connCloseCb(uv_handle_t* handle) {
// impl later // impl later
// //
} }
static void connDestroy(SConn* conn) { static void destroyConn(SConn* conn, bool clear) {
if (conn == NULL) { if (conn == NULL) {
return; return;
} }
if (clear) {
uv_handle_t handle = *((uv_handle_t*)conn->pTcp);
uv_close(&handle, NULL);
}
uv_timer_stop(conn->pTimer); uv_timer_stop(conn->pTimer);
free(conn->pTimer); free(conn->pTimer);
// uv_close((uv_handle_t*)conn->pTcp, connCloseCb);
free(conn->pTcp); free(conn->pTcp);
free(conn->connBuf.buf); free(conn->connBuf.buf);
free(conn->pWriter); free(conn->pWriter);
// free(conn); free(conn);
// handle
} }
static void uvConnDestroy(uv_handle_t* handle) { static void uvDestroyConn(uv_handle_t* handle) {
SConn* conn = handle->data; SConn* conn = handle->data;
connDestroy(conn); destroyConn(conn, false);
} }
static int transAddAuthPart(SConn* pConn, char* msg, int msgLen) { static int transAddAuthPart(SConn* pConn, char* msg, int msgLen) {
STransMsgHead* pHead = (STransMsgHead*)msg; STransMsgHead* pHead = (STransMsgHead*)msg;
@ -546,11 +573,12 @@ void* taosInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads,
for (int i = 0; i < srv->numOfThreads; i++) { for (int i = 0; i < srv->numOfThreads; i++) {
SWorkThrdObj* thrd = (SWorkThrdObj*)calloc(1, sizeof(SWorkThrdObj)); SWorkThrdObj* thrd = (SWorkThrdObj*)calloc(1, sizeof(SWorkThrdObj));
srv->pThreadObj[i] = thrd;
srv->pipe[i] = (uv_pipe_t*)calloc(2, sizeof(uv_pipe_t)); srv->pipe[i] = (uv_pipe_t*)calloc(2, sizeof(uv_pipe_t));
int fds[2]; int fds[2];
if (uv_socketpair(AF_UNIX, SOCK_STREAM, fds, UV_NONBLOCK_PIPE, UV_NONBLOCK_PIPE) != 0) { if (uv_socketpair(AF_UNIX, SOCK_STREAM, fds, UV_NONBLOCK_PIPE, UV_NONBLOCK_PIPE) != 0) {
return NULL; goto End;
} }
uv_pipe_init(srv->loop, &(srv->pipe[i][0]), 1); uv_pipe_init(srv->loop, &(srv->pipe[i][0]), 1);
uv_pipe_open(&(srv->pipe[i][0]), fds[1]); // init write uv_pipe_open(&(srv->pipe[i][0]), fds[1]); // init write
@ -559,7 +587,9 @@ void* taosInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads,
thrd->fd = fds[0]; thrd->fd = fds[0];
thrd->pipe = &(srv->pipe[i][1]); // init read thrd->pipe = &(srv->pipe[i][1]); // init read
initWorkThrdObj(thrd); if (false == addHandleToWorkloop(thrd)) {
goto End;
}
int err = pthread_create(&(thrd->thread), NULL, workerThread, (void*)(thrd)); int err = pthread_create(&(thrd->thread), NULL, workerThread, (void*)(thrd));
if (err == 0) { if (err == 0) {
tDebug("sucess to create worker-thread %d", i); tDebug("sucess to create worker-thread %d", i);
@ -568,9 +598,10 @@ void* taosInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads,
// TODO: clear all other resource later // TODO: clear all other resource later
tError("failed to create worker-thread %d", i); tError("failed to create worker-thread %d", i);
} }
srv->pThreadObj[i] = thrd;
} }
if (false == addHandleToAcceptloop(srv)) {
goto End;
}
int err = pthread_create(&srv->thread, NULL, acceptThread, (void*)srv); int err = pthread_create(&srv->thread, NULL, acceptThread, (void*)srv);
if (err == 0) { if (err == 0) {
tDebug("success to create accept-thread"); tDebug("success to create accept-thread");
@ -579,16 +610,25 @@ void* taosInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads,
} }
return srv; return srv;
End:
taosCloseServer(srv);
return NULL;
}
void destroyWorkThrd(SWorkThrdObj* pThrd) {
if (pThrd == NULL) {
return;
}
pthread_join(pThrd->thread, NULL);
// free(srv->pipe[i]);
free(pThrd->loop);
free(pThrd);
} }
void taosCloseServer(void* arg) { void taosCloseServer(void* arg) {
// impl later // impl later
SServerObj* srv = arg; SServerObj* srv = arg;
for (int i = 0; i < srv->numOfThreads; i++) { for (int i = 0; i < srv->numOfThreads; i++) {
SWorkThrdObj* pThrd = srv->pThreadObj[i]; destroyWorkThrd(srv->pThreadObj[i]);
pthread_join(pThrd->thread, NULL);
free(srv->pipe[i]);
free(pThrd->loop);
free(pThrd);
} }
free(srv->loop); free(srv->loop);
free(srv->pipe); free(srv->pipe);

8
source/libs/transport/test/rclient.c
View File

@ -34,8 +34,8 @@ typedef struct {
static void processResponse(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet) { static void processResponse(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet) {
SInfo *pInfo = (SInfo *)pMsg->ahandle; SInfo *pInfo = (SInfo *)pMsg->ahandle;
// tDebug("thread:%d, response is received, type:%d contLen:%d code:0x%x", pInfo->index, pMsg->msgType, pMsg->contLen, tDebug("thread:%d, response is received, type:%d contLen:%d code:0x%x", pInfo->index, pMsg->msgType, pMsg->contLen,
// pMsg->code); pMsg->code);
if (pEpSet) pInfo->epSet = *pEpSet; if (pEpSet) pInfo->epSet = *pEpSet;
@ -63,6 +63,8 @@ static void *sendRequest(void *param) {
if (pInfo->num % 20000 == 0) tInfo("thread:%d, %d requests have been sent", pInfo->index, pInfo->num); if (pInfo->num % 20000 == 0) tInfo("thread:%d, %d requests have been sent", pInfo->index, pInfo->num);
// tsem_wait(&pInfo->rspSem); // tsem_wait(&pInfo->rspSem);
tsem_wait(&pInfo->rspSem); tsem_wait(&pInfo->rspSem);
tDebug("recv response");
// usleep(100000000);
} }
tDebug("thread:%d, it is over", pInfo->index); tDebug("thread:%d, it is over", pInfo->index);
@ -98,7 +100,7 @@ int main(int argc, char *argv[]) {
rpcInit.numOfThreads = 1; rpcInit.numOfThreads = 1;
rpcInit.cfp = processResponse; rpcInit.cfp = processResponse;
rpcInit.sessions = 100; rpcInit.sessions = 100;
rpcInit.idleTime = tsShellActivityTimer * 1000; rpcInit.idleTime = 100;
rpcInit.user = "michael"; rpcInit.user = "michael";
rpcInit.secret = secret; rpcInit.secret = secret;
rpcInit.ckey = "key"; rpcInit.ckey = "key";

2
source/libs/transport/test/rserver.c
View File

@ -122,7 +122,7 @@ int main(int argc, char *argv[]) {
rpcInit.numOfThreads = 1; rpcInit.numOfThreads = 1;
rpcInit.cfp = processRequestMsg; rpcInit.cfp = processRequestMsg;
rpcInit.sessions = 1000; rpcInit.sessions = 1000;
rpcInit.idleTime = tsShellActivityTimer * 1500; rpcInit.idleTime = 2 * 1500;
rpcInit.afp = retrieveAuthInfo; rpcInit.afp = retrieveAuthInfo;
for (int i = 1; i < argc; ++i) { for (int i = 1; i < argc; ++i) {

2
source/libs/wal/src/walMeta.c
View File

@ -66,7 +66,7 @@ static inline int64_t walScanLogGetLastVer(SWal* pWal) {
struct stat statbuf; struct stat statbuf;
stat(fnameStr, &statbuf); stat(fnameStr, &statbuf);
int readSize = MIN(WAL_MAX_SIZE + 2, statbuf.st_size); int readSize = TMIN(WAL_MAX_SIZE + 2, statbuf.st_size);
pLastFileInfo->fileSize = statbuf.st_size; pLastFileInfo->fileSize = statbuf.st_size;
FileFd fd = taosOpenFileRead(fnameStr); FileFd fd = taosOpenFileRead(fnameStr);

2
source/os/src/osDir.c
View File

@ -98,7 +98,7 @@ void taosRemoveOldFiles(char *dirname, int32_t keepDays) {
} }
if (fileSec <= 100) continue; if (fileSec <= 100) continue;
int32_t days = (int32_t)(ABS(sec - fileSec) / 86400 + 1); int32_t days = (int32_t)(TABS(sec - fileSec) / 86400 + 1);
if (days > keepDays) { if (days > keepDays) {
(void)remove(filename); (void)remove(filename);
//printf("file:%s is removed, days:%d keepDays:%d", filename, days, keepDays); //printf("file:%s is removed, days:%d keepDays:%d", filename, days, keepDays);

2
source/util/src/terror.c
View File

@ -353,7 +353,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_QRY_INVALID_INPUT, "invalid input")
TAOS_DEFINE_ERROR(TSDB_CODE_QRY_SCH_NOT_EXIST, "Scheduler not exist") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_SCH_NOT_EXIST, "Scheduler not exist")
TAOS_DEFINE_ERROR(TSDB_CODE_QRY_TASK_NOT_EXIST, "Task not exist") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_TASK_NOT_EXIST, "Task not exist")
TAOS_DEFINE_ERROR(TSDB_CODE_QRY_TASK_ALREADY_EXIST, "Task already exist") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_TASK_ALREADY_EXIST, "Task already exist")
TAOS_DEFINE_ERROR(TSDB_CODE_QRY_RES_CACHE_NOT_EXIST, "Task result cache not exist") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_TASK_CTX_NOT_EXIST, "Task context not exist")
TAOS_DEFINE_ERROR(TSDB_CODE_QRY_TASK_CANCELLED, "Task cancelled") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_TASK_CANCELLED, "Task cancelled")
TAOS_DEFINE_ERROR(TSDB_CODE_QRY_TASK_DROPPED, "Task dropped") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_TASK_DROPPED, "Task dropped")
TAOS_DEFINE_ERROR(TSDB_CODE_QRY_TASK_CANCELLING, "Task cancelling") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_TASK_CANCELLING, "Task cancelling")

2
source/util/src/thash.c
View File

@ -67,7 +67,7 @@ static FORCE_INLINE void __wr_unlock(void *lock, int32_t type) {
} }
static FORCE_INLINE int32_t taosHashCapacity(int32_t length) { static FORCE_INLINE int32_t taosHashCapacity(int32_t length) {
int32_t len = MIN(length, HASH_MAX_CAPACITY); int32_t len = TMIN(length, HASH_MAX_CAPACITY);
int32_t i = 4; int32_t i = 4;
while (i < len) i = (i << 1u); while (i < len) i = (i << 1u);

2
source/util/src/tlog.c
View File

@ -185,7 +185,7 @@ static void taosKeepOldLog(char *oldName) {
} }
} }
taosRemoveOldFiles(tsLogDir, ABS(tsLogKeepDays)); taosRemoveOldFiles(tsLogDir, TABS(tsLogKeepDays));
} }
static void *taosThreadToOpenNewFile(void *param) { static void *taosThreadToOpenNewFile(void *param) {

4
source/util/src/tstrbuild.c
View File

@ -70,11 +70,11 @@ void taosStringBuilderAppendNull(SStringBuilder* sb) { taosStringBuilderAppendSt
void taosStringBuilderAppendInteger(SStringBuilder* sb, int64_t v) { void taosStringBuilderAppendInteger(SStringBuilder* sb, int64_t v) {
char buf[64]; char buf[64];
size_t len = snprintf(buf, sizeof(buf), "%" PRId64, v); size_t len = snprintf(buf, sizeof(buf), "%" PRId64, v);
taosStringBuilderAppendStringLen(sb, buf, MIN(len, sizeof(buf))); taosStringBuilderAppendStringLen(sb, buf, TMIN(len, sizeof(buf)));
} }
void taosStringBuilderAppendDouble(SStringBuilder* sb, double v) { void taosStringBuilderAppendDouble(SStringBuilder* sb, double v) {
char buf[512]; char buf[512];
size_t len = snprintf(buf, sizeof(buf), "%.9lf", v); size_t len = snprintf(buf, sizeof(buf), "%.9lf", v);
taosStringBuilderAppendStringLen(sb, buf, MIN(len, sizeof(buf))); taosStringBuilderAppendStringLen(sb, buf, TMIN(len, sizeof(buf)));
} }

12
tests/script/jenkins/basic.txt
View File

@ -2,17 +2,17 @@
#======================b1-start=============== #======================b1-start===============
# ---- user # ---- user
./test.sh -f sim/user/basic1.sim ./test.sh -f tsim/user/basic1.sim
# ---- db # ---- db
./test.sh -f sim/db/basic1.sim ./test.sh -f tsim/db/basic1.sim
./test.sh -f sim/db/basic6.sim ./test.sh -f tsim/db/basic6.sim
./test.sh -f sim/db/error1.sim ./test.sh -f tsim/db/error1.sim
# ---- table # ---- table
./test.sh -f sim/table/basic1.sim ./test.sh -f tsim/table/basic1.sim
# ---- dnode # ---- dnode
./test.sh -f sim/dnode/basic1.sim ./test.sh -f tsim/dnode/basic1.sim
#======================b1-end=============== #======================b1-end===============

4
tests/script/sh/deploy.sh
View File

@ -113,8 +113,12 @@ elif [ $NODE -eq 7 ]; then
NODE=7700 NODE=7700
elif [ $NODE -eq 8 ]; then elif [ $NODE -eq 8 ]; then
NODE=7800 NODE=7800
elif [ $NODE -eq 9 ]; then
NODE=7900
fi fi
HOSTNAME=localhost
echo " " >> $TAOS_CFG echo " " >> $TAOS_CFG
echo "firstEp ${HOSTNAME}:7100" >> $TAOS_CFG echo "firstEp ${HOSTNAME}:7100" >> $TAOS_CFG
echo "secondEp ${HOSTNAME}:7200" >> $TAOS_CFG echo "secondEp ${HOSTNAME}:7200" >> $TAOS_CFG

42
tests/script/tmp/prepare.sim
View File

@ -5,7 +5,41 @@ system sh/deploy.sh -n dnode2 -i 2
system sh/deploy.sh -n dnode3 -i 3 system sh/deploy.sh -n dnode3 -i 3
system sh/deploy.sh -n dnode4 -i 4 system sh/deploy.sh -n dnode4 -i 4
system sh/cfg.sh -n dnode1 -c walLevel -v 2 return
system sh/cfg.sh -n dnode2 -c walLevel -v 2
system sh/cfg.sh -n dnode3 -c walLevel -v 2 system sh/deploy.sh -n dnode1 -i 1
system sh/cfg.sh -n dnode4 -c walLevel -v 2 system sh/deploy.sh -n dnode2 -i 2
system sh/deploy.sh -n dnode3 -i 3
system sh/deploy.sh -n dnode4 -i 4
system sh/deploy.sh -n dnode5 -i 5
system sh/deploy.sh -n dnode6 -i 6
system sh/deploy.sh -n dnode7 -i 7
system sh/deploy.sh -n dnode8 -i 8
system sh/deploy.sh -n dnode9 -i 9
system sh/exec.sh -n dnode1 -s start
system sh/exec.sh -n dnode2 -s start
system sh/exec.sh -n dnode3 -s start
system sh/exec.sh -n dnode4 -s start
system sh/exec.sh -n dnode5 -s start
system sh/exec.sh -n dnode6 -s start
system sh/exec.sh -n dnode7 -s start
system sh/exec.sh -n dnode8 -s start
system sh/exec.sh -n dnode9 -s start
sleep 2000
sql create dnode $hostname port 7200
sql create dnode $hostname port 7300
sql create dnode $hostname port 7400
sql create dnode $hostname port 7500
sql create dnode $hostname port 7600
sql create dnode $hostname port 7700
sql create dnode $hostname port 7800
sql create dnode $hostname port 7900
sql show dnodes;
print $data00 $data01
print $data10 $data11
print $data20 $data21
print $data30 $data31
print $data40 $data41

0
tests/script/sim/db/basic1.sim → tests/script/tsim/db/basic1.sim
View File

0
tests/script/sim/db/basic6.sim → tests/script/tsim/db/basic6.sim
View File

0
tests/script/sim/db/error1.sim → tests/script/tsim/db/error1.sim
View File

0
tests/script/sim/dnode/basic1.sim → tests/script/tsim/dnode/basic1.sim
View File

0
tests/script/sim/table/basic1.sim → tests/script/tsim/table/basic1.sim
View File

0
tests/script/sim/user/basic1.sim → tests/script/tsim/user/basic1.sim
View File

58
tests/test/c/create_table.c
View File

@ -31,13 +31,13 @@ int32_t createTable = 1;
int32_t insertData = 0; int32_t insertData = 0;
int32_t batchNumOfTbl = 100; int32_t batchNumOfTbl = 100;
int32_t batchNumOfRow = 1; int32_t batchNumOfRow = 1;
int32_t totalRowsOfPerTbl = 1;
int32_t numOfVgroups = 2; int32_t numOfVgroups = 2;
int32_t showTablesFlag = 0; int32_t showTablesFlag = 0;
int32_t queryFlag = 0; int32_t queryFlag = 0;
int64_t startTimestamp = 1640966400000; // 2020-01-01 00:00:00.000 int64_t startTimestamp = 1640966400000; // 2020-01-01 00:00:00.000
typedef struct { typedef struct {
int64_t tableBeginIndex; int64_t tableBeginIndex;
int64_t tableEndIndex; int64_t tableEndIndex;
@ -89,7 +89,7 @@ void createDbAndStb() {
pRes = taos_query(con, qstr); pRes = taos_query(con, qstr);
code = taos_errno(pRes); code = taos_errno(pRes);
if (code != 0) { if (code != 0) {
pError("failed to use db, code:%d reason:%s", taos_errno(pRes), taos_errstr(pRes)); pError("failed to create stable, code:%d reason:%s", taos_errno(pRes), taos_errstr(pRes));
exit(0); exit(0);
} }
taos_free_result(pRes); taos_free_result(pRes);
@ -107,14 +107,14 @@ void printCreateProgress(SThreadInfo *pInfo, int64_t t) {
totalTables, seconds, speed); totalTables, seconds, speed);
} }
void printInsertProgress(SThreadInfo *pInfo, int64_t t) { void printInsertProgress(SThreadInfo *pInfo, int64_t insertTotalRows) {
int64_t endMs = taosGetTimestampMs(); int64_t endMs = taosGetTimestampMs();
int64_t totalTables = t - pInfo->tableBeginIndex; //int64_t totalTables = t - pInfo->tableBeginIndex;
float seconds = (endMs - pInfo->startMs) / 1000.0; float seconds = (endMs - pInfo->startMs) / 1000.0;
float speed = totalTables / seconds; float speed = insertTotalRows / seconds;
pInfo->insertDataSpeed = speed; pInfo->insertDataSpeed = speed;
pPrint("thread:%d, %" PRId64 " rows inserted, time:%.2f sec, speed:%.1f rows/second, ", pInfo->threadIndex, pPrint("thread:%d, %" PRId64 " rows inserted, time:%.2f sec, speed:%.1f rows/second, ", pInfo->threadIndex,
totalTables, seconds, speed); insertTotalRows, seconds, speed);
} }
static int64_t getResult(TAOS_RES *tres) { static int64_t getResult(TAOS_RES *tres) {
@ -182,18 +182,13 @@ void *threadFunc(void *param) {
exit(1); exit(1);
} }
pPrint("====before thread:%d, table range: %"PRId64 " - %"PRId64 "\n", //pPrint("====before thread:%d, table range: %" PRId64 " - %" PRId64 "\n", pInfo->threadIndex, pInfo->tableBeginIndex,
pInfo->threadIndex, // pInfo->tableEndIndex);
pInfo->tableBeginIndex,
pInfo->tableEndIndex);
pInfo->tableBeginIndex += startOffset; pInfo->tableBeginIndex += startOffset;
pInfo->tableEndIndex += startOffset; pInfo->tableEndIndex += startOffset;
pPrint("====after thread:%d, table range: %"PRId64 " - %"PRId64 "\n", pPrint("====thread:%d, table range: %" PRId64 " - %" PRId64 "\n", pInfo->threadIndex, pInfo->tableBeginIndex, pInfo->tableEndIndex);
pInfo->threadIndex,
pInfo->tableBeginIndex,
pInfo->tableEndIndex);
sprintf(qstr, "use %s", pInfo->dbName); sprintf(qstr, "use %s", pInfo->dbName);
TAOS_RES *pRes = taos_query(con, qstr); TAOS_RES *pRes = taos_query(con, qstr);
@ -221,7 +216,7 @@ void *threadFunc(void *param) {
int64_t startTs = taosGetTimestampUs(); int64_t startTs = taosGetTimestampUs();
TAOS_RES *pRes = taos_query(con, qstr); TAOS_RES *pRes = taos_query(con, qstr);
code = taos_errno(pRes); code = taos_errno(pRes);
if ((code != 0) && (code != TSDB_CODE_RPC_AUTH_REQUIRED)) { if (code != 0) {
pError("failed to create table reason:%s, sql: %s", tstrerror(code), qstr); pError("failed to create table reason:%s, sql: %s", tstrerror(code), qstr);
} }
taos_free_result(pRes); taos_free_result(pRes);
@ -242,30 +237,27 @@ void *threadFunc(void *param) {
} }
if (insertData) { if (insertData) {
int64_t insertTotalRows = 0;
int64_t curMs = 0; int64_t curMs = 0;
int64_t beginMs = taosGetTimestampMs(); int64_t beginMs = taosGetTimestampMs();
pInfo->startMs = beginMs; pInfo->startMs = beginMs;
int64_t t = pInfo->tableBeginIndex; int64_t t = pInfo->tableBeginIndex;
for (; t <= pInfo->tableEndIndex;) { for (; t <= pInfo->tableEndIndex; t++) {
// int64_t batch = (pInfo->tableEndIndex - t); //printf("table name: %"PRId64"\n", t);
// batch = MIN(batch, batchNum);
int32_t len = sprintf(qstr, "insert into ");
for (int32_t i = 0; i < batchNumOfTbl;) {
int64_t ts = startTimestamp; int64_t ts = startTimestamp;
for (int32_t i = 0; i < totalRowsOfPerTbl;) {
int32_t len = sprintf(qstr, "insert into ");
len += sprintf(qstr + len, "%s_t%" PRId64 " values ", stbName, t); len += sprintf(qstr + len, "%s_t%" PRId64 " values ", stbName, t);
for (int32_t j = 0; j < batchNumOfRow; j++) { for (int32_t j = 0; j < batchNumOfRow; j++) {
len += sprintf(qstr + len, "(%" PRId64 ", 6666) ", ts++); len += sprintf(qstr + len, "(%" PRId64 ", 6666) ", ts++);
}
t++;
i++; i++;
if (t > pInfo->tableEndIndex) { insertTotalRows++;
if (i >= totalRowsOfPerTbl) {
break; break;
} }
} }
#if 1
int64_t startTs = taosGetTimestampUs(); int64_t startTs = taosGetTimestampUs();
TAOS_RES *pRes = taos_query(con, qstr); TAOS_RES *pRes = taos_query(con, qstr);
code = taos_errno(pRes); code = taos_errno(pRes);
@ -283,10 +275,12 @@ void *threadFunc(void *param) {
if (curMs - beginMs > 10000) { if (curMs - beginMs > 10000) {
beginMs = curMs; beginMs = curMs;
// printf("==== tableBeginIndex: %"PRId64", t: %"PRId64"\n", pInfo->tableBeginIndex, t); // printf("==== tableBeginIndex: %"PRId64", t: %"PRId64"\n", pInfo->tableBeginIndex, t);
printInsertProgress(pInfo, t); printInsertProgress(pInfo, insertTotalRows);
}
#endif
} }
} }
printInsertProgress(pInfo, t); printInsertProgress(pInfo, insertTotalRows);
} }
taos_close(con); taos_close(con);
@ -324,6 +318,8 @@ void printHelp() {
printf("%s%s%s%d\n", indent, indent, "queryFlag, default is ", queryFlag); printf("%s%s%s%d\n", indent, indent, "queryFlag, default is ", queryFlag);
printf("%s%s\n", indent, "-l"); printf("%s%s\n", indent, "-l");
printf("%s%s%s%d\n", indent, indent, "batchNumOfRow, default is ", batchNumOfRow); printf("%s%s%s%d\n", indent, indent, "batchNumOfRow, default is ", batchNumOfRow);
printf("%s%s\n", indent, "-r");
printf("%s%s%s%d\n", indent, indent, "totalRowsOfPerTbl, default is ", totalRowsOfPerTbl);
exit(EXIT_SUCCESS); exit(EXIT_SUCCESS);
} }
@ -355,6 +351,8 @@ void parseArgument(int32_t argc, char *argv[]) {
batchNumOfTbl = atoi(argv[++i]); batchNumOfTbl = atoi(argv[++i]);
} else if (strcmp(argv[i], "-l") == 0) { } else if (strcmp(argv[i], "-l") == 0) {
batchNumOfRow = atoi(argv[++i]); batchNumOfRow = atoi(argv[++i]);
} else if (strcmp(argv[i], "-r") == 0) {
totalRowsOfPerTbl = atoi(argv[++i]);
} else if (strcmp(argv[i], "-w") == 0) { } else if (strcmp(argv[i], "-w") == 0) {
showTablesFlag = atoi(argv[++i]); showTablesFlag = atoi(argv[++i]);
} else if (strcmp(argv[i], "-q") == 0) { } else if (strcmp(argv[i], "-q") == 0) {
@ -375,6 +373,7 @@ void parseArgument(int32_t argc, char *argv[]) {
pPrint("%s insertData:%d %s", GREEN, insertData, NC); pPrint("%s insertData:%d %s", GREEN, insertData, NC);
pPrint("%s batchNumOfTbl:%d %s", GREEN, batchNumOfTbl, NC); pPrint("%s batchNumOfTbl:%d %s", GREEN, batchNumOfTbl, NC);
pPrint("%s batchNumOfRow:%d %s", GREEN, batchNumOfRow, NC); pPrint("%s batchNumOfRow:%d %s", GREEN, batchNumOfRow, NC);
pPrint("%s totalRowsOfPerTbl:%d %s", GREEN, totalRowsOfPerTbl, NC);
pPrint("%s showTablesFlag:%d %s", GREEN, showTablesFlag, NC); pPrint("%s showTablesFlag:%d %s", GREEN, showTablesFlag, NC);
pPrint("%s queryFlag:%d %s", GREEN, queryFlag, NC); pPrint("%s queryFlag:%d %s", GREEN, queryFlag, NC);
@ -398,7 +397,8 @@ int32_t main(int32_t argc, char *argv[]) {
createDbAndStb(); createDbAndStb();
} }
pPrint("%d threads are spawned to create %" PRId64 " tables, offset is %" PRId64 " ", numOfThreads, numOfTables, startOffset); pPrint("%d threads are spawned to create %" PRId64 " tables, offset is %" PRId64 " ", numOfThreads, numOfTables,
startOffset);
pthread_attr_t thattr; pthread_attr_t thattr;
pthread_attr_init(&thattr); pthread_attr_init(&thattr);

3
tests/tsim/src/simSystem.c
View File

@ -42,7 +42,8 @@ char *simParseArbitratorName(char *varName) {
char *simParseHostName(char *varName) { char *simParseHostName(char *varName) {
static char hostName[140]; static char hostName[140];
sprintf(hostName, "%s", simHostName); //sprintf(hostName, "%s", simHostName);
sprintf(hostName, "%s", "localhost");
return hostName; return hostName;
} }

4
tools/shell/src/backup/tnettest.c
View File

@ -319,7 +319,7 @@ void *taosNetInitRpc(char *secretEncrypt, char spi) {
char user[] = "nettestinternal"; char user[] = "nettestinternal";
char pass[] = "nettestinternal"; char pass[] = "nettestinternal";
taosEncryptPass((uint8_t *)pass, strlen(pass), secretEncrypt); taosEncryptPass_c((uint8_t *)pass, strlen(pass), secretEncrypt);
memset(&rpcInit, 0, sizeof(rpcInit)); memset(&rpcInit, 0, sizeof(rpcInit));
rpcInit.localPort = 0; rpcInit.localPort = 0;
@ -344,7 +344,7 @@ static int32_t taosNetCheckRpc(const char* serverFqdn, uint16_t port, uint16_t p
SRpcMsg rspMsg; SRpcMsg rspMsg;
void * pRpcConn; void * pRpcConn;
char secretEncrypt[32] = {0}; char secretEncrypt[TSDB_PASSWORD_LEN + 1] = {0};
pRpcConn = taosNetInitRpc(secretEncrypt, spi); pRpcConn = taosNetInitRpc(secretEncrypt, spi);
if (NULL == pRpcConn) { if (NULL == pRpcConn) {

30
tools/shell/src/shellEngine.c
View File

@ -751,56 +751,56 @@ static int calcColWidth(TAOS_FIELD *field, int precision) {
switch (field->type) { switch (field->type) {
case TSDB_DATA_TYPE_BOOL: case TSDB_DATA_TYPE_BOOL:
return MAX(5, width); // 'false' return TMAX(5, width); // 'false'
case TSDB_DATA_TYPE_TINYINT: case TSDB_DATA_TYPE_TINYINT:
case TSDB_DATA_TYPE_UTINYINT: case TSDB_DATA_TYPE_UTINYINT:
return MAX(4, width); // '-127' return TMAX(4, width); // '-127'
case TSDB_DATA_TYPE_SMALLINT: case TSDB_DATA_TYPE_SMALLINT:
case TSDB_DATA_TYPE_USMALLINT: case TSDB_DATA_TYPE_USMALLINT:
return MAX(6, width); // '-32767' return TMAX(6, width); // '-32767'
case TSDB_DATA_TYPE_INT: case TSDB_DATA_TYPE_INT:
case TSDB_DATA_TYPE_UINT: case TSDB_DATA_TYPE_UINT:
return MAX(11, width); // '-2147483648' return TMAX(11, width); // '-2147483648'
case TSDB_DATA_TYPE_BIGINT: case TSDB_DATA_TYPE_BIGINT:
case TSDB_DATA_TYPE_UBIGINT: case TSDB_DATA_TYPE_UBIGINT:
return MAX(21, width); // '-9223372036854775807' return TMAX(21, width); // '-9223372036854775807'
case TSDB_DATA_TYPE_FLOAT: case TSDB_DATA_TYPE_FLOAT:
return MAX(20, width); return TMAX(20, width);
case TSDB_DATA_TYPE_DOUBLE: case TSDB_DATA_TYPE_DOUBLE:
return MAX(25, width); return TMAX(25, width);
case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_BINARY:
if (field->bytes > tsMaxBinaryDisplayWidth) { if (field->bytes > tsMaxBinaryDisplayWidth) {
return MAX(tsMaxBinaryDisplayWidth, width); return TMAX(tsMaxBinaryDisplayWidth, width);
} else { } else {
return MAX(field->bytes, width); return TMAX(field->bytes, width);
} }
case TSDB_DATA_TYPE_NCHAR: { case TSDB_DATA_TYPE_NCHAR: {
int16_t bytes = field->bytes * TSDB_NCHAR_SIZE; int16_t bytes = field->bytes * TSDB_NCHAR_SIZE;
if (bytes > tsMaxBinaryDisplayWidth) { if (bytes > tsMaxBinaryDisplayWidth) {
return MAX(tsMaxBinaryDisplayWidth, width); return TMAX(tsMaxBinaryDisplayWidth, width);
} else { } else {
return MAX(bytes, width); return TMAX(bytes, width);
} }
} }
case TSDB_DATA_TYPE_TIMESTAMP: case TSDB_DATA_TYPE_TIMESTAMP:
if (args.is_raw_time) { if (args.is_raw_time) {
return MAX(14, width); return TMAX(14, width);
} }
if (precision == TSDB_TIME_PRECISION_NANO) { if (precision == TSDB_TIME_PRECISION_NANO) {
return MAX(29, width); return TMAX(29, width);
} else if (precision == TSDB_TIME_PRECISION_MICRO) { } else if (precision == TSDB_TIME_PRECISION_MICRO) {
return MAX(26, width); // '2020-01-01 00:00:00.000000' return TMAX(26, width); // '2020-01-01 00:00:00.000000'
} else { } else {
return MAX(23, width); // '2020-01-01 00:00:00.000' return TMAX(23, width); // '2020-01-01 00:00:00.000'
} }
default: default: